##******************************************************************************
##                              INTEL CONFIDENTIAL
##  Copyright(C) 2006-2010 Intel Corporation. All Rights Reserved.
##  The source code contained  or  described herein and all documents related to
##  the source code ("Material") are owned by Intel Corporation or its suppliers
##  or licensors.  Title to the  Material remains with  Intel Corporation or its
##  suppliers and licensors. The Material contains trade secrets and proprietary
##  and  confidential  information of  Intel or its suppliers and licensors. The
##  Material  is  protected  by  worldwide  copyright  and trade secret laws and
##  treaty  provisions. No part of the Material may be used, copied, reproduced,
##  modified, published, uploaded, posted, transmitted, distributed or disclosed
##  in any way without Intel's prior express written permission.
##  No license  under any  patent, copyright, trade secret or other intellectual
##  property right is granted to or conferred upon you by disclosure or delivery
##  of the Materials,  either expressly, by implication, inducement, estoppel or
##  otherwise.  Any  license  under  such  intellectual property  rights must be
##  express and approved by Intel in writing.
##
##******************************************************************************
##  Content:
##  Intel(R) Math Kernel Library
##  C-style Cluster DFT wrappers MPI FFTW examples creation and run
##******************************************************************************

help:
	@echo "Usage: make {lib32|lib64|libem64t} [mpi=intel3|mpich|mpich2|openmpi]"
	@echo "[compiler=intel|gnu] [workdir=path] [mpidir=path]"
	@echo "[precision=FFTW_ENABLE_DOUBLE|FFTW_ENABLE_FLOAT]"
	@echo
	@echo "mpi=intel3 - using Intel (R) MPI Library 3.x, default"
	@echo "mpi=mpich  - using MPICH 1.2.x"
	@echo "mpi=mpich2 - using MPICH2 1.0.x"
	@echo "mpi=openmpi - using Open MPI 1.2"
	@echo
	@echo "compiler=intel - using Intel(R) C Compiler, default"
	@echo "compiler=gnu   - using GNU C compiler"
	@echo
	@echo "workdir=path - path to work directory, which is accessible from any node."
	@echo "If parameter is omitted, executable files and results will be located in"
	@echo "current directory."
	@echo
	@echo "precision=FFTW_ENABLE_DOUBLE - for double precision data, default"
	@echo "precision=FFTW_ENABLE_FLOAT - for single precision data"
	@echo
	@echo "mpidir=path - path to MPI installation directory. MPI scripts are taken"
	@echo "from mpidir/bin (or mpidir/bin64 for Intel (R) MPI Library and Intel(R) 64 architecture). If this directory"
	@echo "is in PATH you can omit mpidir parameter. When using MPICH, make sure"
	@echo "that MPICH was compiled by same compiler as used to build examples."
	@echo
	@echo "Set environment variables LD_LIBRARY_PATH and etc properly before testing."

##*****************************************************************************
## examples of using:
##
## make lib64 mpi=intel3 compiler=intel
##              - compile and run wrappers examples
##                using Intel (R) MPI Library 2.0,Intel(R) C Compiler,
##                for double precision data for
##                Intel(R) Itanium(R)-based applications
##
## make lib32 precision=FFTW_ENABLE_FLOAT mpi=mpich mpidir=/opt/mpich compiler=gnu
##              - compile and run wrappers examples using
##                MPICH 1.2.x from /opt/mpich, GNU C compiler
##                for single precision data for 32-bit applications
##*****************************************************************************

include cdft_wrappers.lst

example = $(WRAPPER)

ifndef mpi
   mpi = intel3
endif

ifndef compiler
   compiler = intel
endif

ifndef workdir
   workdir = .
endif

ifndef precision
   precision = FFTW_ENABLE_DOUBLE
endif

ifeq ($(precision),FFTW_ENABLE_FLOAT)
   FL = -DFFTW_ENABLE_FLOAT
   PR = _SINGLE
   PR_L = MKL_SINGLE
endif

ifeq ($(precision),FFTW_ENABLE_DOUBLE)
   FL =
   PR = _DOUBLE
   PR_L = MKL_DOUBLE
endif

RES = $(addsuffix .res,$(example))

ifndef MKLROOT
   MKLROOT=../..
   MKL_PATH = "$(subst examples/fftw2x_cdft,lib/$(_IA),$(PWD))"
else
   MKL_PATH = "$(MKLROOT)/lib/$(_IA)"
endif

INSTALL_DIR=$(PWD)

COPTS =

ifeq ($(_IA),32)
   ifeq ($(compiler),intel)
      SPEC_OPT=-xK
#This option is required by Intel(R) 11.0 compiler to produce workable binaries for Pentium(R) III.
#If you don't need it, you can remove this option.
   endif
   IFACE_LIB=$(MKL_PATH)/libmkl_intel.$(EXT)
   BLACS_PART=
else
   IFACE_LIB=$(MKL_PATH)/libmkl_intel_lp64.$(EXT)
   BLACS_PART=_lp64
endif

THREADING_LIB=$(MKL_PATH)/libmkl_intel_thread.$(EXT)
OMP_LIB = -L$(MKL_PATH) -liomp5

CORE_LIB=$(MKL_PATH)/libmkl_core.$(EXT)

MKL_LIBS= -Wl,--start-group $(MKL_PATH)/libmkl_cdft_core.$(EXT) $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) -lmkl_blacs$(Bs)$(BLACS_PART) -Wl,--end-group $(OMP_LIB)

ifeq ($(_IA),32)
   RES_DIR=_results/$(compiler)_$(_IA)_$(RES_EXT)$Z
else
   RES_DIR=_results/$(compiler)_$(_IA)_$(RES_EXT)$Z
endif

ifeq ($(mpi),mpich2)
   ifeq ($(compiler),intel)
      CS = mpicc -cc=icc
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -cc=gcc
   endif
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
   export OMP_NUM_THREADS=1
endif

ifeq ($(mpi),intel3)
   ifeq ($(compiler),intel)
      CS = mpiicc -mt_mpi
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -mt_mpi
   endif
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/$(Ibin)/$(CS)
      _RS = $(mpidir)/$(Ibin)/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
endif

ifeq ($(mpi),mpich)
   ifeq ($(compiler),intel)
      CS = mpicc -cc=icc
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -cc=gcc
   endif
   RS = mpirun -np 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   DelF = rm $(RES_DIR)/*.o
   export OMP_NUM_THREADS=1
endif

ifeq ($(mpi),openmpi)
   ifeq ($(compiler),intel)
      CS = mpicc -cc=icc $(COPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -cc=gcc  $(COPTS)
   endif
   RS = mpiexec -np 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _openmpi
   DelF = rm $(RES_DIR)/*.o
endif

#-------------------------------------------------------------------------------

vpath %.c source

lib32:
	$(MAKE) wrap_lib _IA=32
	$(MAKE) $(RES) _IA=32 Ibin=bin EXT=a RES_EXT=lib
libem64t:
	$(MAKE) wrap_lib _IA=em64t
	$(MAKE) $(RES) _IA=em64t Ibin=bin64 EXT=a RES_EXT=lib
lib64:
	$(MAKE) wrap_lib _IA=64
	$(MAKE) $(RES) _IA=64 Ibin=bin EXT=a RES_EXT=lib

wrap_lib:
	cd "$(MKLROOT)"/interfaces/fftw2x_cdft && $(MAKE) lib$(_IA) mpi=$(mpi) compiler=$(compiler) mpidir=$(mpidir) PRECISION=$(PR_L) INSTALL_DIR=$(INSTALL_DIR)/lib/$(_IA) SPEC_OPT=$(SPEC_OPT)

$(RES): %.res: %.c
	mkdir -p $(RES_DIR)
	$(_CS) $(SPEC_OPT) $(FL) $< -I $(MKLROOT)/include -I $(MKLROOT)/include/fftw -c -o $(RES_DIR)/$*.o
	$(_CS) $(SPEC_OPT) $(FL) $(RES_DIR)/$*.o -O0 -L"$(INSTALL_DIR)/lib/$(_IA)" -lfftw2x_cdft$(PR) -L $(MKLROOT)/lib/$(_IA) $(MKL_LIBS) -lpthread -lm -o $(RES_DIR)/$*.exe
	export LD_LIBRARY_PATH=$(MKL_PATH):"$(LD_LIBRARY_PATH)"; $(_RS) $(RES_DIR)/$*.exe > $(RES_DIR)/$@;
	-$(DelF)
#-------------------------------------------------------------------------------
