##******************************************************************************
##                              INTEL CONFIDENTIAL
##  Copyright(C) 2005-2010 Intel Corporation. All Rights Reserved.
##  The source code contained  or  described herein and all documents related to
##  the source code ("Material") are owned by Intel Corporation or its suppliers
##  or licensors.  Title to the  Material remains with  Intel Corporation or its
##  suppliers and licensors. The Material contains trade secrets and proprietary
##  and  confidential  information of  Intel or its suppliers and licensors. The
##  Material  is  protected  by  worldwide  copyright  and trade secret laws and
##  treaty  provisions. No part of the Material may be used, copied, reproduced,
##  modified, published, uploaded, posted, transmitted, distributed or disclosed
##  in any way without Intel's prior express written permission.
##  No license  under any  patent, copyright, trade secret or other intellectual
##  property right is granted to or conferred upon you by disclosure or delivery
##  of the Materials,  either expressly, by implication, inducement, estoppel or
##  otherwise.  Any  license  under  such  intellectual property  rights must be
##  express and approved by Intel in writing.
##
##******************************************************************************
##  Content:
##     Intel(R) Math Kernel Library Fortran-style Cluster DFT
##     examples creation and run
##******************************************************************************

help:
	@echo "Usage: make {lib32|lib64|libem64t} [example=name] [mpi=intel3|intel4|mpich|mpich2|openmpi]"
	@echo "[compiler=intel|gnu] [interface=interface_name] [workdir=path] [mpidir=path]"
	@echo
	@echo "example=name - name of example. Please see names in cdftf.lst file."
	@echo "If parameter is omitted, all examples will be compiled."
	@echo
	@echo "mpi=intel3  - using Intel (R) MPI Library 3.x, default"
	@echo "mpi=intel4  - using Intel (R) MPI Library 4.x"
	@echo "mpi=mpich   - using MPICH 1.2.x"
	@echo "mpi=mpich2  - using MPICH2 1.0.x"
	@echo "mpi=openmpi - using Open MPI 1.2"
	@echo
	@echo "compiler=intel - using Intel (R) Fortran Compiler, default"
	@echo "compiler=gnu   - using GNU gfortran compiler"
	@echo "interface_name - can be lp64 or ilp64 for em64t and ia64. Default value is lp64."
	@echo
	@echo "workdir=path - path to work directory, which is accessible from any node."
	@echo "If parameter is omitted, executable files and results will be located in"
	@echo "current directory."
	@echo
	@echo "mpidir=path - path to MPI installation directory. MPI scripts are taken"
	@echo "from mpidir/bin (or mpidir/bin64 for Intel (R) MPI Library and Intel(R) 64 architecture). If this directory"
	@echo "is in PATH you can omit mpidir parameter."
	@echo "MPICH should be compiled by the compiler with same version as compiler used to build examples."
	@echo
	@echo "Set environment variables LD_LIBRARY_PATH and etc properly before testing."

##------------------------------------------------------------------------------
## examples of using:
##
## make lib32 example=dm_complex_2d_double_ex1 workdir=/share
##                 - build and run DM_COMPLEX_2D_DOUBLE_EX1
##                   example for 32-bit applications in
##                   dir /share, static linking, using
##                   Intel (R) MPI Library 3.x, Intel(R) Fortran Compiler
##
## make lib64 mpi=mpich compiler=intel mpidir=/opt/mpich
##                 - build and run all examples of CDFT for
##                   Intel(R) Itanium(R) processor family applications
##                   in current dir, static linking, using
##                   MPICH 1.2.x from /opt/mpich,
##                   Intel(R) Fortran Compiler
##------------------------------------------------------------------------------

include cdftf.lst

ifndef example
   example = $(CDFT)
endif

ifndef mpi
   mpi = intel3
endif

ifndef compiler
   compiler = intel
endif

ifndef workdir
   workdir = .
endif

ifndef interface
   interface=lp64
endif

override threading=parallel

RES = $(addsuffix .res,$(example))

ifndef MKLROOT
   MKLROOT=../..
   MKL_PATH = "$(subst examples/cdftf,lib/$(_IA),$(PWD))"
else
   MKL_PATH = "$(MKLROOT)/lib/$(_IA)"
endif

FOPTS =

ifeq ($(compiler),intel)
    IFACE_COMP_PART=intel
    IFACE_THREADING_PART=intel
    MOD_DIR_OPTION=-module
    OMP_LIB = -openmp
    CPP_OPT = -cpp
    FOPTS += -I. -warn all
endif

ifeq ($(compiler),gnu)
    IFACE_COMP_PART=gf
    IFACE_THREADING_PART=gnu
    MOD_DIR_OPTION=-M
    OMP_LIB = -fopenmp
    CPP_OPT = -x f95-cpp-input
    FOPTS += -I. -Wall
endif

ifeq ($(interface),ilp64)
   IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_ilp64.$(EXT)
   ifeq ($(compiler),intel)
       FOPTS += -i8
   endif
   ifeq ($(compiler),gnu)
       FOPTS += -fdefault-integer-8
   endif
   BLACS_PART=_ilp64
else
   IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_lp64.$(EXT)
   BLACS_PART=_lp64
endif

ifeq ($(_IA),32)
   ifeq ($(compiler),intel)
      SPEC_OPT=-xK
#This option is required by Intel(R) 11.0 compiler to produce workable binaries for Pentium(R) III.
#If you don't need it, you can remove this option.
   endif
   IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART).$(EXT)
   FOPTS =
   BLACS_PART=
endif

THREADING_LIB=$(MKL_PATH)/libmkl_$(IFACE_THREADING_PART)_thread.$(EXT)

CORE_LIB=$(MKL_PATH)/libmkl_core.$(EXT)

MKL_LIBS= -Wl,--start-group $(MKL_PATH)/libmkl_cdft_core.a $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) -Wl,--end-group $(OMP_LIB)

ifeq ($(_IA),32)
   RES_DIR=_results/$(compiler)_$(_IA)_$(RES_EXT)$Z
else
   RES_DIR=_results/$(compiler)_$(interface)_$(_IA)_$(RES_EXT)$Z
endif

lib32:
	$(MAKE) $(RES) _IA=32 Ibin=bin EXT=a RES_EXT=lib
libem64t:
	$(MAKE) $(RES) _IA=em64t Ibin=bin64 EXT=a RES_EXT=lib
lib64:
	$(MAKE) $(RES) _IA=64 Ibin=bin EXT=a RES_EXT=lib

ifeq ($(mpi),mpich2)
   ifeq ($(compiler),intel)
      CS = mpif90 -f90=ifort $(FOPTS)
      MPI_FMT=-fixed
   endif
   ifeq ($(compiler),gnu)
      CS = mpif90 -f90=gfortran $(FOPTS)
      MPI_FMT=-ffixed-form
   endif
   FOPTS += -DMPI_KIND_=4
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
   DelF = rm $(RES_DIR)/*.mod
endif

ifeq ($(mpi),intel3)
   ifeq ($(compiler),intel)
      CS = mpiifort $(FOPTS)
      MPI_FMT=-fixed
   endif
   ifeq ($(compiler),gnu)
      CS = mpif90 -f90=gfortran $(FOPTS)
      MPI_FMT=-ffixed-form
   endif
   FOPTS += -DMPI_KIND_=4
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/$(Ibin)/$(CS)
      _RS = $(mpidir)/$(Ibin)/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
   DelF = rm $(RES_DIR)/*.mod
endif

ifeq ($(mpi),intel4)
   ifeq ($(compiler),intel)
      CS = mpiifort $(FOPTS)
      MPI_FMT=-fixed
   endif
   ifeq ($(compiler),gnu)
      CS = mpif90 -f90=gfortran $(FOPTS)
      MPI_FMT=-ffixed-form
   endif
   ifeq ($(interface),ilp64)
       ifeq ($(compiler),intel)
           # -i8 enables Intel MPI 4.x ilp64 interface
           FOPTS += -DMPI_KIND_=8
       else
           FOPTS += -DMPI_KIND_=4
       endif
   else
       FOPTS += -DMPI_KIND_=4
   endif
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/$(Ibin)/$(CS)
      _RS = $(mpidir)/$(Ibin)/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
   DelF = rm $(RES_DIR)/*.mod
endif

ifeq ($(mpi),mpich)
   ifeq ($(compiler),intel)
      CS = mpif90 -f90=ifort $(FOPTS)
      MPI_FMT=-fixed
   endif
   ifeq ($(compiler),gnu)
      CS = mpif90 -f90=gfortran $(FOPTS)
      MPI_FMT=-ffixed-form
   endif
   FOPTS += -DMPI_KIND_=4
   RS = mpirun -np 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   DelF = rm $(RES_DIR)/*.o $(RES_DIR)/*.mod
endif

ifeq ($(mpi),openmpi)
   ifeq ($(compiler),intel)
      CS = mpif90 -f90=ifort $(FOPTS)
      MPI_FMT=-fixed
   endif
   ifeq ($(compiler),gnu)
      CS = mpif90 -f90=gfortran $(FOPTS)
      MPI_FMT=-ffixed-form
   endif
   FOPTS += -DMPI_KIND_=4
   RS = mpiexec -np 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _openmpi
   DelF = rm $(RES_DIR)/*.o $(RES_DIR)/*.mod
endif

COMMON = "$(MKLROOT)"/include/mkl_cdft.f90 source/cdft_example_support.f90

#-------------------------------------------------------------------------------

vpath %.f90 source

$(RES): %.res: %.f90
	mkdir -p $(RES_DIR)
	$(_CS) $(SPEC_OPT) $(CPP_OPT) "$(MKLROOT)"/include/mkl_cdft.f90 -c $(MOD_DIR_OPTION) $(RES_DIR) -o $(RES_DIR)/mkl_cdft.o
	$(_CS) $(SPEC_OPT) $(CPP_OPT) $(MPI_FMT) source/cdft_example_support.f90 -c $(MOD_DIR_OPTION) $(RES_DIR) -o $(RES_DIR)/cdft_example_support.o
	$(_CS) $(SPEC_OPT) $(CPP_OPT) $(MPI_FMT) $< -c $(MOD_DIR_OPTION) $(RES_DIR) -o $(RES_DIR)/$*.o
	$(_CS) $(SPEC_OPT) $(RES_DIR)/$*.o $(RES_DIR)/cdft_example_support.o $(RES_DIR)/mkl_cdft.o -L $(MKL_PATH) $(MKL_LIBS) $(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).$(EXT) -lpthread -lm -o $(RES_DIR)/$*.exe
	export LD_LIBRARY_PATH=$(MKL_PATH):"$(LD_LIBRARY_PATH)"; $(_RS) $(RES_DIR)/$*.exe < data/$*.dat > $(RES_DIR)/$@
	-$(DelF)
#-------------------------------------------------------------------------------
