#!/bin/bash
#
echo "This is a SAMPLE run script.  Change it to reflect the correct number"
echo "of CPUs/threads, number of nodes, MPI processes per node, etc.."
#

export KMP_AFFINITY=nowarnings,compact
#          This setting is useful for thread pinning
#          You can try it disabled, for some cases (when 4 threads on
#          Intel dual-core based systems) the disabling of pinning helps much.  

export OMP_NUM_THREADS=4
export MKL_NUM_THREADS=4
#          These settings are useful for setting the desired number of 
#          OpenMP threads.  Both parameters should be used and set to 
#          the same value.

#
# You can find description of all Intel MPI parameters in the 
# Intel MPI Reference Manual.
# See <intel mpi installdir>/doc/Reference_manual.pdf
#

export I_MPI_PIN_DOMAIN=omp
#          This setting enables domain creation for OpenMP threads.
#          Each domain has the size equal to the number of threads.
#          This setting is critical for proper hybrid mode performance.
#          Otherwise the threads may be pinned to wrong cores.

date

cp HPL_hybrid.dat HPL.dat
echo -n "This run was done on: "

# Capture some meaningful data for future reference:
echo -n "This run was done on: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
date >> xhpl_hybrid_ipf_dynamic_outputs.txt
echo "HPL.dat: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
cat HPL.dat >> xhpl_hybrid_ipf_dynamic_outputs.txt
echo "Binary name: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
ls -l xhpl_hybrid_ipf_dynamic >> xhpl_hybrid_ipf_dynamic_outputs.txt
echo "This script: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
cat runme_hybrid_ipf_dynamic >> xhpl_hybrid_ipf_dynamic_outputs.txt
echo "Environment variables: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
env >> xhpl_hybrid_ipf_dynamic_outputs.txt
echo "Actual run: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
 
# Environment variables can also be also be set on the Intel MPI command line 
# using the -genv option (to appear before the -np 1):
 
mpiexec -np 1 ./xhpl_hybrid_ipf_dynamic | tee -a xhpl_hybrid_ipf_dynamic_outputs.txt
 
# In case of multiple nodes involved, please set the number of MPI processes 
# per node (ppn=1,2 typically) through the -perhost option (because the 
# default is all cores):
 
# mpiexec -perhost <ppn> -np <n> ./xhpl_hybrid_ipf_dynamic
 
echo -n "Done: " >> xhpl_hybrid_ipf_dynamic_outputs.txt
date >> xhpl_hybrid_ipf_dynamic_outputs.txt
 
echo -n "Done: "
date
