# Copyright 2016-2019 Intel Corporation
#
# This software and the related documents are Intel copyrighted materials, and
# your use of them is governed by the express license under which they were
# provided to you (License). Unless the License provides otherwise, you may
# not use, modify, copy, publish, distribute, disclose, or transmit this
# software or the related documents without Intel's prior written permission.
#
# This software and the related documents are provided as is, with no express
# or implied warranties, other than those that are expressly stated in the
# License.

function compilervars() {

    _arch_token

    local basearray="/opt/intel"

    for base_entry in ${basearray}
    do
        if [ -e "${base_entry}/bin/compilervars.sh" ] ; then
            source ${base_entry}/bin/compilervars.sh ${_arch_token}
            return 0
        fi

        if [ -e "${base_entry}/psxe_runtime/linux/bin/psxevars.sh" ] ; then
            source ${base_entry}/psxe_runtime/linux/bin/psxevars.sh ${_arch_token}
            return 0
        fi

        if [ -e "${base_entry}/composerxe/bin/compilervars.sh" ] ; then
            source ${base_entry}/composerxe/bin/compilervars.sh ${_arch_token}
            return 0
        fi
    done
    # redefine $base
    local base="/opt/intel/cce"
    local versions=`\find ${base} -maxdepth 1 -regex "${base}/[0-9\.]*" -print 2> /dev/null | awk -F/ '{print $5}'`
    if [[ "$versions" == "" ]] ; then
        _stderr "no version of Intel(R) C++ Compiler found"
        return 1
    fi

    local maxver="0.0.0"
    for version in ${versions}
    do
        _vercmp $version $maxver
        if [ $? == 1 ] ; then
            maxver=$version
        fi
    done

    local iccvars="${base}/${maxver}/bin/iccvars.sh"
    if [ -e ${iccvars} ] ; then
        source ${iccvars} ${_arch_token}
        return 0
    fi

    return 1
}

# Detects presence of all required LSB tools
lsb_tools () {
  get_tool_presence "["
  get_tool_presence "ar"
  get_tool_presence "awk"
  get_tool_presence "basename"
  get_tool_presence "bc"
  get_tool_presence "cat"
  get_tool_presence "chmod"
  get_tool_presence "chown"
  get_tool_presence "cksum"
  get_tool_presence "cmp"
  get_tool_presence "comm"
  get_tool_presence "cp"
  get_tool_presence "csplit"
  get_tool_presence "cut"
  get_tool_presence "date"
  get_tool_presence "dd"
  get_tool_presence "diff"
  get_tool_presence "dirname"
  get_tool_presence "du"
  get_tool_presence "echo"
  get_tool_presence "ed"
  get_tool_presence "egrep"
  get_tool_presence "env"
  get_tool_presence "ex"
  get_tool_presence "expr"
  get_tool_presence "false"
  get_tool_presence "fgrep"
  get_tool_presence "file"
  get_tool_presence "find"
  get_tool_presence "fold"
  get_tool_presence "fuser"
  get_tool_presence "getconf"
  get_tool_presence "grep"
  get_tool_presence "head"
  get_tool_presence "hostname"
  get_tool_presence "iconv"
  get_tool_presence "id"
  get_tool_presence "join"
  get_tool_presence "kill"
  get_tool_presence "killall"
  get_tool_presence "ln"
  get_tool_presence "logname"
  get_tool_presence "ls"
  get_tool_presence "mkdir"
  get_tool_presence "mkfifo"
  get_tool_presence "mktemp"
  get_tool_presence "more"
  get_tool_presence "mv"
  get_tool_presence "nice"
  get_tool_presence "nl"
  get_tool_presence "nohup"
  get_tool_presence "od"
  get_tool_presence "paste"
  get_tool_presence "patch"
  get_tool_presence "pathchk"
  get_tool_presence "pidof"
  get_tool_presence "printf"
  get_tool_presence "ps"
  get_tool_presence "pwd"
  get_tool_presence "rm"
  get_tool_presence "rmdir"
  get_tool_presence "sed"
  get_tool_presence "seq"
  get_tool_presence "sh"
  get_tool_presence "sleep"
  get_tool_presence "sort"
  get_tool_presence "split"
  get_tool_presence "strings"
  get_tool_presence "tail"
  get_tool_presence "tar"
  get_tool_presence "tee"
  get_tool_presence "test"
  get_tool_presence "time"
  get_tool_presence "touch"
  get_tool_presence "tr"
  get_tool_presence "true"
  get_tool_presence "uname"
  get_tool_presence "uniq"
  get_tool_presence "vi"
  get_tool_presence "wc"
  get_tool_presence "xargs"
}

# Defines a variable named $memory_size that contains the memory size
# in megabytes (default).  If not successful, $memory_size = 0 and the
# exit status is 1.
function memory_size() {
    local conversion=1024 # default to MB
    memory_size=0

    # silently ignore unrecognized options
    case "$1" in
        "KB" | "kb")
            conversion=1
            ;;
        "MB" | "mb")
            conversion=1024
            ;;
    esac

    # /proc/meminfo units are KB
    local mem=`\grep "MemTotal:" /proc/meminfo | awk '{printf "%d", $2}'`
    if [[ $? == 0 && -n ${mem} ]] ; then
        memory_size=$((${mem} / ${conversion})) # convert KB to ...
        return 0
    fi

    _stderr "could not determine the memory size"
    return 1
}

# Defines a variable named $memory_size_cluster_min that contains the
# minimum memory size across a set of nodes.  If not successful,
# $memory_size_cluster_min = 0 and the exit status is 1.  Uses pdsh.
function memory_size_cluster_min() {

    local nodefile=$1
    local prefix=$2
    local units=$3
    memory_size_cluster_min=0

    _arch_token

    get_pdsh
    local self="${prefix}/../../functions"

    local mem=`$pdsh -N -w ^$nodefile "bash -c '. $self && memory_size $units && echo \\\$memory_size'" | sort -un | head -1`
    if [[ $? == 0 && -n ${mem} && ${mem} =~ ^[0-9]+$ ]] ; then
        memory_size_cluster_min=$mem
        return 0
    fi

    _stderr "could not determine the minimum cluster memory size"
    return 1
}

# Defines a variable named $memory_size_cluster_min that contains the
# minimum memory size across a set of nodes.  If not successful,
# $memory_size_cluster_min = 0 and the exit status is 1.  Uses mpirun.
function memory_size_cluster_min_mpirun() {
    local nodefile=$1
    local self="$2/../../functions"
    local units=$3
    memory_size_cluster_min=0

    mpivars

    local mem=`mpirun -rr -f $nodefile bash -c ". $self && memory_size $units && echo \\\$memory_size" | grep -E ^[0-9]+$ | sort -un | head -1`
    if [[ $? == 0 && -n ${mem} ]] ; then
        memory_size_cluster_min=$mem
        return 0
    fi

    _stderr "could not determine the minimum cluster memory size"
    return 1
}

# Defines a variable named $min_mic_count that contains the min number
# of coprocessors installed on any one node in the nodefile. If not
# successful min_mic_count will be set to zero and the exit status is 1.  Uses pdsh.
function mic_coprocessor_count_cluster_min() {

    local nodefile=$1
    local prefix=$2

    min_mic_count=0

    _arch_token

    get_pdsh

    local mic_count=`$pdsh -N -w ^$nodefile '/sbin/lspci -x | grep -c "Co-processor: Intel"' | sort | head -n 1`
    if [[ $? == 0 && -n ${mic_count} && ${mic_count} =~ ^-?[0-9]+$ ]] ; then
          min_mic_count=${mic_count}
          return 0
    fi

    _stderr "could not determine if Intel(R) Xeon Phi(TM) coprocessors were installed."
    return 1
}

# Defines a variable named $min_mic_count that contains the min number
# of coprocessors installed on any one node in the nodefile. If not
# successful min_mic_count will be set to zero and the exit status is 1.  Uses mpirun.
function mic_coprocessor_count_cluster_min_mpirun() {
    local nodefile=$1
    min_mic_count=0

    mpivars

    local mic_count=`mpirun -rr -f $nodefile bash -c '/sbin/lspci -x | grep -c "Co-processor: Intel"' | grep -E ^[0-9]+$ | sort | head -1`
    if [[ $? == 0 && -n ${mic_count} ]] ; then
          min_mic_count=${mic_count}
          return 0
    fi

    _stderr "could not determine if Intel(R) Xeon Phi(TM) coprocessors were installed."
    return 1
}

# Defines a variable named $num_logical_cores that contains the
# number of logical cores.  If not successful,
# $num_logical_cores = 0 and the exit status is 1.
function num_logical_cores() {
    local ncores=`lscpu -p=core | grep -v "^#" | wc -l`
    if [[ $? == 0 && -n ${ncores} ]] ; then
        num_logical_cores=$ncores
        return 0
    fi

    _stderr "could not determine the number of logical cores"
    return 1
}

# Defines a variable named $num_physical_cores that contains the
# number of physical (not logical) cores.  If not successful,
# $num_physical_cores = 0 and the exit status is 1.
function num_physical_cores() {
    local ncores=`lscpu -p=core | grep -v "^#" | sort -u | wc -l`
    if [[ $? == 0 && -n ${ncores} ]] ; then
        num_physical_cores=$ncores
        return 0
    fi

    _stderr "could not determine the number of physical cores"
    return 1
}

# Defines a variable named $num_physical_cores_cluster_min that
# contains the minimum number of physical cores across a set of nodes.
# If not successful,$num_physical_cores_cluster_min = 0 and the exit
# status is 1.  Uses pdsh.
function num_physical_cores_cluster_min() {

    local nodefile=$1
    local prefix=$2
    num_physical_cores_cluster_min=0

    _arch_token

    get_pdsh
    local self="${prefix}/../../functions"

    local ncores=`$pdsh -N -w ^$nodefile "bash -c '. $self && num_physical_cores && echo \\\$num_physical_cores'" | sort -un | head -1`
    if [[ $? == 0 && -n ${ncores} && ${ncores} =~ ^[0-9]+$ ]] ; then
        num_physical_cores_cluster_min=$ncores
        return 0
    fi

    _stderr "could not determine the minimum cluster number of physical cores"
    return 1
}

# Defines a variable named $num_physical_cores_cluster_min that
# contains the minimum number of physical cores across a set of nodes.
# If not successful,$num_physical_cores_cluster_min = 0 and the exit
# status is 1.  Uses mpirun.
function num_physical_cores_cluster_min_mpirun() {
    local nodefile=$1
    local self="$2/../../functions"
    num_physical_cores_cluster_min=0

    mpivars

    local ncores=`mpirun -rr -f $nodefile bash -c ". $self && num_physical_cores && echo \\\$num_physical_cores" | grep -E ^[0-9]+$ | sort -un | head -1`
    if [[ $? == 0 && -n ${ncores} ]] ; then
        num_physical_cores_cluster_min=$ncores
        return 0
    fi

    _stderr "could not determine the minimum cluster number of physical cores"
    return 1
}

# Setup the Intel(R) MPI Library shell environment.  The exit status
# is 0 if successful, 1 otherwise.  First tries I_MPI_ROOT, then
# /opt/intel/impi/latest, and if that does not exist, tries
# /opt/intel/impi/x.y.z where x.y.z is the largest (i.e., latest)
# value if multiple versions are installed.  Exports CLCK_I_MPI_ROOT
# to the environment used.
function mpivars() {
    local mpibase="/opt/intel/impi"
    _arch_token

    local mpivars=""

    if [ ! -z ${I_MPI_ROOT+x} ] ; then
        # MPI environment is already defined, use it.
        mpivars="${I_MPI_ROOT}/${_arch_token}/bin/mpivars.sh"
        CLCK_I_MPI_ROOT="${I_MPI_ROOT}"
        return 0
    else
        for base_entry in ${mpibase}
        do
        if [ -e "${base_entry}/latest/${_arch_token}/bin/mpivars.sh" ] ; then
            # /opt/intel/impi/latest exists, use it.
            mpivars="${base_entry}/latest/${_arch_token}/bin/mpivars.sh"
            CLCK_I_MPI_ROOT="${base_entry}/latest"
        else
            # look for the latest version in /opt/intel/impi.
            # looks for ${mpibase}/1.2.3.4 and converts it to 1.2.3.4
            local versions=`\find ${base_entry} -maxdepth 1 -regex "${base_entry}/[0-9\.]*" -print 2> /dev/null | awk -F/ '{print $5}'`
            if [[ "$versions" == "" ]] ; then
                _stderr "no version of Intel(R) MPI Library found"
                continue
            fi

            local maxver="0.0.0"
            for version in ${versions}
            do
                _vercmp $version $maxver
                if [ $? == 1 ] ; then
                    maxver=$version
                fi
            done

            mpivars="${base_entry}/${maxver}/${_arch_token}/bin/mpivars.sh"
            CLCK_I_MPI_ROOT="${base_entry}/${maxver}"
        fi

        if [ -e ${mpivars} ] ; then
            source ${mpivars}
            return 0
        fi
        done
    fi
    _stderr "unable to setup Intel(R) MPI Library environment"
    return 1
}

########
# All functions below this point are prefixed with an underscore.
# This denotes that the functions are private and not intended to be
# used outside this file.  Warning: these functions may change without
# warning.
########

# For the architecture, return the corresponding token used by Intel
# tools, x86_64 -> intel64.
function _arch_token() {
    local arch=`uname -m`

    if [[ $arch != "x86_64" ]] ; then
        _stderr "only intel64 architecture supported"
        return 1;
    fi

    _arch_token="intel64"

    return 0
}

# Echo to STDERR.
function _stderr() {
    echo $* >&2
}

# Compare two version strings of the format a.b.c.d.  Returns 1 if the
# first version is greater, 0 if it less or the same.
function _vercmp() {
    local ver1=`echo $1 | awk -F. '{printf "%03d%03d%03d%03d\n", $1, $2, $3, $4}'`
    local ver2=`echo $2 | awk -F. '{printf "%03d%03d%03d%03d\n", $1, $2, $3, $4}'`

    if [ $ver1 -gt $ver2 ] ; then
        return 1
    else
        return 0
    fi
}


function get_lustre_filesystems {
    LFS_BIN=$1

    #number of filesystems
    num_fs=0

    #filesystems as array of strings
    fs_list=()

    #number of osts per filesystem as array of integers
    num_ost_list=()

    #list of mount points as array of strings
    mount_points=()

    num_osts=0

    #causes for loop to iterate using newline as the delimiter
    IFS=$'\n'

    for line in $(${LFS_BIN} df)
    do
	#if line only contains uuid, then fs count is 0 and fs name is emtied
	echo $line | grep -q "^UUID" &> /dev/null
	if [ $? -eq 0 ]
	then
	    num_osts=0
	fi

	#  if fs count is 0, then set the name and increment it, else increment
	echo $line | grep -q "\[OST:" &> /dev/null
	if [ $? -eq 0 ]
	then
	    if [ $num_osts -eq 0 ]
	    then
		#add filesystem to fs list
		fs=$(echo $line | awk 'BEGIN { FS="-" }; {print $1}')
		fs_list[${num_fs}]=$fs
	    fi
	    num_osts=$((num_osts + 1))
	fi

	#if last line, then add number of osts to list and increment number of
	# filesystems
	echo $line | grep -q "^filesystem summary:" &> /dev/null
	if [ $? -eq 0 ]
	then
	    num_ost_list[${num_fs}]=$num_osts

	    echo $line | awk '{print $NF}' &> /dev/null
	    mount_points[${num_fs}]=$(echo $line | awk '{print $NF}')

	    num_fs=$((num_fs + 1))
	fi
    done

    unset IFS
}

# Defines a variable named $num_threads_per_core that contains the
# number of logical threads per core.  If not successful,
# $num_threads_per_core = 0 and the exit status is 1.
function num_threads_per_core() {
    local ncores=`lscpu -p=core | grep -v "^#" | sort -u | wc -l`
    local ntotalthreads=`lscpu -p=cpu | grep -v "^#" | sort -u | wc -l`
    if [[ $? == 0 && -n ${ncores}  && -n ${ntotalthreads} ]] ; then
        let num_threads_per_core=$ntotalthreads/$ncores
        return 0
    fi

    _stderr "could not determine the number of threads per core"
    return 1
}

# Prints the tool name and fully qualified path in the format:
# <tool_name>: <fully qualified path>
get_tool_presence() {
    if [ $# -eq 2 ]; then
        get_tool_path $1 $2
    else
        get_tool_path $1
    fi

    echo "${tool}: $tool_path"
}

# Defines a variable named tool_path that contains a fully qualified path
# for a given tool/command
get_tool_path() {
    tool=$1
    path="/usr/sbin:/sbin"
    if [ $# -eq 2 ]; then
        path=$2
    fi
    tool_path=`PATH="$PATH":$path \which --skip-functions --skip-alias $tool 2>/dev/null`
    if [ $? != 0 ]; then
        tool_path="CLCK_LOCATION_NOT_FOUND"
    fi
}

# Defines variables  clck_cpu_family, clck_cpu_model and clck_cpu_vendors based
# on /proc/cpuinfo output
function detect_model_family_vendor () {
    clck_cpu_family=`\cat /proc/cpuinfo  | \grep -E "^cpu family\s+:" | \sort -u | \awk '{print $NF}'`
    clck_cpu_model=`\cat /proc/cpuinfo  | \grep -E "^model\s+:" | \sort -u | \awk '{print $NF}'`
    clck_cpu_vendor=`\cat /proc/cpuinfo  | \grep -E "^vendor_id\s+" | \sort -u | \awk '{print $NF}'`

    if [[ $clck_cpu_family == "" || $clck_cpu_model == "" || $clck_cpu_vendor == "" ]] ; then
        _stderr "Could not determine model/family/vendorid of the cpu"
        return 1
    fi
    return 0
}

# Defines variable clck_cpu_type based on model, family, vendor and architecture
function get_cpu_type () {
    _arch_token
    detect_model_family_vendor

    clck_cpu_type="generic"

    # Intel(R) Xeon Phi(TM) x200 CPU
    if [[ $_arch_token == "intel64" ]] ; then
      if [[ $clck_cpu_family == 6 && $clck_cpu_model == 87 && $clck_cpu_vendor == GenuineIntel ]] ; then
        clck_cpu_type="xeonphi"
        return 0
      elif [[ $clck_cpu_family == 6 && $clck_cpu_model == 133 && $clck_cpu_vendor == GenuineIntel ]] ; then
        clck_cpu_type="xeonphi_m"
        return 0
      elif [[ $clck_cpu_family == 6 && $clck_cpu_model == 85 && $clck_cpu_vendor == GenuineIntel ]] ; then
        clck_cpu_type="xeon_scalable"
        return 0
      fi
    fi
    return 1
}

# Setup the Intel(R) Math Kernel Library shell environment.  The exit status
# # is 0 if successful, 1 otherwise.  First tries MKLROOT, then
# # /opt/intel/mkl/. Exports CLCK_MKLROOT to the environment used.
function mklvars() {
   local mklbase="/opt/intel/compilers_and_libraries_2018/linux/mkl/ /opt/intel/compilers_and_libraries_2019/linux/mkl/ /opt/intel/compilers_and_libraries_2020/linux/mkl/ /opt/intel/psxe_runtime_2018/linux/mkl/bin/ /opt/intel/psxe_runtime/linux/mkl/bin/"

   local mklvars=""

   _arch_token

   if [[ ! -z ${MKLROOT+x}  && ! ${MKLROOT} =~ ^[[:graph:]]*201[3-7][[:graph:]]*$ ]] ; then
       # MKL environment is already defined, use it if it is from PSXE2018 or newer.
       mklvars="${MKLROOT}/bin/mklvars.sh"
       CLCK_MKLROOT="${MKLROOT}"
       return 0
   else
       for base_entry in ${mklbase}; do
          if [ -e "${base_entry}/bin/mklvars.sh" ] ; then
              mklvars="${base_entry}/bin/mklvars.sh"
              CLCK_MKLROOT="${base_entry}"
          fi

          if [ -e ${mklvars} ] ; then
             source ${mklvars} ${_arch_token}
             return 0
          fi
       done
   fi
   _stderr "unable to setup Intel(R) MKL Library environment"
   return 1
}

# Defines the variable pdsh contained in the pdsh variable.
# If successful returns 0 otherwise returns 1
function get_pdsh() {
   CURRENT_DIR=$(dirname $0)

   PATH_TO_PDSH="$CURRENT_DIR/../../../libexec/intel64"

   if [[ $PATH_TO_PDSH != 0 ]] ; then
     pdsh="$PATH_TO_PDSH/pdsh"
     return 0
   fi

   _stderr "could not determine the pdsh path"
   return 1
}

# Defines a variable named $num_sockets that contains the
# number of sockets.  If not successful,
# $num_sockets = 0 and the exit status is 1.
function num_sockets() {
    local nsockets=`lscpu -p=socket | grep -v "^#" | sort -u | wc -l`
    if [[ $? == 0 && -n ${nsockets} ]] ; then
        num_sockets=$nsockets
        return 0
    fi

    _stderr "could not determine the number of sockets"
    return 1
}



# Defines a variable named $num_sockets_cluster_min that
# contains the minimum number of sockets across a set of nodes.
# If not successful,$num_sockets_cluster_min = 0 and the exit
# status is 1.
function num_sockets_cluster_min() {

    local nodefile=$1
    local prefix=$2
    num_sockets_cluster_min=0

    _arch_token

    get_pdsh
    local self="${prefix}/../../functions"

    local nsockets=`$pdsh -N -w ^$nodefile "bash -c '. $self && num_sockets && echo \\\$num_sockets'" | sort -un | head -1`
    echo $nsockets
    if [[ $? == 0 && -n ${nsockets} && ${nsockets} =~ ^[0-9]+$ ]] ; then
        num_sockets_cluster_min=$nsockets
        return 0
    fi

    _stderr "could not determine the minimum cluster number of sockets"
    return 1
}
