#!/bin/bash
echo o comando: $@
#SBATCH --nodes=1                #Número de Nós
#SBATCH --ntasks=8               #Numero total de tarefas MPI
#SBATCH -p batch                 #Fila (partition) a ser utilizada
#SBATCH -J atmosphere_model      #Nome job
#SBATCH --time=01:20:00          #Obrigatório
##SBATCH --ntasks-per-node=4     #Número de tarefas por Nó

export SBATCH_DEBUG=1 

hpctkOptions=""
hpctkOptions="hpcrun -e CPUTIME -t";

function main() {
   settings
   modulesLoad
   mpasConfig

   if [ ".$hpctkOptions." != ".." ]
   then
    echo COM HPCTOOLKIT
    hpctoolkitLoad
    resultdir=${mpastype}/hpctoolkit/partition-${SLURM_JOB_PARTITION}/NUMNODES-$SLURM_JOB_NUM_NODES/MPI-${SLURM_NTASKS}/JOBID-${SLURM_JOBID}
   else
    echo SEM HPCTOOLKIT
    resultdir=${mpastype}/partition-${SLURM_JOB_PARTITION}/NUMNODES-$SLURM_JOB_NUM_NODES/MPI-${SLURM_NTASKS}/JOBID-${SLURM_JOBID}
   fi 

   comandoMPIRUN="mpirun -np $SLURM_NTASKS ./${executable} "
   echo $comandoMPIRUN
   eval $comandoMPIRUN

   posprocessing
   if [ ".$hpctkOptions." != ".." ]
   then
     hpctoolkitPosprocessing; 
   fi 
   return
}

function  hpctoolkitLoad(){
  export PATH=$PATH:/mnt/beegfs/denis.eiras/spack/spack_ufs/opt/spack/linux-rhel8-zen2/gcc-9.4.0/hpctoolkit-master-bwxocdkn7yvjkl2uff3eu43j57pe4cm6/bin/
#   spack load hpctoolkit@2021.10.15
#   spack load --list
}

function hpctoolkitPosprocessing(){
   echo em hpctoolkitPosprocessing +++

   comandoHPCTK="hpcstruct ${executable}"
   echo $comandoHPCTK; eval $comandoHPCTK
   comandoHPCTK="hpcprof \
 -I /mnt/beegfs/denis.eiras/monan_github_issues_workspace/128/mpas_src/MPAS-Model_v6.3_egeon.spack_v0.18.1_oneapi/+ \
 -S ${executable}.hpcstruct hpctoolkit-atmosphere_model-measurements-${SLURM_JOBID}"
   echo $comandoHPCTK; eval $comandoHPCTK

   comandoMk="mkdir -p ${resultdir}"
   echo $comandoMk; eval $comandoMk
   comandoMV="mv hpctoolkit-atmosphere_model-database-${SLURM_JOBID}     ${resultdir}"
   echo $comandoMV; eval $comandoMV
   comandoMV="mv hpctoolkit-atmosphere_model-measurements-${SLURM_JOBID} ${resultdir}"
   echo $comandoMV; eval $comandoMV
   comandoMV="mv ${executable}.hpcstruct                                 ${resultdir}"
   echo $comandoMV; eval $comandoMV
}

function settings(){
   mpastype=${1:-"v6.3_single"}
   echo "mpastype="$mpastype
   gpuranks=0
   dynranks=${2:-"$gpuranks"}
   echo "dynranks="$dynranks
   rad_ranksD=$((SLURM_NTASKS-${gpuranks}))
   echo "rad_ranksD="$rad_ranksD
   radranks=${3:-$rad_ranksD}
   echo "rad_ranksD="$rad_ranksD
   arqSequanaD=24
   ppr=${4:-"$arqSequanaD"}
   executable=atmosphere_model_${mpastype}
   cd $SLURM_SUBMIT_DIR
}


function  modulesLoad(){
  . ./1_source_spack_env_mpas_intel_impi_egeon.sh
  . ./2_source_env_egeon_intel.sh
  spack env status
  spack load --only dependencies mpas-model%intel@2021.4.0
}

function  mpasConfig(){
   export MPAS_DYNAMICS_RANKS_PER_NODE=${dynranks}
   export MPAS_RADIATION_RANKS_PER_NODE=${radranks}
   echo
   echo "MPAS_DYNAMICS_RANKS_PER_NODE: $MPAS_DYNAMICS_RANKS_PER_NODE"
   echo "MPAS_RADIATION_RANKS_PER_NODE: $MPAS_RADIATION_RANKS_PER_NODE"
   echo
}

function posprocessing(){
   echo em posprocessing +++
   comandoMk="mkdir -p ${resultdir}"
   echo $comandoMk; eval $comandoMk
   comandoCP="cp slurm-${SLURM_JOBID}.out ${resultdir}"
   echo $comandoCP; eval $comandoCP
   comandoMV="mv log.atmosphere.*.out     ${resultdir}"
   echo $comandoMV; eval $comandoMV
#   mv diag.*                   ${resultdir}/ 
}

   echo "Iniciando no diretÃrio "$PWD
   dataInicio=$(date)
    start_time="$(date -u +%s)"
         main
    end_time="$(date -u +%s)"
   dataFinal=$(date)
   echo " +++ inicio dAS simulacao: $dataInicio"
   echo " +++ fim .. dAS simulacao: $dataFinal "
   elapsed="$(($end_time-$start_time))"
   echo "Total of $elapsed seconds elapsed"