Commit 96300ab0 authored by snuverink_j's avatar snuverink_j
Browse files

merge with master

parents c68aa395 4f78de6a
......@@ -8,6 +8,7 @@ Simulation class handles batch job related things
import sys,re,os,string,fileinput,shutil,glob, subprocess #,commands
import time
import numpy as np
class Simulation:
......@@ -141,6 +142,12 @@ class Simulation:
ram = os.getenv("SLURM_RAM", "36")
partition = os.getenv("SLURM_PARTITION", "merlin")
self.WriteMerlin5(opalexe, oinpFile, CORES, time, ram, info, runfile, partition)
# ANL theta.alcf.anl.gov
elif (hostname.startswith("theta")):
batchsys = 'COBALT'
runfile = 'run.theta'
self.WriteTheta(opalexe, oinpFile, CORES, time, ram, info, queue, hypert)
# ANL blues.lcrc.anl.gov
elif (hostname.startswith("blogin")):
......@@ -424,7 +431,7 @@ class Simulation:
coresPerNode = 36 * (hypert+1) # hypert == 0 -> no hyper threading
elif (queue=='knl' or queue=='knlall' or queue=='knld'):
print('Running on KNL')
coresPerNode = 32 * (hypert+1)
coresPerNode = 64 * (hypert+1)
else:
print('You have picked a non-valid queue!! Your run will fail!!')
......@@ -470,3 +477,67 @@ class Simulation:
#s1 += "--hint=nomultithread " + opalexe + " " + oinpFile + "\n"
myfile.write(s1)
def WriteTheta(self, opalexe, oinpFile, cores, time, ram, info, queue, hypert):
# queue = default, debug-cache-quad, debug-flat-quad
# cores = min of 8 nodes for default queue
try:
v = os.environ["OPAL_EXE_PATH"]
except KeyError:
print("OPAL_EXE_PATH not set bye bye")
sys.exit(1)
cores = int(cores)
coresPerNode = 64 * (hypert+1)
if (cores % coresPerNode) is 0:
if (cores < coresPerNode):
nodes = 1
else:
nodes = cores / coresPerNode
tasks_per_node = cores/nodes
else:
while((cores % coresPerNode) != 0):
coresPerNode -= 1
nodes = cores/coresPerNode
tasks_per_node = cores/nodes
print(nodes,cores, tasks_per_node)
if cores < 512:
queue = 'debug-cache-quad'
#elif cores > 512:
#nodes = np.ceil(cores/64)
total_mpi_ranks = nodes*coresPerNode
title=oinpFile.partition(".")[0]
myfile = open('run.sh','w')
s1 = "#!/bin/bash \n"
s1 += "#COBALT -t " + time + " \n"
s1 += "#COBALT -n " + str(nodes) + " \n"
s1 += "#COBALT -q " + queue + " \n"
s1 += "#COBALT --attrs mcdram=cache:numa=quad \n"
s1 += "#COBALT -A awa \n"
s1 += 'echo "Starting Cobalt job script"\n'
s1 += "export n_nodes=$COBALT_JOBSIZE \n"
s1 += "export n_mpi_ranks_per_node=" + str(coresPerNode)+ " \n"
s1 += "export n_mpi_ranks=" + str(total_mpi_ranks) + "\n"
#s1 += "export n_openmp_threads_per_rank=4"
#s1 += "export n_hyperthreads_per_core=2"
#s1 += "export n_hyperthreads_skipped_between_ranks=4"
s1 += "####################################################\n"
s1 += "ARGS=" + "\"" + oinpFile + " --info " + str(info) + " --warn 6 \"\n"
s1 += "CMD=$OPAL_EXE_PATH/opal \n"
s1 += "MPICMD=\"aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node $CMD $ARGS\" \n"
s1 += "echo $MPICMD\n"
s1 += "$MPICMD \n"
s1 += "####################################################\n"
myfile.write(s1)
myfile.close()
#aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node \
#--env OMP_NUM_THREADS=$n_openmp_threads_per_rank -cc depth \
# -d $n_hyperthreads_skipped_between_ranks \
# -j $n_hyperthreads_per_core \
# <executable> <executable args>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment