Commit 4f78de6a authored by ext-neveu_n's avatar ext-neveu_n
Browse files

Adding option to run on theta at ANL

parent 74f4022d
......@@ -123,12 +123,12 @@ def main(argv):
restart_step = str(getNearestRestartStep(restart_pos,restart_file))
elif arg.startswith("--keep"):
doKeep = True
elif arg.startswith("--nobatch"):
doNobatch = True
elif arg.startswith("--optPilot"):
doOptPilot = True
elif arg.startswith("--queue"):
queue = arg.split("=")[1]
elif arg.startswith("--nobatch"):
doNobatch = True
elif arg.startswith("--optPilot"):
doOptPilot = True
elif arg.startswith("--queue"):
queue = arg.split("=")[1]
elif arg.startswith("--help"):
printUsage()
exit()
......@@ -149,7 +149,7 @@ def main(argv):
if not os.path.isfile(inputfilePath + ".tmpl"):
os.system('cp ' + inputfilePath + '/*.tmpl .')
else:
ext1 ='*.tmpl'
ext1 ='*.tmpl'
if (glob.glob(os.path.join('.',ext1))):
inputfilePath = '../'
else:
......
......@@ -7,6 +7,7 @@ Simulation class handles batch job related things
"""
import sys,os,shutil,commands
import numpy as np
class Simulation:
......@@ -139,6 +140,12 @@ class Simulation:
ram = os.getenv("SLURM_RAM", "36")
partition = os.getenv("SLURM_PARTITION", "merlin")
self.WriteMerlin5(opalexe, oinpFile, CORES, time, ram, info, runfile, partition)
# ANL theta.alcf.anl.gov
elif (hostname.startswith("theta")):
batchsys = 'COBALT'
runfile = 'run.theta'
self.WriteTheta(opalexe, oinpFile, CORES, time, ram, info, queue, hypert)
# ANL blues.lcrc.anl.gov
elif (hostname.startswith("blogin")):
......@@ -393,7 +400,7 @@ class Simulation:
s1 += "cat $PBS_NODEFILE\n"
s1 += "NSLOTS=$(wc -l < $PBS_NODEFILE)\n"
s1 += "####################################################\n"
s1 += "CMD=$OPAL_EXE_PATH/opal \n"
s1 += "CMD=$OPAL_EXE_PATH/opal \n"
s1 += "echo $CMD\n"
s1 += "ARGS=" + "\"" + oinpFile + " --info " + str(info) + " --warn 6 \"\n"
s1 += "####################################################\n"
......@@ -418,7 +425,7 @@ class Simulation:
coresPerNode = 36 * (hypert+1) # hypert == 0 -> no hyper threading
elif (queue=='knl' or queue=='knlall' or queue=='knld'):
print('Running on KNL')
coresPerNode = 32 * (hypert+1)
coresPerNode = 64 * (hypert+1)
else:
print('You have picked a non-valid queue!! Your run will fail!!')
......@@ -463,4 +470,69 @@ class Simulation:
#s1 += "export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK \n"
#s1 += "--hint=nomultithread " + opalexe + " " + oinpFile + "\n"
myfile.write(s1)
\ No newline at end of file
myfile.write(s1)
def WriteTheta(self, opalexe, oinpFile, cores, time, ram, info, queue, hypert):
# queue = default, debug-cache-quad, debug-flat-quad
# cores = min of 8 nodes for default queue
try:
v = os.environ["OPAL_EXE_PATH"]
except KeyError:
print("OPAL_EXE_PATH not set bye bye")
sys.exit(1)
cores = int(cores)
coresPerNode = 64 * (hypert+1)
if (cores % coresPerNode) is 0:
if (cores < coresPerNode):
nodes = 1
else:
nodes = cores / coresPerNode
tasks_per_node = cores/nodes
else:
while((cores % coresPerNode) != 0):
coresPerNode -= 1
nodes = cores/coresPerNode
tasks_per_node = cores/nodes
print(nodes,cores, tasks_per_node)
if cores < 512:
queue = 'debug-cache-quad'
#elif cores > 512:
#nodes = np.ceil(cores/64)
total_mpi_ranks = nodes*coresPerNode
title=oinpFile.partition(".")[0]
myfile = open('run.sh','w')
s1 = "#!/bin/bash \n"
s1 += "#COBALT -t " + time + " \n"
s1 += "#COBALT -n " + str(nodes) + " \n"
s1 += "#COBALT -q " + queue + " \n"
s1 += "#COBALT --attrs mcdram=cache:numa=quad \n"
s1 += "#COBALT -A awa \n"
s1 += 'echo "Starting Cobalt job script"\n'
s1 += "export n_nodes=$COBALT_JOBSIZE \n"
s1 += "export n_mpi_ranks_per_node=" + str(coresPerNode)+ " \n"
s1 += "export n_mpi_ranks=" + str(total_mpi_ranks) + "\n"
#s1 += "export n_openmp_threads_per_rank=4"
#s1 += "export n_hyperthreads_per_core=2"
#s1 += "export n_hyperthreads_skipped_between_ranks=4"
s1 += "####################################################\n"
s1 += "ARGS=" + "\"" + oinpFile + " --info " + str(info) + " --warn 6 \"\n"
s1 += "CMD=$OPAL_EXE_PATH/opal \n"
s1 += "MPICMD=\"aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node $CMD $ARGS\" \n"
s1 += "echo $MPICMD\n"
s1 += "$MPICMD \n"
s1 += "####################################################\n"
myfile.write(s1)
myfile.close()
#aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node \
#--env OMP_NUM_THREADS=$n_openmp_threads_per_rank -cc depth \
# -d $n_hyperthreads_skipped_between_ranks \
# -j $n_hyperthreads_per_core \
# <executable> <executable args>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment