Commit e2529c43 authored by snuverink_j's avatar snuverink_j
Browse files

Merge branch 'anl-nicole'

parents d8d54985 9e7bc61f
......@@ -50,7 +50,7 @@ def getBaseName():
return name
def printUsage():
print "./runOPAL.py [--restart-file=FILE [--restart-step=STEPNR | --restart-pos=POS]] [--info num] [--test] [--keep] [ATTR=SCANVALUE] {[ATTR=VALUE]}"
print "./runOPAL.py [--restart-file=FILE [--restart-step=STEPNR | --restart-pos=POS]] [--info=num] [--test] [--keep] [--hypert=num] [ATTR=SCANVALUE] {[ATTR=VALUE]}"
print ""
print "Unit of POS is meter."
print "SCANVALUE=start:end:step, example TFWHM=0.85:0.90:0.01 "
......@@ -59,8 +59,9 @@ def printUsage():
print "--keep if same simulation has been run before, keep old data and abort"
print "--nobatch run opal locally not using the batch system and waits until the job is done"
print "--optPilot run optPilot (at the moment with default MOGA settings on host merlinl01.psi.ch)"
print "--info <num> steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output"
print "--info=<num> steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output"
print "--queue=<qname> defines in which queue the job goes. Overwrites QUEUE (deprecated SGE_QUEUE)"
print "--hypert=<num> defines the number of Hyper-Threads used. Default 0"
print "Recognized environment variables: DISTRIBUTIONS, FIELDMAPS, OPAL_EXE_PATH, TEMPLATES, QUEUE, RAM, TIME (deprecated SGE_)"
"""
......@@ -106,6 +107,7 @@ def main(argv):
restart_file = ""
queue = ""
info = 6
hypert = 0
qid = -1
for arg in argv:
......@@ -115,6 +117,8 @@ def main(argv):
restart_file = str.lstrip(arg,"--restart-file=")
elif arg.startswith("--restart-step"):
restart_step = arg.split("=")[1]
elif arg.startswith("--hypert"):
hypert = int(arg.split("=")[1])
elif arg.startswith("--restart-pos"):
restart_pos = arg.split("=")[1]
restart_step = str(getNearestRestartStep(restart_pos,restart_file))
......@@ -179,7 +183,7 @@ def main(argv):
print "SGE-ID= ",qid,'\n'
elif not opaldict.hasRanges():
sim = Simulation(opaldict)
qid = sim.run(N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue)
qid = sim.run(N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue, hypert)
if doNobatch:
print "... finished!\n"
else:
......@@ -209,7 +213,7 @@ def main(argv):
print ranges
#run simulations of all possible combinations
args = [N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue]
args = [N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue, hypert]
traverseRanges(ranges.items(), opaldict, args)
# clean up
os.system("rm -f *.bak ")
......
......@@ -127,7 +127,10 @@ class Simulation:
ram = os.getenv("RAM", "4")
if not queue:
queue = os.getenv("SGE_QUEUE", "prime_bd.q")
try:
queue = os.environ.get('QUEUE')
except:
queue = os.getenv("SGE_QUEUE", "prime_bd.q")
# Merlin5
if (hostname.startswith("merlin-l")):
......@@ -144,6 +147,11 @@ class Simulation:
runfile = 'run.blues'
self.WritePBSBlues(opalexe, oinpFile, CORES, time, ram, info, queue)
elif (hostname.startswith("bebop")):
batchsys = 'SLURM'
runfile = 'run.bebop'
self.WriteBebop(opalexe, oinpFile, CORES, time, ram, info, runfile, queue, hypert)
elif (hostname.startswith("cori")):
batchsys = 'SLURM'
runfile = 'run.cori'
......@@ -395,4 +403,65 @@ class Simulation:
s1 += "$MPICMD \n"
s1 += "####################################################\n"
myfile.write(s1)
myfile.close()
myfile.close()
def WriteBebop(self, opalexe, oinpFile, cores, time, ram, info, name, queue, hypert):
# BDW and KNL Compute Nodes at ANL
# http://www.lcrc.anl.gov/for-users/using-lcrc/running-jobs/running-jobs-on-bebop/
if type(cores) is str:
cores = int(cores)
else:
cores = int(cores)
#Checking that a valid queue is selected
#Adjusting number of cores for specified queue
if (queue=='bdw' or queue=='bdwall' or queue=='bdwd'):
print('Running on BDW')
coresPerNode = 36 * (hypert+1) # hypert == 0 -> no hyper threading
elif (queue=='knl' or queue=='knlall' or queue=='knld'):
print('Running on KNL')
coresPerNode = 32 * (hypert+1)
else:
print('You have picked a non-valid queue!! Your run will fail!!')
#Calculating # of nodes needed, and # of tasks per node
# Only calc tasks per node if total core number
# is not evenly divisible by # of nodes
if (cores % coresPerNode) is 0:
if (cores < coresPerNode):
nodes = 1
else:
nodes = cores / coresPerNode
tasks_per_node = cores/nodes
else:
while((cores % coresPerNode) != 0):
coresPerNode -= 1
nodes = cores/coresPerNode
tasks_per_node = cores/nodes
print(nodes,cores, tasks_per_node)
title = oinpFile.partition(".")[0]
myfile = open(name, 'w')
s1 = "#!/bin/bash -l \n"
s1 += "#SBATCH --job-name=" + title + "\n"
s1 += "#SBATCH -o " + title + ".%j.%N.out \n"
s1 += "#SBATCH -e " + title + ".%j.%N.error \n"
s1 += "#SBATCH -p " + queue + " \n"
s1 += "#SBATCH --time=" + time + "\n"
s1 += "#SBATCH --ntasks=" + str(cores) + "\n"
s1 += "#SBATCH --ntasks-per-node=" + str(coresPerNode) + "\n"
s1 += "export I_MPI_SLURM_EXT=0 \n"
s1 += "export I_MPI_FABRICS=shm:tmi \n"
if (queue=='knl' or queue=='knlall' or queue=='knld'):
s1 += "#SBATCH -C knl,quad,cache \n"
if int(nodes) > 1:
s1 += "#SBATCH --ntasks-per-node=" + str(tasks_per_node) + " \n"
s1 += "mpirun -n $SLURM_NTASKS "+ opalexe + " " + oinpFile + "\n"
else:
s1 += "mpirun -n $SLURM_NTASKS " + opalexe + " " + oinpFile + "\n"
#s1 += "#SBATCH --mem=" + ram + "GB \n"
#s1 += "export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK \n"
#s1 += "--hint=nomultithread " + opalexe + " " + oinpFile + "\n"
myfile.write(s1)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment