Commit 5109425e authored by ext-neveu_n's avatar ext-neveu_n
Browse files

Making files python3 compatible

parent b311b099
......@@ -31,7 +31,7 @@ class PathNameGenerator:
return path_name
h = hashlib.new('ripemd160')
h.update(path_name)
h.update(path_name.encode('utf-8'))
path_name_hex = h.hexdigest()
self.mapping[path_name_hex] = path_name
......
......@@ -35,8 +35,8 @@ class OpalDict:
def __getitem__(self, key):
return self.dict[key]
def iteritems(self):
return self.dict.iteritems()
def items(self):
return self.dict.items()
def fillDictionary(self, fileName):
fp = open(fileName,"r")
......@@ -65,7 +65,8 @@ class OpalDict:
return self.path_name_generator.compress(dirname)
def scaleDictVar(self, var, scaleWith):
if self.dict.has_key(var):
if var in self.dict:
#if self.dict.has_key(var):
self.dict[var] = float(self.dict[var])*scaleWith
def getType(self,s):
......@@ -92,7 +93,8 @@ class OpalDict:
var = data[2:eqsidx]
rhs = data[eqsidx+1:len(data)-2]
if self.dict.has_key(var):
if var in self.dict:
#if self.dict.has_key(var):
#check if we have a range
if rhs.find(':') > 0:
range = rhs.split(":")
......@@ -103,7 +105,7 @@ class OpalDict:
self.rangevars[var] = rvar
self.numRanges = self.numRanges + 1
else:
print "OpalDict: Range has to be of the form from:to:step!"
print( "OpalDict: Range has to be of the form from:to:step!")
sys.exit(1)
else:
try:
......@@ -116,5 +118,5 @@ class OpalDict:
self.dict[var] = rhs
else:
if var.find("--") < 0: # not a regular option
print 'OpalDict: Key (' + var + ')not found can not add to dictionary, check the OPAL template file'
print( 'OpalDict: Key (' + var + ')not found can not add to dictionary, check the OPAL template file')
sys.exit(1)
......@@ -10,7 +10,7 @@ Script that launches OPAL simulations
"""
import sys,re,os,string,fileinput,shutil,glob
import commands
#import commands
import subprocess
from simulation import Simulation
......@@ -19,7 +19,7 @@ from opaldict import OpalDict
def getNearestRestartStep(restart_pos,fn):
# print 'pos= ',restart_pos,' fn= ',fn
res = commands.getoutput('H5getStep ' + str(restart_pos) + " " + fn);
#res = commands.getoutput('H5getStep ' + str(restart_pos) + " " + fn);
return int(res.split("=")[2])
def getBaseName():
......@@ -31,12 +31,12 @@ def getBaseName():
if templates:
str1 = templates[0]
else:
print 'No template file (.tmpl) found'
print('No template file (.tmpl) found')
sys.exit()
if datafiles:
str2 = datafiles[0]
else:
print 'No data file (.data) found'
print('No data file (.data) found')
sys.exit()
str1spl = str1.split('.')
......@@ -45,24 +45,24 @@ def getBaseName():
name = str2spl[1]
name = name.split('/')[1]
else:
print 'Template and data filename do not match', str1spl, str2spl
print('Template and data filename do not match', str1spl, str2spl)
sys.exit()
return name
def printUsage():
print "./runOPAL.py [--restart-file=FILE [--restart-step=STEPNR | --restart-pos=POS]] [--info=num] [--test] [--keep] [--hypert=num] [ATTR=SCANVALUE] {[ATTR=VALUE]}"
print ""
print "Unit of POS is meter."
print "SCANVALUE=start:end:step, example TFWHM=0.85:0.90:0.01 "
print ""
print "--test does everything but submitting the job"
print "--keep if same simulation has been run before, keep old data and abort"
print "--nobatch run opal locally not using the batch system and waits until the job is done"
print "--optPilot run optPilot (at the moment with default MOGA settings on host merlinl01.psi.ch)"
print "--info=<num> steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output"
print "--queue=<qname> defines in which queue the job goes. Overwrites QUEUE (deprecated SGE_QUEUE)"
print "--hypert=<num> defines the number of Hyper-Threads used. Default 0"
print "Recognized environment variables: DISTRIBUTIONS, FIELDMAPS, OPAL_EXE_PATH, TEMPLATES, QUEUE, RAM, TIME (deprecated SGE_)"
print("./runOPAL.py [--restart-file=FILE [--restart-step=STEPNR | --restart-pos=POS]] [--info=num] [--test] [--keep] [--hypert=num] [ATTR=SCANVALUE] {[ATTR=VALUE]}")
print( "")
print("Unit of POS is meter.")
print("SCANVALUE=start:end:step, example TFWHM=0.85:0.90:0.01 ")
print("")
print("--test does everything but submitting the job")
print("--keep if same simulation has been run before, keep old data and abort")
print("--nobatch run opal locally not using the batch system and waits until the job is done")
print("--optPilot run optPilot (at the moment with default MOGA settings on host merlinl01.psi.ch)")
print("--info=<num> steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output")
print("--queue=<qname> defines in which queue the job goes. Overwrites QUEUE (deprecated SGE_QUEUE)")
print("--hypert=<num> defines the number of Hyper-Threads used. Default 0")
print("Recognized environment variables: DISTRIBUTIONS, FIELDMAPS, OPAL_EXE_PATH, TEMPLATES, QUEUE, RAM, TIME (deprecated SGE_)")
"""
Traverse all possible combinations of range variable values. Start simulation
......@@ -78,7 +78,7 @@ def traverseRanges(list, opaldict, args):
step = head[1][2]
qid = -1
if curval > endval:
print 'range is empty, start value',curval,'needs to be higher than end value',endval
print('range is empty, start value',curval,'needs to be higher than end value',endval)
while curval <= endval:
opaldict[head[0]] = curval
if len(tail) == 0:
......@@ -86,9 +86,9 @@ def traverseRanges(list, opaldict, args):
sim = Simulation(opaldict)
qid = sim.run(*args)
if args[10]:
print "... finished!\n"
print("... finished!\n")
else:
print "SGE-ID= ",qid,'\n'
print("SGE-ID= ",qid,'\n')
else:
traverseRanges(tail, opaldict, args)
curval = curval + step
......@@ -124,21 +124,21 @@ def main(argv):
restart_step = str(getNearestRestartStep(restart_pos,restart_file))
elif arg.startswith("--keep"):
doKeep = True
elif arg.startswith("--nobatch"):
doNobatch = True
elif arg.startswith("--optPilot"):
doOptPilot = True
elif arg.startswith("--queue"):
queue = arg.split("=")[1]
elif arg.startswith("--nobatch"):
doNobatch = True
elif arg.startswith("--optPilot"):
doOptPilot = True
elif arg.startswith("--queue"):
queue = arg.split("=")[1]
elif arg.startswith("--help"):
printUsage()
exit()
elif arg.startswith("--"):
print arg,'is not a valid option, see --help for the available options'
print(arg,'is not a valid option, see --help for the available options')
exit()
if (os.getcwd() == os.environ.get('TEMPLATES')):
print 'Working directory is the same as the TEMPLATES directory! This is not allowed... bye!'
print('Working directory is the same as the TEMPLATES directory! This is not allowed... bye!')
sys.exit()
else:
# make sure we have no junk around
......@@ -150,11 +150,11 @@ def main(argv):
if not os.path.isfile(inputfilePath + ".tmpl"):
os.system('cp ' + inputfilePath + '/*.tmpl .')
else:
ext1 ='*.tmpl'
ext1 ='*.tmpl'
if (glob.glob(os.path.join('.',ext1))):
inputfilePath = '../'
else:
print 'Template file unknown -> exiting ...'
print('Template file unknown -> exiting ...')
sys.exit()
baseFileName = getBaseName()
......@@ -178,16 +178,16 @@ def main(argv):
opt = Runoptpilot(opaldict)
qid = opt.run(N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch)
if doNobatch:
print "... finished!\n"
else:
print "SGE-ID= ",qid,'\n'
print("... finished!\n")
else:
print("SGE-ID= ",qid,'\n')
elif not opaldict.hasRanges():
sim = Simulation(opaldict)
qid = sim.run(N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue, hypert)
if doNobatch:
print "... finished!\n"
else:
print "SGE-ID= ",qid,'\n'
print( "... finished!\n")
else:
print( "SGE-ID= ",qid,'\n')
else:
ranges = opaldict.Range()
......@@ -200,9 +200,9 @@ def main(argv):
# If there's already a directory remove it...
if os.path.isdir(dirname):
if doKeep:
print 'KEEP existing directory ', dirname
print( 'KEEP existing directory ', dirname)
else:
print 'REMOVE existing directory', dirname
print( 'REMOVE existing directory', dirname)
shutil.rmtree(dirname)
# create directory and change to the directory
os.mkdir(dirname)
......@@ -211,7 +211,7 @@ def main(argv):
os.chdir(dirname)
print ranges
print(ranges)
#run simulations of all possible combinations
args = [N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue, hypert]
traverseRanges(ranges.items(), opaldict, args)
......
......@@ -6,7 +6,7 @@
@version: 0.1
"""
import sys,re,os,string,fileinput,shutil,glob,commands
import sys,re,os,string,fileinput,shutil,glob,subprocess #,commands
import time
class Runoptpilot:
......@@ -29,10 +29,10 @@ class Runoptpilot:
# If there's already an file remove it...
if os.path.isdir(self.dirname):
if doKeep:
print 'KEEP existing directory ', self.dirname
print( 'KEEP existing directory ', self.dirname)
return False
else:
print 'REMOVE existing directory ', self.dirname
print( 'REMOVE existing directory ', self.dirname)
shutil.rmtree(self.dirname)
# create directory
......@@ -45,8 +45,10 @@ class Runoptpilot:
if CORES == 0:
CORES = 1
hostname = commands.getoutput("hostname")
#hostname = commands.getoutput("hostname")
p = subprocess.Popen(['hostname'], stdout=subprocess.PIPE)
hostname, err = p.communicate()
time = os.getenv("SGE_TIME", "s_rt=11:59:00,h_rt=12:00:00")
ram = os.getenv("SGE_RAM", "4")
optpilotexepath = os.getenv("OPT_PILOT_EXE_PATH")
......@@ -64,17 +66,17 @@ class Runoptpilot:
mutProb = os.getenv("MUTATIONPROB","0.5")
if (hostname.startswith("merlin")):
print 'Done with setup of the optPilot simulation and executing the job on ' + str(CORES) + ' cores...\n'
print( 'Done with setup of the optPilot simulation and executing the job on ' + str(CORES) + ' cores...\n')
self.WriteSGEOrteMerlin(optpilotexepath, baseFileName, CORES, time, ram, maxGenerations,
initialPopulation, numCoworkers, numMasters, dumpDat,
solSynch, resultDir, tmpDir, mutProb)
if not doTest:
qid = commands.getoutput('qsub -V -q ' + queue + ' -pe openmpi '+str(CORES)+' runopt.sge | awk \'{print $3}\' ')
print 'Done with setup of the optPilot simulation and submitting the job to ',queue, 'with ',CORES,'cores \n'
print( 'Done with setup of the optPilot simulation and submitting the job to ',queue, 'with ',CORES,'cores \n')
else:
print 'Done with setup of the optPilot simulation but no batch sumbmission \n'
print( 'Done with setup of the optPilot simulation but no batch sumbmission \n')
elif not doNobatch:
print "optPilot is running only on Merlin for now\n"
print( "optPilot is running only on Merlin for now\n")
def WriteSGEOrteMerlin(self, optpilotexepath, oinpFile, cores, time, ram, maxGenerations, initialPopulation, numCoworkers, numMasters, dumpDat, solSynch, resultDir, tmpDir, mutProb):
......
......@@ -6,7 +6,7 @@ Simulation class handles batch job related things
@version: 0.1
"""
import sys,re,os,string,fileinput,shutil,glob,commands
import sys,re,os,string,fileinput,shutil,glob, subprocess #,commands
import time
class Simulation:
......@@ -29,11 +29,11 @@ class Simulation:
# If there's already a directory remove it...
if os.path.isdir(self.dirname):
if doKeep:
print 'KEEP existing directory ', self.dirname
print self.dirname
print( 'KEEP existing directory ', self.dirname)
print( self.dirname)
return False
else:
print 'REMOVE existing directory ', self.dirname
print( 'REMOVE existing directory ', self.dirname)
shutil.rmtree(self.dirname)
# create directory
......@@ -55,7 +55,7 @@ class Simulation:
sys.exit(1)
if self.createDirectory(self.dirname, doKeep) == False:
print "Simulation results already exist"
print( "Simulation results already exist")
return
os.chdir(self.dirname)
......@@ -65,7 +65,7 @@ class Simulation:
else:
fieldmapPath = '../fieldmaps'
if not (os.path.isdir(fieldmapPath)):
print 'Fieldmap directory unknown exiting ...'
print( 'Fieldmap directory unknown exiting ...')
sys.exit()
......@@ -89,7 +89,7 @@ class Simulation:
filedata = file.read()
# do the replacements in the templatefile
for s,value in self.opaldict.iteritems():
for s,value in self.opaldict.items():
# Replace the target string
filedata = filedata.replace('_'+s+'_', str(value))
......@@ -102,26 +102,28 @@ class Simulation:
else:
opalexe = 'opal'
print 'Simulation directory is ' + self.dirname + ' using OPAL at ', os.environ.get('OPAL_EXE_PATH')
print 'Using templatefile at ' + inputfilePath
print 'Using fieldmaps at ' + fieldmapPath
print 'Parameter set in ' + oinpFile + ' are:'
print( 'Simulation directory is ' + self.dirname + ' using OPAL at ', os.environ.get('OPAL_EXE_PATH'))
print( 'Using templatefile at ' + inputfilePath)
print( 'Using fieldmaps at ' + fieldmapPath)
print( 'Parameter set in ' + oinpFile + ' are:')
for s, value in sorted(self.opaldict.iteritems()):
print ' :::: ' + s + ' = ' + str(value)
for s, value in sorted(self.opaldict.items()):
print( ' :::: ' + s + ' = ' + str(value))
if not doNobatch:
hostname = commands.getoutput("hostname")
p = subprocess.Popen(['hostname'], stdout=subprocess.PIPE)
hostname, err = p.communicate()
#hostname = commands.getoutput("hostname")
print("On host ",hostname)
if os.getenv("SGE_TIME"):
print "You use deprecated environment variable SGE_TIME. Please use in the future TIME"
print( "You use deprecated environment variable SGE_TIME. Please use in the future TIME")
time = os.getenv("SGE_TIME")
else:
time = os.getenv("TIME", "s_rt=23:59:00,h_rt=24:00:00")
if os.getenv("SGE_RAM"):
print "You use deprecated environment variable SGE_RAM. Please use in the future RAM"
print( "You use deprecated environment variable SGE_RAM. Please use in the future RAM")
ram = os.getenv("SGE_RAM")
else:
ram = os.getenv("RAM", "4")
......@@ -133,7 +135,7 @@ class Simulation:
queue = os.getenv("SGE_QUEUE", "prime_bd.q")
# Merlin5
if (hostname.startswith("merlin-l")):
if (hostname.startswith(b"merlin-l")):
batchsys = 'SLURM'
runfile = 'run.merlin5'
time = os.getenv("SLURM_TIME", "24:00:00")
......@@ -142,27 +144,27 @@ class Simulation:
self.WriteMerlin5(opalexe, oinpFile, CORES, time, ram, info, runfile, partition)
# ANL blues.lcrc.anl.gov
elif (hostname.startswith("blogin")):
elif (hostname.startswith(b"blogin")):
batchsys = 'PBS'
runfile = 'run.blues'
self.WritePBSBlues(opalexe, oinpFile, CORES, time, ram, info, queue)
elif (hostname.startswith("bebop")):
elif (hostname.startswith(b"bebop")):
batchsys = 'SLURM'
runfile = 'run.bebop'
self.WriteBebop(opalexe, oinpFile, CORES, time, ram, info, runfile, queue, hypert)
elif (hostname.startswith("cori")):
elif (hostname.startswith(b"cori")):
batchsys = 'SLURM'
runfile = 'run.cori'
self.WriteCori(opalexe, oinpFile, CORES, time, ram, info, runfile)
elif (hostname.startswith("edison")):
elif (hostname.startswith(b"edison")):
batchsys = 'SLURM'
runfile = 'run.edison'
self.WriteEdison(opalexe, oinpFile, CORES, time, ram, info, runfile)
elif (hostname.startswith("daint")):
elif (hostname.startswith(b"daint")):
batchsys = 'SLURM'
runfile = 'run.daint'
time = os.getenv("SLURM_TIME", "24:00:00")
......@@ -170,7 +172,7 @@ class Simulation:
partition = os.getenv("SLURM_PARTITION", "normal")
self.WritePizDaint(opalexe, oinpFile, CORES, time, ram, info, runfile, partition)
elif (hostname.startswith("eofe")):
elif (hostname.startswith(b"eofe")):
batchsys = 'SLURM'
runfile = 'run.engaging'
time = os.getenv("SLURM_TIME", "24:00:00")
......@@ -182,15 +184,15 @@ class Simulation:
sys.exit(1)
if doTest:
print 'Done with setup of the OPAL simulation but not submitting the job (--test) \n\n\n'
print( 'Done with setup of the OPAL simulation but not submitting the job (--test) \n\n\n')
qid = -1
os.chdir('..')
return qid
if doNobatch:
print 'Done with setup of the OPAL simulation and executing the job on ' + str(CORES) + ' cores...\n\n\n'
print( 'Done with setup of the OPAL simulation and executing the job on ' + str(CORES) + ' cores...\n\n\n')
ofn, fileExtension = os.path.splitext(oinpFile)
print 'STD output is written to ' + ofn + '.out'
print( 'STD output is written to ' + ofn + '.out')
execommand = 'mpirun -np ' + str(CORES) + ' ' + opalexe + ' ' + oinpFile + ' 2>&1 | tee ' + ofn + '.out'
qid = commands.getoutput(execommand)
os.chdir('..')
......@@ -200,10 +202,10 @@ class Simulation:
if batchsys == 'SLURM':
qid = commands.getoutput('sbatch ' + runfile + ' | awk \'{print $3}\' ')
print 'Done with setup of the OPAL simulation and submitting the job with ', CORES, ' cores \n\n\n'
print( 'Done with setup of the OPAL simulation and submitting the job with ', CORES, ' cores \n\n\n')
elif batchsys == 'PBS':
print 'Done with setup of the OPAL simulation, please submit the job yourself'
print( 'Done with setup of the OPAL simulation, please submit the job yourself')
os.chdir('..')
return qid
......@@ -394,7 +396,7 @@ class Simulation:
s1 += "cat $PBS_NODEFILE\n"
s1 += "NSLOTS=$(wc -l < $PBS_NODEFILE)\n"
s1 += "####################################################\n"
s1 += "CMD=$OPAL_EXE_PATH/opal \n"
s1 += "CMD=$OPAL_EXE_PATH/opal \n"
s1 += "echo $CMD\n"
s1 += "ARGS=" + "\"" + oinpFile + " --info " + str(info) + " --warn 6 \"\n"
s1 += "####################################################\n"
......@@ -438,7 +440,7 @@ class Simulation:
nodes = cores/coresPerNode
tasks_per_node = cores/nodes
print(nodes,cores, tasks_per_node)
#print(nodes,cores, tasks_per_node)
title = oinpFile.partition(".")[0]
myfile = open(name, 'w')
......@@ -464,4 +466,4 @@ class Simulation:
#s1 += "export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK \n"
#s1 += "--hint=nomultithread " + opalexe + " " + oinpFile + "\n"
myfile.write(s1)
\ No newline at end of file
myfile.write(s1)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment