Commit c2111cbc authored by snuverink_j's avatar snuverink_j
Browse files

Merge branch 'python3' into 'master'

Python3

Closes #2

See merge request !2
parents 4f78de6a 96300ab0
......@@ -31,7 +31,7 @@ class PathNameGenerator:
return path_name
h = hashlib.new('ripemd160')
h.update(path_name)
h.update(path_name.encode('utf-8'))
path_name_hex = h.hexdigest()
self.mapping[path_name_hex] = path_name
......
......@@ -35,8 +35,8 @@ class OpalDict:
def __getitem__(self, key):
return self.dict[key]
def iteritems(self):
return self.dict.iteritems()
def items(self):
return self.dict.items()
def fillDictionary(self, fileName):
fp = open(fileName,"r")
......@@ -65,7 +65,8 @@ class OpalDict:
return self.path_name_generator.compress(dirname)
def scaleDictVar(self, var, scaleWith):
if self.dict.has_key(var):
if var in self.dict:
#if self.dict.has_key(var):
self.dict[var] = float(self.dict[var])*scaleWith
def getType(self,s):
......@@ -92,7 +93,8 @@ class OpalDict:
var = data[2:eqsidx]
rhs = data[eqsidx+1:len(data)-2]
if self.dict.has_key(var):
if var in self.dict:
#if self.dict.has_key(var):
#check if we have a range
if rhs.find(':') > 0:
range = rhs.split(":")
......@@ -103,7 +105,7 @@ class OpalDict:
self.rangevars[var] = rvar
self.numRanges = self.numRanges + 1
else:
print "OpalDict: Range has to be of the form from:to:step!"
print( "OpalDict: Range has to be of the form from:to:step!")
sys.exit(1)
else:
try:
......@@ -116,5 +118,5 @@ class OpalDict:
self.dict[var] = rhs
else:
if var.find("--") < 0: # not a regular option
print 'OpalDict: Key (' + var + ')not found can not add to dictionary, check the OPAL template file'
print( 'OpalDict: Key (' + var + ')not found can not add to dictionary, check the OPAL template file')
sys.exit(1)
#!/usr/bin/python
#!/usr/bin/env python
"""
Script that launches OPAL simulations
......@@ -9,16 +9,17 @@ Script that launches OPAL simulations
"""
import sys,os,shutil,glob
import commands
import sys,re,os,string,fileinput,shutil,glob
#import commands
import subprocess
from simulation import Simulation
from runoptpilot import Runoptpilot
from opaldict import OpalDict
def getNearestRestartStep(restart_pos,fn):
# print 'pos= ',restart_pos,' fn= ',fn
res = commands.getoutput('H5getStep ' + str(restart_pos) + " " + fn);
print('pos= ',restart_pos,' fn= ',fn)
res = subprocess.check_output('H5getStep ' + str(restart_pos) + " " + fn);
return int(res.split("=")[2])
def getBaseName():
......@@ -30,12 +31,12 @@ def getBaseName():
if templates:
str1 = templates[0]
else:
print 'No template file (.tmpl) found'
print('No template file (.tmpl) found')
sys.exit()
if datafiles:
str2 = datafiles[0]
else:
print 'No data file (.data) found'
print('No data file (.data) found')
sys.exit()
str1spl = str1.split('.')
......@@ -44,24 +45,24 @@ def getBaseName():
name = str2spl[1]
name = name.split('/')[1]
else:
print 'Template and data filename do not match', str1spl, str2spl
print('Template and data filename do not match', str1spl, str2spl)
sys.exit()
return name
def printUsage():
print "./runOPAL.py [--restart-file=FILE [--restart-step=STEPNR | --restart-pos=POS]] [--info=num] [--test] [--keep] [--hypert=num] [ATTR=SCANVALUE] {[ATTR=VALUE]}"
print ""
print "Unit of POS is meter."
print "SCANVALUE=start:end:step, example TFWHM=0.85:0.90:0.01 "
print ""
print "--test does everything but submitting the job"
print "--keep if same simulation has been run before, keep old data and abort"
print "--nobatch run opal locally not using the batch system and waits until the job is done"
print "--optPilot run optPilot (at the moment with default MOGA settings on host merlin-l-01.psi.ch)"
print "--info=<num> steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output"
print "--queue=<qname> defines in which queue the job goes. Overwrites QUEUE (deprecated SGE_QUEUE)"
print "--hypert=<num> defines the number of Hyper-Threads used. Default 0"
print "Recognized environment variables: DISTRIBUTIONS, FIELDMAPS, OPAL_EXE_PATH, TEMPLATES, QUEUE, RAM, TIME (deprecated SGE_)"
print("./runOPAL.py [--restart-file=FILE [--restart-step=STEPNR | --restart-pos=POS]] [--info=num] [--test] [--keep] [--hypert=num] [ATTR=SCANVALUE] {[ATTR=VALUE]}")
print( "")
print("Unit of POS is meter.")
print("SCANVALUE=start:end:step, example TFWHM=0.85:0.90:0.01 ")
print("")
print("--test does everything but submitting the job")
print("--keep if same simulation has been run before, keep old data and abort")
print("--nobatch run opal locally not using the batch system and waits until the job is done")
print("--optPilot run optPilot (at the moment with default MOGA settings on host merlinl01.psi.ch)")
print("--info=<num> steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output")
print("--queue=<qname> defines in which queue the job goes. Overwrites QUEUE (deprecated SGE_QUEUE)")
print("--hypert=<num> defines the number of Hyper-Threads used. Default 0")
print("Recognized environment variables: DISTRIBUTIONS, FIELDMAPS, OPAL_EXE_PATH, TEMPLATES, QUEUE, RAM, TIME (deprecated SGE_)")
"""
Traverse all possible combinations of range variable values. Start simulation
......@@ -77,7 +78,7 @@ def traverseRanges(list, opaldict, args):
step = head[1][2]
qid = -1
if curval > endval:
print 'range is empty, start value',curval,'needs to be higher than end value',endval
print('range is empty, start value',curval,'needs to be higher than end value',endval)
while curval <= endval:
opaldict[head[0]] = curval
if len(tail) == 0:
......@@ -85,9 +86,9 @@ def traverseRanges(list, opaldict, args):
sim = Simulation(opaldict)
qid = sim.run(*args)
if args[10]:
print "... finished!\n"
print("... finished!\n")
else:
print "SGE-ID= ",qid,'\n'
print("SGE-ID= {}\n".format(qid))
else:
traverseRanges(tail, opaldict, args)
curval = curval + step
......@@ -133,11 +134,11 @@ def main(argv):
printUsage()
exit()
elif arg.startswith("--"):
print arg,'is not a valid option, see --help for the available options'
print(arg,'is not a valid option, see --help for the available options')
exit()
if (os.getcwd() == os.environ.get('TEMPLATES')):
print 'Working directory is the same as the TEMPLATES directory! This is not allowed... bye!'
print('Working directory is the same as the TEMPLATES directory! This is not allowed... bye!')
sys.exit()
else:
# make sure we have no junk around
......@@ -153,7 +154,7 @@ def main(argv):
if (glob.glob(os.path.join('.',ext1))):
inputfilePath = '../'
else:
print 'Template file unknown -> exiting ...'
print('Template file unknown -> exiting ...')
sys.exit()
baseFileName = getBaseName()
......@@ -177,16 +178,16 @@ def main(argv):
opt = Runoptpilot(opaldict)
qid = opt.run(N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch)
if doNobatch:
print "... finished!\n"
else:
print "SGE-ID= ",qid,'\n'
print("... finished!\n")
else:
print("SGE-ID= {}\n".format(qid))
elif not opaldict.hasRanges():
sim = Simulation(opaldict)
qid = sim.run(N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue, hypert)
if doNobatch:
print "... finished!\n"
else:
print "SGE-ID= ",qid,'\n'
print( "... finished!\n")
else:
print( "SGE-ID= {}\n".format(qid))
else:
ranges = opaldict.Range()
......@@ -199,9 +200,9 @@ def main(argv):
# If there's already a directory remove it...
if os.path.isdir(dirname):
if doKeep:
print 'KEEP existing directory ', dirname
print( 'KEEP existing directory ', dirname)
else:
print 'REMOVE existing directory', dirname
print( 'REMOVE existing directory', dirname)
shutil.rmtree(dirname)
# create directory and change to the directory
os.mkdir(dirname)
......@@ -210,7 +211,7 @@ def main(argv):
os.chdir(dirname)
print ranges
print(ranges)
#run simulations of all possible combinations
args = [N, baseFileName, restart_step, inputfilePath, tmplFile, oinpFile, restart_file, doTest, doKeep, doNobatch, info, queue, hypert]
traverseRanges(ranges.items(), opaldict, args)
......
......@@ -6,7 +6,7 @@
@version: 0.1
"""
import sys,re,os,string,fileinput,shutil,glob,commands
import sys,re,os,string,fileinput,shutil,glob,subprocess #,commands
import time
class Runoptpilot:
......@@ -29,10 +29,10 @@ class Runoptpilot:
# If there's already an file remove it...
if os.path.isdir(self.dirname):
if doKeep:
print 'KEEP existing directory ', self.dirname
print( 'KEEP existing directory ', self.dirname)
return False
else:
print 'REMOVE existing directory ', self.dirname
print( 'REMOVE existing directory ', self.dirname)
shutil.rmtree(self.dirname)
# create directory
......@@ -45,8 +45,10 @@ class Runoptpilot:
if CORES == 0:
CORES = 1
hostname = commands.getoutput("hostname")
#hostname = commands.getoutput("hostname")
p = subprocess.Popen(['hostname'], stdout=subprocess.PIPE)
hostname, err = p.communicate()
time = os.getenv("SGE_TIME", "s_rt=11:59:00,h_rt=12:00:00")
ram = os.getenv("SGE_RAM", "4")
optpilotexepath = os.getenv("OPT_PILOT_EXE_PATH")
......@@ -64,17 +66,17 @@ class Runoptpilot:
mutProb = os.getenv("MUTATIONPROB","0.5")
if (hostname.startswith("merlin")):
print 'Done with setup of the optPilot simulation and executing the job on ' + str(CORES) + ' cores...\n'
print( 'Done with setup of the optPilot simulation and executing the job on ' + str(CORES) + ' cores...\n')
self.WriteSGEOrteMerlin(optpilotexepath, baseFileName, CORES, time, ram, maxGenerations,
initialPopulation, numCoworkers, numMasters, dumpDat,
solSynch, resultDir, tmpDir, mutProb)
if not doTest:
qid = commands.getoutput('qsub -V -q ' + queue + ' -pe openmpi '+str(CORES)+' runopt.sge | awk \'{print $3}\' ')
print 'Done with setup of the optPilot simulation and submitting the job to ',queue, 'with ',CORES,'cores \n'
print( 'Done with setup of the optPilot simulation and submitting the job to ',queue, 'with ',CORES,'cores \n')
else:
print 'Done with setup of the optPilot simulation but no batch sumbmission \n'
print( 'Done with setup of the optPilot simulation but no batch sumbmission \n')
elif not doNobatch:
print "optPilot is running only on Merlin for now\n"
print( "optPilot is running only on Merlin for now\n")
def WriteSGEOrteMerlin(self, optpilotexepath, oinpFile, cores, time, ram, maxGenerations, initialPopulation, numCoworkers, numMasters, dumpDat, solSynch, resultDir, tmpDir, mutProb):
......
......@@ -6,7 +6,8 @@ Simulation class handles batch job related things
@version: 0.1
"""
import sys,os,shutil,commands
import sys,re,os,string,fileinput,shutil,glob, subprocess #,commands
import time
import numpy as np
class Simulation:
......@@ -29,11 +30,11 @@ class Simulation:
# If there's already a directory remove it...
if os.path.isdir(self.dirname):
if doKeep:
print 'KEEP existing directory ', self.dirname
print self.dirname
print( 'KEEP existing directory {}'.format(self.dirname))
print( self.dirname)
return False
else:
print 'REMOVE existing directory ', self.dirname
print( 'REMOVE existing directory {}'.format(self.dirname))
shutil.rmtree(self.dirname)
# create directory
......@@ -55,7 +56,7 @@ class Simulation:
sys.exit(1)
if self.createDirectory(self.dirname, doKeep) == False:
print "Simulation results already exist"
print( "Simulation results already exist")
return
os.chdir(self.dirname)
......@@ -65,7 +66,7 @@ class Simulation:
else:
fieldmapPath = '../fieldmaps'
if not (os.path.isdir(fieldmapPath)):
print 'Fieldmap directory unknown exiting ...'
print( 'Fieldmap directory unknown exiting ...')
sys.exit()
......@@ -89,7 +90,7 @@ class Simulation:
filedata = file.read()
# do the replacements in the templatefile
for s,value in self.opaldict.iteritems():
for s,value in self.opaldict.items():
# Replace the target string
filedata = filedata.replace('_'+s+'_', str(value))
......@@ -102,26 +103,27 @@ class Simulation:
else:
opalexe = 'opal'
print 'Simulation directory is ' + self.dirname + ' using OPAL at ', os.environ.get('OPAL_EXE_PATH')
print 'Using templatefile at ' + inputfilePath
print 'Using fieldmaps at ' + fieldmapPath
print 'Parameter set in ' + oinpFile + ' are:'
print( 'Simulation directory is {} using OPAL at {}'.format(self.dirname, os.environ.get('OPAL_EXE_PATH')))
print( 'Using templatefile at ' + inputfilePath)
print( 'Using fieldmaps at ' + fieldmapPath)
print( 'Parameter set in ' + oinpFile + ' are:')
for s, value in sorted(self.opaldict.iteritems()):
print ' :::: ' + s + ' = ' + str(value)
for s, value in sorted(self.opaldict.items()):
print( ' :::: ' + s + ' = ' + str(value))
if not doNobatch:
hostname = commands.getoutput("hostname")
print("On host ",hostname)
#hostname = commands.getoutput("hostname")
hostname = (subprocess.check_output('hostname').decode('utf-8')).strip()
print("On host {}".format(hostname))
if os.getenv("SGE_TIME"):
print "You use deprecated environment variable SGE_TIME. Please use in the future TIME"
print( "You use deprecated environment variable SGE_TIME. Please use in the future TIME")
time = os.getenv("SGE_TIME")
else:
time = os.getenv("TIME", "s_rt=23:59:00,h_rt=24:00:00")
if os.getenv("SGE_RAM"):
print "You use deprecated environment variable SGE_RAM. Please use in the future RAM"
print( "You use deprecated environment variable SGE_RAM. Please use in the future RAM")
ram = os.getenv("SGE_RAM")
else:
ram = os.getenv("RAM", "4")
......@@ -153,7 +155,7 @@ class Simulation:
runfile = 'run.blues'
self.WritePBSBlues(opalexe, oinpFile, CORES, time, ram, info, queue)
elif (hostname.startswith("bebop")):
elif (hostname.startswith("bebop") or hostname.startswith("bdw") or hostname.startswith("knl")):
batchsys = 'SLURM'
runfile = 'run.bebop'
self.WriteBebop(opalexe, oinpFile, CORES, time, ram, info, runfile, queue, hypert)
......@@ -188,28 +190,32 @@ class Simulation:
sys.exit(1)
if doTest:
print 'Done with setup of the OPAL simulation but not submitting the job (--test) \n\n\n'
print( 'Done with setup of the OPAL simulation but not submitting the job (--test) \n\n\n')
qid = -1
os.chdir('..')
return qid
if doNobatch:
print 'Done with setup of the OPAL simulation and executing the job on ' + str(CORES) + ' cores...\n\n\n'
print( 'Done with setup of the OPAL simulation and executing the job on {} cores...\n\n\n'.format(CORES))
ofn, fileExtension = os.path.splitext(oinpFile)
print 'STD output is written to ' + ofn + '.out'
execommand = 'mpirun -np ' + str(CORES) + ' ' + opalexe + ' ' + oinpFile + ' 2>&1 | tee ' + ofn + '.out'
qid = commands.getoutput(execommand)
print( 'STD output is written to {}.out'.format(ofn))
#execommand = 'mpirun -np ' + str(CORES) + ' ' + opalexe + ' ' + oinpFile + ' 2>&1 | tee ' + ofn + '.out'
outfileName = ofn +'.out'
# Currently not writing to screen anymore
# There is a solution described at https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously
with open(outfileName,'w') as outfile:
qid = subprocess.call(['mpirun', '-np', str(CORES), opalexe, oinpFile], stdout=outfile, stderr=outfile)
os.chdir('..')
return -1
else:
qid = ""
if batchsys == 'SLURM':
qid = commands.getoutput('sbatch ' + runfile + ' | awk \'{print $3}\' ')
print 'Done with setup of the OPAL simulation and submitting the job with ', CORES, ' cores \n\n\n'
qid = subprocess.call(['sbatch', runfile, '|', 'awk','\'{print $3}\''])
print( 'Done with setup of the OPAL simulation and submitting the job with {} cores \n\n\n'.format(CORES))
elif batchsys == 'PBS':
print 'Done with setup of the OPAL simulation, please submit the job yourself'
print( 'Done with setup of the OPAL simulation, please submit the job yourself')
os.chdir('..')
return qid
......@@ -444,7 +450,7 @@ class Simulation:
nodes = cores/coresPerNode
tasks_per_node = cores/nodes
print(nodes,cores, tasks_per_node)
#print(nodes,cores, tasks_per_node)
title = oinpFile.partition(".")[0]
myfile = open(name, 'w')
......@@ -535,4 +541,3 @@ class Simulation:
# -d $n_hyperthreads_skipped_between_ranks \
# -j $n_hyperthreads_per_core \
# <executable> <executable args>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment