runOPAL.py 7.47 KB
Newer Older
1
#!/usr/bin/env python
gsell's avatar
gsell committed
2 3

"""
4
Script that launches OPAL simulations
gsell's avatar
gsell committed
5 6

@author: Andreas Adelmann <andreas.adelmann@psi.ch>
7
@author: Yves Ineichen
gsell's avatar
gsell committed
8 9 10 11
@version: 0.1

"""

12
import sys,os,shutil,glob
13
import subprocess
gsell's avatar
gsell committed
14 15 16 17 18 19 20
from simulation import Simulation
from opaldict import OpalDict

def getBaseName():
    path ='.'
    ext1 ='*.tmpl'
    ext2 ='*.data'
21 22 23 24 25
    templates = glob.glob(os.path.join(path,ext1))
    datafiles = glob.glob(os.path.join(path,ext2))
    if templates:
        str1 = templates[0]
    else:
ext-neveu_n's avatar
ext-neveu_n committed
26
        print('No template file (.tmpl) found')
27 28 29 30
        sys.exit()
    if datafiles:
        str2 = datafiles[0]
    else:
ext-neveu_n's avatar
ext-neveu_n committed
31
        print('No data file (.data) found')
32 33
        sys.exit()

gsell's avatar
gsell committed
34 35 36 37 38 39
    str1spl = str1.split('.')
    str2spl = str2.split('.')
    if str1spl[1] == str2spl[1]:
        name = str2spl[1]
        name = name.split('/')[1]
    else:
ext-neveu_n's avatar
ext-neveu_n committed
40
        print('Template and data filename do not match', str1spl, str2spl)
gsell's avatar
gsell committed
41 42 43 44
        sys.exit()
    return name

def printUsage():
45
    print("./runOPAL.py [--help] [--quiet] [--info=num] [--test] [--keep] [--queue=qname] [--hypert=num] [--nobatch] [ATTR=SCANVALUE] {[ATTR=VALUE]}")
ext-neveu_n's avatar
ext-neveu_n committed
46
    print("")
47
    print("--help          prints this message")
ext-neveu_n's avatar
ext-neveu_n committed
48 49 50
    print("--test          does everything but submitting the job")
    print("--keep          if same simulation has been run before, keep old data and abort")
    print("--nobatch       run opal locally not using the batch system and waits until the job is done")
51
    print("--quiet         suppress debug printout")
ext-neveu_n's avatar
ext-neveu_n committed
52 53 54
    print("--info=<num>    steers the std-output of OPAL. The range is 0 < num < 6 (default), from minimal to maximum output")
    print("--queue=<qname> defines in which queue the job goes. Overwrites QUEUE (deprecated SGE_QUEUE)")
    print("--hypert=<num>  defines the number of Hyper-Threads used. Default 0")
55 56 57
    print("")
    print("SCANVALUE=start:end:step, scans a parameter space, e.g. example TFWHM=0.85:0.90:0.01 ")
    print("ATTR refers to a name in the data file")
58
    print("")
ext-neveu_n's avatar
ext-neveu_n committed
59
    print("Recognized environment variables: DISTRIBUTIONS, FIELDMAPS, OPAL_EXE_PATH, TEMPLATES, QUEUE, RAM, TIME (deprecated SGE_)")
60 61 62
    # temporary see issue #8
    print("")
    print("Important: runOPAL is currently not compatible with the commands OPTIMIZE and SAMPLE")
gsell's avatar
gsell committed
63

64
def traverseRanges(list, opaldict, args, doNobatch):
65 66 67 68 69 70
    """
    Traverse all possible combinations of range variable values. Start simulation
    once all range variables are fixed to a value. A list entry has the following
    structure:
    ['name of var', start_value, end_value, step_value]
    """
gsell's avatar
gsell committed
71 72 73
    head = list[0]
    tail = list[1:]
    curval = head[1][0]
74 75
    endval = head[1][1]
    step   = head[1][2]
gsell's avatar
gsell committed
76
    qid = -1
77
    if curval > endval:
ext-neveu_n's avatar
ext-neveu_n committed
78
        print('range is empty, start value',curval,'needs to be higher than end value',endval)
79
    while curval <= endval:
gsell's avatar
gsell committed
80 81
            opaldict[head[0]] = curval
            if len(tail) == 0:
82
                #run simulation
gsell's avatar
gsell committed
83 84
                sim = Simulation(opaldict)
                qid = sim.run(*args)
85
                if doNobatch:
ext-neveu_n's avatar
ext-neveu_n committed
86
                    print("... finished!\n")
87
                else:
88
                    print("SGE-ID= {}\n".format(qid))
gsell's avatar
gsell committed
89
            else:
90
                traverseRanges(tail, opaldict, args, doNobatch)
91
            curval = curval + step
92
 
gsell's avatar
gsell committed
93
def main(argv):
94 95 96
    """
    main method
    """
97
    N = -1               # a running number; if given use it to label directory!
ext-neveu_n's avatar
ext-neveu_n committed
98
    quiet  = False
99 100 101
    doTest = False
    doKeep = False
    doNobatch = False
102
    queue = ""
103
    info = 6
adelmann's avatar
adelmann committed
104
    hypert = 0 
105 106 107 108 109
    qid = -1

    for arg in argv:
        if arg.startswith("--test"):
            doTest = True
adelmann's avatar
adelmann committed
110 111
        elif arg.startswith("--hypert"):
            hypert = int(arg.split("=")[1])
112 113
        elif arg.startswith("--keep"):
            doKeep = True
ext-neveu_n's avatar
ext-neveu_n committed
114 115
        elif arg.startswith("--nobatch"):
            doNobatch = True
116 117
        elif arg.startswith("--info"):
            info = arg.split("=")[1]
ext-neveu_n's avatar
ext-neveu_n committed
118 119
        elif arg.startswith("--queue"):
            queue = arg.split("=")[1]
120 121 122
        elif arg.startswith("--help"):
            printUsage()
            exit()
ext-neveu_n's avatar
ext-neveu_n committed
123 124
        elif arg.startswith("--quiet"):
            quiet = True
125
        elif arg.startswith("--"):
ext-neveu_n's avatar
ext-neveu_n committed
126
            print(arg,'is not a valid option, see --help for the available options')
127
            exit()
128

gsell's avatar
gsell committed
129
    if (os.getcwd() == os.environ.get('TEMPLATES')):
ext-neveu_n's avatar
ext-neveu_n committed
130
        print('Working directory is the same as the TEMPLATES directory! This is not allowed... bye!')
gsell's avatar
gsell committed
131 132 133 134 135 136 137
        sys.exit()
    else:
        # make sure we have no junk around
        os.system('rm -f tmplbak17.tmpl')

    if (os.environ.get('TEMPLATES')):
        inputfilePath = os.environ.get('TEMPLATES')
138
        
adelmann's avatar
adelmann committed
139
        if not os.path.isfile(inputfilePath + ".tmpl"):
140
            os.system('cp ' + inputfilePath + '/*.tmpl .')	
gsell's avatar
gsell committed
141
    else:
ext-neveu_n's avatar
ext-neveu_n committed
142
        ext1 ='*.tmpl'
gsell's avatar
gsell committed
143 144 145
        if (glob.glob(os.path.join('.',ext1))):
            inputfilePath = '../'
        else:
ext-neveu_n's avatar
ext-neveu_n committed
146
            print('Template file unknown -> exiting ...')
gsell's avatar
gsell committed
147
            sys.exit()
148
            
gsell's avatar
gsell committed
149 150 151 152 153 154
    baseFileName = getBaseName()

    dataFile = baseFileName + '.data'
    tmplFile = baseFileName + '.tmpl'
    oinpFile = baseFileName + '.in' # the resulting OPAL input file

155 156 157 158 159
    # Optimizer and Sample command currently not compatible with runOPAL (issue #8)
    templateFile = open(tmplFile,'r')
    for line in templateFile:
        if line.startswith('//'):
            continue
snuverink_j's avatar
snuverink_j committed
160
        if 'OPTIMIZE' in line or 'SAMPLE' in line:
161 162 163 164 165 166 167 168
            print('OPTIMIZE and SAMPLE command currently not compatible with runOPAL')
            sys.exit()
    templateFile.close()

    if os.environ.get('TEMPLATES'):
        if os.path.isfile(baseFileName + ".tmpl"):
	        os.system("mv " + baseFileName + ".tmpl tmplbak17.tmpl")

gsell's avatar
gsell committed
169 170 171 172 173 174 175
    #create the dictionary
    opaldict = OpalDict(dataFile)
    # check if template values must be changed
    # if so add update the dictionary with the default values
    opaldict.addUserValues(argv)
    opaldict.scale()

176
    if not opaldict.hasRanges():
gsell's avatar
gsell committed
177
        sim = Simulation(opaldict)
178
        qid = sim.run(N, baseFileName, inputfilePath, tmplFile, oinpFile, doTest, doKeep, doNobatch, info, queue, hypert, quiet)
179
        if doNobatch:
ext-neveu_n's avatar
ext-neveu_n committed
180
            print( "... finished!\n")
ext-neveu_n's avatar
ext-neveu_n committed
181 182
        #else:
        #    print( "SGE-ID= {}\n".format(qid))
gsell's avatar
gsell committed
183 184 185 186 187 188 189 190 191
    else:
        ranges = opaldict.Range()

        #create range toplevel dir
        dirname = baseFileName
        for p in opaldict.uservars:
            dirname += "_" + str(p[0]) + "=" + str(p[1])
        for (k, v) in ranges.items():
            dirname += "_" + k + "=" + str(v[0]) + ":" + str(v[1]) + ":" + str(v[2])
192
        # If there's already a directory remove it...
gsell's avatar
gsell committed
193
        if os.path.isdir(dirname):
194
            if doKeep:
ext-neveu_n's avatar
ext-neveu_n committed
195
                print( 'KEEP existing directory ', dirname)
196
            else:
ext-neveu_n's avatar
ext-neveu_n committed
197
                print( 'REMOVE existing directory', dirname)
198 199 200
                shutil.rmtree(dirname)
                # create directory and change to the directory
                os.mkdir(dirname)
201 202
        else:
            os.mkdir(dirname)
203
            
204
        os.chdir(dirname)
gsell's avatar
gsell committed
205

ext-neveu_n's avatar
ext-neveu_n committed
206
        print(ranges)
gsell's avatar
gsell committed
207
        #run simulations of all possible combinations
208
        args = [N, baseFileName, inputfilePath, tmplFile, oinpFile, doTest, doKeep, doNobatch, info, queue, hypert, quiet]
209
        traverseRanges(list(ranges.items()), opaldict, args, doNobatch)
adelmann's avatar
adelmann committed
210 211
        # clean up
        os.system("rm -f *.bak ")
gsell's avatar
gsell committed
212 213

        os.chdir("..")
214
        
215
    # clean up
gsell's avatar
gsell committed
216
    if os.path.isfile("tmplbak17.tmpl"):
217
        os.system("mv tmplbak17.tmpl " + "." + baseFileName + ".tmpl")
adelmann's avatar
adelmann committed
218

gsell's avatar
gsell committed
219 220 221
#call main
if __name__ == "__main__":
    main(sys.argv[1:])