#!/usr/bin/env python
import filecmp, json, os, re, shutil, sys
#====================================================================
def badparms( msg):
print '\nError: %s' % (msg,)
print 'Parms:'
print ' -bugLev debug level'
print ' -specFile input file in some ancestor dir'
sys.exit( 1)
#====================================================================
#====================================================================
[docs]def main():
'''
Mimics the execution of a program by checking that
the input files are as we expect, and then copying
previously calculated output files.
This program is used to mimic the execution of VASP
for validity testing. See pylada/testValid.
Command line parameters:
============== =========== ===============================================
Parameter Type Description
============== =========== ===============================================
**-bugLev** integer Debug level. Normally 0.
**-specFile** string JSON file containing specs. See below.
============== =========== ===============================================
**specFile Contents:**
=================== =================================================
Parameter Description
=================== =================================================
**refRoot** Root dir of the reference tree
**workRoot** Root dir of the tree to be checked
**inFiles** List of files to be checked in each work dir
**outFiles** List of files to copy from ref dir to work
dir. If '', copy all except inFiles and omits.
**omitFiles** List of regexs of outFiles to omit.
**absEpsilon** Absolute epsilon for comparing floats of
work inFile vs ref inFile.
**relEpsilon** Relative epsilon for comparing floats of
work inFile vs ref inFile.
**logFile** Log file for execMimic output.
=================== =================================================
**inSpec file example:**::
{
"absEpsilon": "1.e-6",
"relEpsilon": "1.e-6",
"refRoot": "/home/me/reference",
"workRoot": "/home/me/test",
"inFiles": "INCAR,KPOINTS,POSCAR,POTCAR",
"outFiles": "",
"omitFiles": "^pbs.*$,^std.*$",
"logFile": "/home/me/execMimic.log"
}
'''
bugLev = 1
specFile = 'pylada.execMimic.json'
if len(sys.argv) % 2 != 1:
badparms('parms must be key/value pairs')
for iarg in range( 1, len(sys.argv), 2):
key = sys.argv[iarg]
val = sys.argv[iarg+1]
if key == '-bugLev': bugLev = int( val)
elif key == '-specFile': specFile = val
else: badparms('unknown parm: "%s"' % (key,))
# Find the spec file by looking at ancestor dirs
specPath = None
curDir = os.getcwd().rstrip('/') # no trailing '/'
while curDir != '':
spath = os.path.join( curDir, specFile)
if os.access( spath, os.R_OK):
specPath = spath
break
ix = curDir.rfind('/')
if ix < 0: break
curDir = curDir[:ix]
# Omit print as it overwrites PyLada's stdout
##if bugLev >= 1: print 'specPath: %s' % (specPath,)
if specPath == None:
throwerr( None, 'missing specFile: %s' % specFile, None)
with open( specPath) as fin:
specMap = json.load( fin)
# Omit print as it overwrites PyLada's stdout
##if bugLev >= 1: print 'specMap: %s' % (specMap,)
if type(specMap).__name__ != 'dict':
throwerr( None, 'invalid json type', specPath)
refRoot = specMap.get('refRoot', None)
workRoot = specMap.get('workRoot', None)
refRoot = os.path.expanduser( refRoot)
workRoot = os.path.expanduser( workRoot)
inFileStg = specMap.get('inFiles', None)
outFileStg = specMap.get('outFiles', None)
omitFileStg = specMap.get('omitFiles', None)
absEpsilon = float( specMap.get('absEpsilon', None))
relEpsilon = float( specMap.get('relEpsilon', None))
logFileStg = specMap.get('logFile', None)
if refRoot == None or refRoot != os.path.abspath( refRoot):
throwerr( None, 'refRoot is missing or not absolute', specPath)
if workRoot == None or workRoot != os.path.abspath( workRoot):
throwerr( None, 'workRoot is missing or not absolute', specPath)
if logFileStg == None or len(logFileStg) == 0:
throwerr( None, 'logFile is missing', specPath)
logName = os.path.expanduser( logFileStg)
if bugLev >= 1:
printLog( logName, 'specPath: %s' % (specPath,))
printLog( logName, 'refRoot: %s' % (refRoot,))
printLog( logName, 'workRoot: %s' % (workRoot,))
printLog( logName, 'inFileStg: %s' % (inFileStg,))
printLog( logName, 'outFileStg: %s' % (outFileStg,))
printLog( logName, 'omitFileStg: %s' % (omitFileStg,))
printLog( logName, 'absEpsilon: %s' % (absEpsilon,))
printLog( logName, 'relEpsilon: %s' % (relEpsilon,))
printLog( logName, 'logFile: %s' % (logFileStg,))
if inFileStg == None or len(inFileStg) == 0:
throwerr( logName, 'inFiles is missing', specPath)
if outFileStg == None:
throwerr( logName, 'outFiles is missing', specPath)
if omitFileStg == None:
throwerr( logName, 'omitFiles is missing', specPath)
inFiles = inFileStg.split(',')
if outFileStg == '': outFiles = None # implies all non-input files
else: outFiles = outFileStg.split(',')
if omitFileStg == '': omitFiles = None
else: omitFiles = omitFileStg.split(',')
if bugLev >= 1:
printLog( logName, 'inFiles: %s' % (inFiles,))
printLog( logName, 'outFiles: %s' % (outFiles,))
printLog( logName, 'omitFiles: %s' % (omitFiles,))
workDir = os.getcwd()
if bugLev >= 1:
printLog( logName, 'workDir: %s' % (workDir,))
if not workDir.startswith( workRoot):
throwerr( logName, 'cwd not within workRoot', specPath)
deltaPath = workDir[ len(workRoot) :].strip('/')
refDir = os.path.join( refRoot, deltaPath)
if bugLev >= 1:
printLog( logName, 'deltaPath: %s' % (deltaPath,))
printLog( logName, 'refDir: %s' % (refDir,))
# Make sure all input files match
for inFile in inFiles:
refPath = os.path.join( refDir, inFile)
workPath = os.path.join( workDir, inFile)
if bugLev >= 1:
printLog( logName, 'inFile: %s' % (inFile,))
printLog( logName, ' refPath: %s' % (refPath,))
printLog( logName, ' workPath: %s' % (workPath,))
if not os.access( refPath, os.R_OK):
throwerr( logName, 'cannot read refPath: %s' % (refPath,), specPath)
if not os.access( workPath, os.R_OK):
throwerr( logName, 'cannot read workPath: %s' % (workPath,), specPath)
# We cannot use filecmp.cmp since roundoff errors
# may cause slight differences between the machine
# creating the reference and the machine running the test.
# cmp = filecmp.cmp( refPath, workPath)
#if not cmp:
# throwerr( logName, 'infile mismatch.\n refPath: %s\n workPath: %s' \
# % (refPath, workPath,), specPath)
checkFiles( logName, absEpsilon, relEpsilon, refPath, workPath)
# If we copy all output files except the input files
if outFiles == None: fnames = os.listdir( refDir)
else: fnames = outFiles # Else we copy only the named output files
fnames.sort()
for outFile in fnames:
isMatched = testRegexs( omitFiles, outFile)
if outFile in inFiles:
if bugLev >= 1:
printLog( logName, 'outFile omitted (inFile): %s' % (outFile,))
elif isMatched:
if bugLev >= 1:
printLog( logName, 'outFile omitted (omitFile): %s' % (outFile,))
else:
outRef = os.path.join( refDir, outFile)
outWork = os.path.join( workDir, outFile)
if bugLev >= 1:
printLog( logName, 'outFile: %s' % (outFile,))
printLog( logName, ' outRef: %s' % (outRef,))
printLog( logName, ' outWork: %s' % (outWork,))
if os.path.isfile( outRef):
# Allow overwrite
#if os.access( outWork, os.F_OK):
# if not filecmp.cmp( outRef, outWork):
# throwerr( logName,
# 'outWork differs.\n outRef: %s\n outWork: %s'
# % (outRef, outWork,), specPath)
shutil.copyfile( outRef, outWork)
#====================================================================
[docs]def testRegexs( regexs, fname):
'''
Returns True if fname matches any of the regex.
'''
bres = False
if regexs != None:
for regex in regexs:
if re.match( regex, fname):
bres = True
break
return bres
#====================================================================
[docs]def checkFiles( logName, absEpsilon, relEpsilon, patha, pathb):
'''
Compares two files. If they are identical except
for possible floating points within
absolute absEpsilon and relative relEpsilon,
return True.
'''
fina = open( patha)
finb = open( pathb)
errMsg = None
iline = 0
linea = None
lineb = None
while True:
linea = fina.readline()
lineb = finb.readline()
if linea == '' and lineb == '': break
if linea == '':
errMsg = 'early eof on A'
break
if lineb == '':
errMsg = 'early eof on B'
break
iline += 1
tokas = linea.split()
tokbs = lineb.split()
if len(tokas) != len(tokbs):
errMsg = 'num tokens differs'
break
for ii in range(len(tokas)):
toka = tokas[ii]
tokb = tokbs[ii]
vala = None
valb = None
try: vala = float( toka)
except ValueError, exc: pass
try: valb = float( tokb)
except ValueError, exc: pass
if vala == None and valb == None: # if neither is float
if toka != tokb:
errMsg = 'strings differ'
break
elif vala == None or valb == None: # if only one is float
errMsg = 'only one token is float'
break
else:
# Both are float
if abs( vala - valb) >= absEpsilon:
errMsg = 'absEpsilon err: ii: %d\n vala: %g\n valb: %g\n' \
% (ii, vala, valb,)
errMsg += ' valb-vala: %g\n' % (valb - vala,)
break
if vala != 0:
relErr = abs( (valb - vala) / vala)
if relErr >= relEpsilon:
errMsg = 'relEpsilon err: ii: %d vala: %g valb: %g valb-vala: %g relErr: %g' \
% (ii, vala, valb, valb - vala, relErr)
break
if errMsg != None: break
fina.close()
finb.close()
if errMsg != None:
fullMsg = 'Mismatch:\n' \
+ errMsg + '\n' \
+ ' iline: %d\n' % (iline,) \
+ ' linea: %s\n' % (linea,) \
+ ' lineb: %s\n' % (lineb,) \
+ ' patha: %s\n' % (patha,) \
+ ' pathb: %s\n' % (pathb,) \
+ ' absEpsilon: %g\n' % (absEpsilon,) \
+ ' relEpsilon: %g\n' % (relEpsilon,)
throwerr( logName, fullMsg, patha)
#====================================================================
def printLog( logName, msg):
logFile = open( os.path.expanduser( logName), 'a')
print >> logFile, msg
logFile.close()
#====================================================================
def throwerr( logName, msg, specPath):
fullMsg = 'Error: %s\n cwd: %s\n specPath: %s\n' \
% (msg, os.getcwd(), specPath,)
if logName != None: printLog( logName, fullMsg)
raise Exception( fullMsg)
#====================================================================
if __name__ == '__main__': main()