development/validation/comp_md.py

import sys
import math
import logging
import os
import subprocess
import logging

fraw_list = []#List of all .md files found (even the includes).
fmd_list = []#List of all config .md files that can be run (not the includes).
fmd_base_list = []#List of the names of the .md files that can be run.

dir_cwd = ""#Current working directory.
dir_openmd = ""#Absolute path for openmd
dir_base = ""#Directory where the script is run from.

FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT)

"""
Function sets up the dir_base and dir_openmd. If an openmd executable is not
found, script exits. Function looks for the openmd in the relative location
../build/bin but stores the openmd location as an absolute path.
@author Samuel Njoroge
"""
def setupDirectories():
        global dir_base, dir_openmd, dir_cwd
        logger = logging.getLogger("tcpserver")
        dir_base = os.getcwd()
        if(os.path.isfile("../build/bin/openmd")):
                os.chdir("../build/bin/")
                dir_openmd = os.getcwd()
                os.chdir(dir_base)
        elif(os.path.isfile("../bin/openmd")):
                os.chdir("../bin/")
                dir_openmd = os.getcwd()
                os.chdir(dir_base)      
        else:
                logger.error("OpenMD : %s", "openmd executable not found at the expected location. Script Will Quit...")
                sys.exit()
        forcefld_path = os.path.abspath("../forceFields")
        os.environ["FORCE_PARAM_PATH"] = forcefld_path
        
"""
Function checks if the sample_file and validate_file (.md files) have the same
statusTime = interval time for the stats file.
@author Samuel Njoroge
@param string sample_file - .md file that is being run.
@param string validate_file - .md file the result is being compared to.
@return boolean
"""
def validate_md_time(sample_file, validate_file):
  sample_status_time = 0
  sample_sample_time = 0
  sample_run_time = 0
  validate_status_time = 0
  validate_sample_time = 0
  validate_run_time = 0
  logger = logging.getLogger("tcpserver")

  samplefh = open(sample_file, "r")
  validatefh = open(validate_file, "r")
  
  line = samplefh.readline()
  while line:
    if "statusTime" in line:
      arr = line.split('=')
      temp = arr[1]
      sample_status_time = float(temp.replace(';', ''))
    elif "sampleTime" in line:
      arr = line.split('=')
      temp = arr[1]
      sample_sample_time = float(temp.replace(';', ''))
    elif "runTime" in line:
      arr = line.split('=')
      temp = arr[1]
      sample_run_time = float(temp.replace(';', ''))

    line = samplefh.readline()

  line = validatefh.readline()
  while line:
    if "statusTime" in line:
      arr = line.split('=')
      temp = arr[1]
      validate_status_time = float(temp.replace(';', ''))
    elif "sampleTime" in line:
      arr = line.split('=')
      temp = arr[1]
      validate_sample_time = float(temp.replace(';', ''))
    elif "runTime" in line:
      arr = line.split('=')
      temp = arr[1]
      validate_run_time = float(temp.replace(';', ''))

    line = validatefh.readline()

  if (sample_status_time > 0) or (validate_status_time > 0):
    if sample_status_time == validate_status_time:
      return True

  if (sample_sample_time > 0) or (validate_sample_time > 0):
    if sample_sample_time == validate_sample_time:
      return True

  if (sample_run_time > 0) or (validate_run_time > 0):
    if sample_run_time == validate_run_time:
      return True

  logger.warning("MD File: %s", "Sample/Validation times do not match.")
  return False
  
"""
Function checks if an .md config file and not an include file.
@author Samuel Njoroge
@param string file - .md file name
@return boolean
"""
def file_is_md(filename):
        file_handle = open(filename)
        line = file_handle.readline()
        
        while line:
                if "<OpenMD version=" in line:
                        return True
                line = file_handle.readline()
                        
        return False
        
"""
Function compares two numbers.
@author Samuel Njoroge and ().
@param float one - first number to compare.
@param float two - second number to compare.
@param boolean ignore_sign - if sign is a factor in the comparison.
@return float diff - difference between the two numbers.
"""
def absDiff(one, two, ignore_sign = False):
        if ignore_sign:
                one, two = math.fabs(one), math.fabs(two)
        return max(one, two) - min(one, two)
        
"""
Function compares two files.
@author Samuel Njoroge and ().
@param string fExpected - name of the expected file.
@param string fNew - name of the new test file.
@param float epsilon - Precision of the comparison of the files.
@param boolean ignore_sign - if sign will be a factor in comparing the digits.
@return boolean
"""
def compare(fExpected, fNew, epsilon = 0.00001, ignore_sign=False):
        logger = logging.getLogger("tcpserver")
        fone = open(fExpected, 'r')
        ftwo = open(fNew, 'r')

        diffs = 0
        
        i = 0
        for lineone in fone:
                linetwo = ftwo.readline()

                elementsone = lineone.split()
                elementstwo = linetwo.split()

                i = i + 1

                lenone = len(elementsone)
                lentwo = len(elementstwo)

                if lenone != lentwo:
                        diffs = diffs + 1
                        logger.warning("Line: %d - %s", i, "no mach")
                        return True
                else:
                        for j in range(lenone):
                                # used to ignore XYZ meta data such as # of frames and # of atoms
                                try:
                                        feone = int(elementsone[j])
                                        fetwo = int(elementstwo[j])
                                        # these are ints -- skip this pair
                                        continue
                                except ValueError:
                                        pass
                                try:
                                        feone = float(elementsone[j])
                                        fetwo = float(elementstwo[j])

                                        fediff = absDiff(feone, fetwo, ignore_sign)

                                        if fediff > epsilon:
                                                diffs = diffs + 1
                                                print "Line " + str(i) + " do not match in the files."
                                                return True
                                except ValueError:
                                        pass
        return False#diffs == 0
        
"""
Function scans the directory for .md config files which will be run for
testing.
@author Samuel Njoroge
"""
def scanForMdFiles():
        paths = []
        variable = "sam is here"
        for p in os.listdir("../samples/"):
                paths.append(os.path.abspath("../samples/" + p))
        
        while len(paths):
                p = paths.pop()
                if os.path.isfile(p):
                        fraw_list.append(p)
                else:
                        for np in os.listdir(p):
                                paths.append(os.path.abspath(p + "/" + np))
                                
        for f in fraw_list:
                if (".md" in f) and (not ".svn" in f) and file_is_md(f):
                        fmd_list.append(f)
                        temp = os.path.basename(f)
                        temp = temp.replace(".md",'')
                        fmd_base_list.append(temp)
                
"""
Function runs OpenMD on the files the compares them with the already run files (*_v.stat)
@author Samuel Njoroge
"""
def runMdFiles():
        logger = logging.getLogger("tcpserver")
        global dir_base, dir_openmd, dir_cwd
        output = []
        for x in range(0, len(fmd_list)):
                #subprocess.call(["export FORCE_PARAM_PATH=/Users/snjoroge/Documents/openmd/development/forceFields"])
                if "argon" in fmd_list[x]:
                        logger.debug("Switching to Directory: %s", os.path.dirname(fmd_list[x]))
                        os.chdir(os.path.dirname(fmd_list[x]))
                        logger.debug("Running: %s", fmd_list[x])
                        output = subprocess.call([dir_openmd + "/openmd", fmd_list[x]])
                        if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat")):
                                #print "Renaming File: " + fmd_base_list[x] + ".stat - " + fmd_base_list[x] + "_v.stat"
                                #subprocess.call(["cp", os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat", os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + "_v.stat"])
                                logger.debug("Comparing: %s", "Comparing: " + fmd_base_list[x] + ".stat <=> " + fmd_base_list[x] + "_v.stat")
                                if(compare(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat", os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + "_v.stat")):
                                        logger.warning("Files: %s", "Files do not match.")
                                else:
                                        logger.debug("Files Match")
                os.chdir(dir_base)

def cleanUp():
        print "Delete all files generated."
        for x in range(0, len(fmd_list)):
                if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".eor")):
                        print "DELETE:" + os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".eor"
                        os.remove(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".eor")
                if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat")):
                        print "DELETE:" + os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat"
                        os.remove(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat")
                if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".dump")):
                        print "DELETE:" + os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".dump"
                        os.remove(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".dump")
        
Revision:	1661
Committed:	Mon Nov 7 13:05:14 2011 UTC (13 years, 11 months ago) by chuckv
Content type:	text/x-python
File size:	8384 byte(s)
Log Message:	Minor modifications to set environment variable.
#	User	Rev	Content
1	chuckv	1659	import sys
2			import math
3			import logging
4			import os
5			import subprocess
6	chuckv	1660	import logging
7	chuckv	1659
8			fraw_list = []#List of all .md files found (even the includes).
9			fmd_list = []#List of all config .md files that can be run (not the includes).
10			fmd_base_list = []#List of the names of the .md files that can be run.
11
12			dir_cwd = ""#Current working directory.
13			dir_openmd = ""#Absolute path for openmd
14			dir_base = ""#Directory where the script is run from.
15
16	chuckv	1660	FORMAT = '%(asctime)-15s %(message)s'
17			logging.basicConfig(format=FORMAT)
18
19	chuckv	1659	"""
20			Function sets up the dir_base and dir_openmd. If an openmd executable is not
21			found, script exits. Function looks for the openmd in the relative location
22			../build/bin but stores the openmd location as an absolute path.
23			@author Samuel Njoroge
24			"""
25			def setupDirectories():
26			global dir_base, dir_openmd, dir_cwd
27	chuckv	1660	logger = logging.getLogger("tcpserver")
28	chuckv	1659	dir_base = os.getcwd()
29			if(os.path.isfile("../build/bin/openmd")):
30			os.chdir("../build/bin/")
31			dir_openmd = os.getcwd()
32			os.chdir(dir_base)
33	chuckv	1661	elif(os.path.isfile("../bin/openmd")):
34			os.chdir("../bin/")
35			dir_openmd = os.getcwd()
36			os.chdir(dir_base)
37	chuckv	1659	else:
38	chuckv	1660	logger.error("OpenMD : %s", "openmd executable not found at the expected location. Script Will Quit...")
39	chuckv	1659	sys.exit()
40	chuckv	1661	forcefld_path = os.path.abspath("../forceFields")
41			os.environ["FORCE_PARAM_PATH"] = forcefld_path
42	chuckv	1659
43			"""
44			Function checks if the sample_file and validate_file (.md files) have the same
45			statusTime = interval time for the stats file.
46			@author Samuel Njoroge
47			@param string sample_file - .md file that is being run.
48			@param string validate_file - .md file the result is being compared to.
49			@return boolean
50			"""
51			def validate_md_time(sample_file, validate_file):
52			sample_status_time = 0
53			sample_sample_time = 0
54			sample_run_time = 0
55			validate_status_time = 0
56			validate_sample_time = 0
57			validate_run_time = 0
58	chuckv	1660	logger = logging.getLogger("tcpserver")
59
60	chuckv	1659	samplefh = open(sample_file, "r")
61			validatefh = open(validate_file, "r")
62
63			line = samplefh.readline()
64			while line:
65			if "statusTime" in line:
66			arr = line.split('=')
67			temp = arr[1]
68			sample_status_time = float(temp.replace(';', ''))
69			elif "sampleTime" in line:
70			arr = line.split('=')
71			temp = arr[1]
72			sample_sample_time = float(temp.replace(';', ''))
73			elif "runTime" in line:
74			arr = line.split('=')
75			temp = arr[1]
76			sample_run_time = float(temp.replace(';', ''))
77
78			line = samplefh.readline()
79
80			line = validatefh.readline()
81			while line:
82			if "statusTime" in line:
83			arr = line.split('=')
84			temp = arr[1]
85			validate_status_time = float(temp.replace(';', ''))
86			elif "sampleTime" in line:
87			arr = line.split('=')
88			temp = arr[1]
89			validate_sample_time = float(temp.replace(';', ''))
90			elif "runTime" in line:
91			arr = line.split('=')
92			temp = arr[1]
93			validate_run_time = float(temp.replace(';', ''))
94
95			line = validatefh.readline()
96
97			if (sample_status_time > 0) or (validate_status_time > 0):
98			if sample_status_time == validate_status_time:
99			return True
100
101			if (sample_sample_time > 0) or (validate_sample_time > 0):
102			if sample_sample_time == validate_sample_time:
103			return True
104
105			if (sample_run_time > 0) or (validate_run_time > 0):
106			if sample_run_time == validate_run_time:
107			return True
108
109	chuckv	1660	logger.warning("MD File: %s", "Sample/Validation times do not match.")
110	chuckv	1659	return False
111
112			"""
113			Function checks if an .md config file and not an include file.
114			@author Samuel Njoroge
115			@param string file - .md file name
116			@return boolean
117			"""
118			def file_is_md(filename):
119			file_handle = open(filename)
120			line = file_handle.readline()
121
122			while line:
123			if "<OpenMD version=" in line:
124			return True
125			line = file_handle.readline()
126
127			return False
128
129			"""
130			Function compares two numbers.
131			@author Samuel Njoroge and ().
132			@param float one - first number to compare.
133			@param float two - second number to compare.
134			@param boolean ignore_sign - if sign is a factor in the comparison.
135			@return float diff - difference between the two numbers.
136			"""
137			def absDiff(one, two, ignore_sign = False):
138			if ignore_sign:
139			one, two = math.fabs(one), math.fabs(two)
140			return max(one, two) - min(one, two)
141
142			"""
143			Function compares two files.
144			@author Samuel Njoroge and ().
145			@param string fExpected - name of the expected file.
146			@param string fNew - name of the new test file.
147			@param float epsilon - Precision of the comparison of the files.
148			@param boolean ignore_sign - if sign will be a factor in comparing the digits.
149			@return boolean
150			"""
151			def compare(fExpected, fNew, epsilon = 0.00001, ignore_sign=False):
152	chuckv	1660	logger = logging.getLogger("tcpserver")
153	chuckv	1659	fone = open(fExpected, 'r')
154			ftwo = open(fNew, 'r')
155
156			diffs = 0
157
158			i = 0
159			for lineone in fone:
160			linetwo = ftwo.readline()
161
162			elementsone = lineone.split()
163			elementstwo = linetwo.split()
164
165			i = i + 1
166
167			lenone = len(elementsone)
168			lentwo = len(elementstwo)
169
170			if lenone != lentwo:
171			diffs = diffs + 1
172	chuckv	1660	logger.warning("Line: %d - %s", i, "no mach")
173	chuckv	1659	return True
174			else:
175			for j in range(lenone):
176			# used to ignore XYZ meta data such as # of frames and # of atoms
177			try:
178			feone = int(elementsone[j])
179			fetwo = int(elementstwo[j])
180			# these are ints -- skip this pair
181			continue
182			except ValueError:
183			pass
184			try:
185			feone = float(elementsone[j])
186			fetwo = float(elementstwo[j])
187
188			fediff = absDiff(feone, fetwo, ignore_sign)
189
190			if fediff > epsilon:
191			diffs = diffs + 1
192			print "Line " + str(i) + " do not match in the files."
193			return True
194			except ValueError:
195			pass
196			return False#diffs == 0
197
198			"""
199			Function scans the directory for .md config files which will be run for
200			testing.
201			@author Samuel Njoroge
202			"""
203			def scanForMdFiles():
204			paths = []
205			variable = "sam is here"
206			for p in os.listdir("../samples/"):
207			paths.append(os.path.abspath("../samples/" + p))
208
209			while len(paths):
210			p = paths.pop()
211			if os.path.isfile(p):
212			fraw_list.append(p)
213			else:
214			for np in os.listdir(p):
215			paths.append(os.path.abspath(p + "/" + np))
216
217			for f in fraw_list:
218			if (".md" in f) and (not ".svn" in f) and file_is_md(f):
219			fmd_list.append(f)
220			temp = os.path.basename(f)
221			temp = temp.replace(".md",'')
222			fmd_base_list.append(temp)
223
224			"""
225			Function runs OpenMD on the files the compares them with the already run files (*_v.stat)
226			@author Samuel Njoroge
227			"""
228			def runMdFiles():
229	chuckv	1660	logger = logging.getLogger("tcpserver")
230	chuckv	1659	global dir_base, dir_openmd, dir_cwd
231			output = []
232			for x in range(0, len(fmd_list)):
233			#subprocess.call(["export FORCE_PARAM_PATH=/Users/snjoroge/Documents/openmd/development/forceFields"])
234			if "argon" in fmd_list[x]:
235	chuckv	1660	logger.debug("Switching to Directory: %s", os.path.dirname(fmd_list[x]))
236	chuckv	1659	os.chdir(os.path.dirname(fmd_list[x]))
237	chuckv	1660	logger.debug("Running: %s", fmd_list[x])
238	chuckv	1659	output = subprocess.call([dir_openmd + "/openmd", fmd_list[x]])
239			if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat")):
240			#print "Renaming File: " + fmd_base_list[x] + ".stat - " + fmd_base_list[x] + "_v.stat"
241			#subprocess.call(["cp", os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat", os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + "_v.stat"])
242	chuckv	1660	logger.debug("Comparing: %s", "Comparing: " + fmd_base_list[x] + ".stat <=> " + fmd_base_list[x] + "_v.stat")
243	chuckv	1659	if(compare(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat", os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + "_v.stat")):
244	chuckv	1660	logger.warning("Files: %s", "Files do not match.")
245	chuckv	1659	else:
246	chuckv	1660	logger.debug("Files Match")
247	chuckv	1659	os.chdir(dir_base)
248
249			def cleanUp():
250	chuckv	1660	print "Delete all files generated."
251	chuckv	1659	for x in range(0, len(fmd_list)):
252	chuckv	1660	if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".eor")):
253			print "DELETE:" + os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".eor"
254			os.remove(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".eor")
255			if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat")):
256			print "DELETE:" + os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat"
257			os.remove(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".stat")
258			if(os.path.exists(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".dump")):
259			print "DELETE:" + os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".dump"
260			os.remove(os.path.dirname(fmd_list[x]) + "/" + fmd_base_list[x] + ".dump")
261	chuckv	1659