source: coopr.opt/trunk/coopr/opt/testing/pyunit.py @ 2259

Last change on this file since 2259 was 2259, checked in by wehart, 10 years ago

Adding new termination conditions.

Fixing problem setting up testing utilities when YAML is not installed.

File size: 6.3 KB
Line 
1#  _________________________________________________________________________
2#
3#  PyUtilib: A Python utility library.
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  _________________________________________________________________________
9
10
11__all__ = ['TestCase']
12
13import os
14import pyutilib.th
15import copy
16import pyutilib.subprocess
17from coopr.opt import SolverResults
18import re
19from StringIO import StringIO
20import math
21try:
22    import yaml
23    using_yaml = True
24except Exception:
25    using_yaml = False
26
27
28def yaml_fix(val):
29    if not isinstance(val,basestring):
30        return val
31    return val.replace(':','\\x3a')
32
33def extract_results(lines):
34    ans = []
35    status = 0
36    for line in lines:
37        tokens = re.split('[\t ]+',line.strip())
38        if tokens[:6] == ['#','=','Solver','Results','-','BEGIN']:
39            status = 1
40        if tokens[:6] == ['#','=','Solver','Results','-','END']:
41            return "\n".join(ans)
42        if status:
43            if tokens[0] != '#':
44                ans.append(line)
45    return "\n".join(ans)
46
47
48def recursive_compare(baseline, output, tolerance, prefix="<root>"):
49    if type(baseline) != type(output):
50        raise IOError, "(%s) Structural difference: baseline=%s output=%s" % (prefix, str(baseline), str(output))
51    #
52    if type(baseline) is list:
53        if len(baseline) > len(output):
54            raise IOError, "(%s) Baseline has longer list than output: baseline=%s output=%s" % (prefix, str(baseline), str(output))
55        for i in xrange(len(baseline)):
56            recursive_compare(baseline[i], output[i], tolerance, prefix=prefix+"["+str(i)+"]")
57    #
58    elif type(baseline) is dict:
59        for key in baseline:
60            if not key in output:
61                raise IOError, "(%s) Baseline key %s that does not exist in output: baseline=%s output=%s" % (prefix, key, str(baseline), str(output))
62            recursive_compare(baseline[key], output[key], tolerance, prefix=prefix+"."+key)
63    #
64    else:
65        if (type(baseline) is float or type(output) is float) and type(baseline) in [int,float] and type(output) in [int,float] and math.fabs(baseline-output) > tolerance:
66            raise ValueError, "(%s) Floating point values differ: baseline=%f and output=%f" % (prefix, baseline, output)
67        elif baseline != output:
68            raise ValueError, "(%s) Values differ: baseline=%s and output=%s" % (prefix, str(baseline), str(output))
69       
70   
71def compare_results(output_, baseline_, tolerance=1e-6):
72    if not using_yaml:
73        raise ValueError, "Cannot compare results without a YAML parser"
74    options = copy.copy(SolverResults.default_print_options)
75    options.ignore_defaults=True
76    #
77    #print "Reading test output results ..."
78    ##output = SolverResults()
79    ##output.read(istream=StringIO(output_))
80    ##orepn = output._repn_(options)
81    try:
82        output__ = output_
83        orepn = yaml.load(StringIO(output__), Loader=yaml.SafeLoader)
84    except yaml.scanner.ScannerError, err:
85        print "ERROR parsing solver results: %s" % str(err)
86        print ""
87        print "OUTPUT:"
88        print output__
89        raise IOError, "YAML parse error"
90    #
91    #print "Reading baseline results ..."
92    ##baseline = SolverResults()
93    ##baseline.read(istream=StringIO(baseline_))
94    ##brepn = baseline._repn_(options)
95    try:
96        baseline__ = baseline_
97        brepn = yaml.load(StringIO(baseline__), Loader=yaml.SafeLoader)
98    except yaml.scanner.ScannerError, err:
99        print "ERROR parsing baseline results: %s" % str(err)
100        print ""
101        print "OUTPUT:"
102        print baseline__
103        raise IOError, "YAML parse error"
104    #print "Comparing ..."
105    recursive_compare(brepn, orepn, tolerance)
106   
107
108def _failIfCooprResultsDiffer(self, cmd=None, baseline=None, cwd=None):
109    if cwd is not None:
110        oldpwd = os.getcwd()
111        os.chdir(cwd)
112    output = pyutilib.subprocess.run(cmd)
113    if output[0] != 0:
114        self.fail("Command terminated with nonzero status: '%s'" % cmd)
115    results = extract_results( re.split('\n',output[1]) )
116    if os.path.exists(baseline):
117        INPUT = open(baseline, 'r')
118        baseline = "\n".join(INPUT.readlines())
119        INPUT.close()
120    try:
121        compare_results(results, baseline)
122    except IOError, err:
123        self.fail("Command failed to generate results that can be compared with the baseline: '%s'" % str(err))
124    except ValueError, err:
125        self.fail("Difference between results and baseline: '%s'" % str(err))
126
127
128class TestCase(pyutilib.th.TestCase):
129
130    def __init__(self, methodName='runTest'):
131        pyutilib.th.TestCase.__init__(self, methodName)
132
133    def failIfCooprResultsDiffer(self, cmd, baseline, cwd=None):
134        if not using_yaml:
135            self.fail("Cannot compare Coopr results because PyYaml is not installed")
136        _failIfCooprresultsDiffer(self, cmd=cmd, baseline=baseline, cwd=cwd)
137
138    def add_coopr_results_test(name=None, cmd=None, fn=None, baseline=None):
139        if not using_yaml:
140            return
141        if cmd is None and fn is None:
142            print "ERROR: must specify either the 'cmd' or 'fn' option to define how the output file is generated"
143            return
144        if name is None and baseline is None:
145            print "ERROR: must specify a baseline comparison file, or the test name"
146            return
147        if baseline is None:
148            baseline=name+".txt"
149        tmp = name.replace("/","_")
150        tmp = tmp.replace("\\","_")
151        tmp = tmp.replace(".","_")
152        #
153        # Create an explicit function so we can assign it a __name__ attribute.
154        # This is needed by the 'nose' package
155        #
156        if fn is None:
157            currdir=os.getcwd()
158            func = lambda self,c1=currdir,c2=cmd,c3=tmp+".out",c4=baseline: _failIfCooprResultsDiffer(self,cwd=c1,cmd=c2,baseline=c4)
159        else:
160            # This option isn't implemented...
161            sys.exit(1)
162            func = lambda self,c1=fn,c2=tmp,c3=baseline: _failIfCooprResultsDiffer(self,fn=c1,name=c2,baseline=c3)
163        func.__name__ = "test_"+tmp
164        setattr(TestCase, "test_"+tmp, func)
165    add_coopr_results_test=staticmethod(add_coopr_results_test)
166
167
168
Note: See TracBrowser for help on using the repository browser.