source: coopr.opt/trunk/coopr/opt/testing/pyunit.py @ 2258

Last change on this file since 2258 was 2258, checked in by wehart, 10 years ago

Adding a subpackage for optimization-specific testing tools.
Currently, this contains a customized PyUnit? testing class
that contains new testing functions.

File size: 5.9 KB
Line 
1#  _________________________________________________________________________
2#
3#  PyUtilib: A Python utility library.
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  _________________________________________________________________________
9
10
11__all__ = ['TestCase']
12
13import os
14import pyutilib.th
15import copy
16import pyutilib.subprocess
17from coopr.opt import SolverResults
18import re
19from StringIO import StringIO
20import math
21import yaml
22
23
24def yaml_fix(val):
25    if not isinstance(val,basestring):
26        return val
27    return val.replace(':','\\x3a')
28
29def extract_results(lines):
30    ans = []
31    status = 0
32    for line in lines:
33        tokens = re.split('[\t ]+',line.strip())
34        if tokens[:6] == ['#','=','Solver','Results','-','BEGIN']:
35            status = 1
36        if tokens[:6] == ['#','=','Solver','Results','-','END']:
37            return "\n".join(ans)
38        if status:
39            if tokens[0] != '#':
40                ans.append(line)
41    return "\n".join(ans)
42
43
44def recursive_compare(baseline, output, tolerance, prefix="<root>"):
45    if type(baseline) != type(output):
46        raise IOError, "(%s) Structural difference: baseline=%s output=%s" % (prefix, str(baseline), str(output))
47    #
48    if type(baseline) is list:
49        if len(baseline) > len(output):
50            raise IOError, "(%s) Baseline has longer list than output: baseline=%s output=%s" % (prefix, str(baseline), str(output))
51        for i in xrange(len(baseline)):
52            recursive_compare(baseline[i], output[i], tolerance, prefix=prefix+"["+str(i)+"]")
53    #
54    elif type(baseline) is dict:
55        for key in baseline:
56            if not key in output:
57                raise IOError, "(%s) Baseline key %s that does not exist in output: baseline=%s output=%s" % (prefix, key, str(baseline), str(output))
58            recursive_compare(baseline[key], output[key], tolerance, prefix=prefix+"."+key)
59    #
60    else:
61        if (type(baseline) is float or type(output) is float) and type(baseline) in [int,float] and type(output) in [int,float] and math.fabs(baseline-output) > tolerance:
62            raise ValueError, "(%s) Floating point values differ: baseline=%f and output=%f" % (prefix, baseline, output)
63        elif baseline != output:
64            raise ValueError, "(%s) Values differ: baseline=%s and output=%s" % (prefix, str(baseline), str(output))
65       
66   
67def compare_results(output_, baseline_, tolerance=1e-6):
68    options = copy.copy(SolverResults.default_print_options)
69    options.ignore_defaults=True
70    #
71    #print "Reading test output results ..."
72    ##output = SolverResults()
73    ##output.read(istream=StringIO(output_))
74    ##orepn = output._repn_(options)
75    try:
76        output__ = output_
77        orepn = yaml.load(StringIO(output__), Loader=yaml.SafeLoader)
78    except yaml.scanner.ScannerError, err:
79        print "ERROR parsing solver results: %s" % str(err)
80        print ""
81        print "OUTPUT:"
82        print output__
83        raise IOError, "YAML parse error"
84    #
85    #print "Reading baseline results ..."
86    ##baseline = SolverResults()
87    ##baseline.read(istream=StringIO(baseline_))
88    ##brepn = baseline._repn_(options)
89    try:
90        baseline__ = baseline_
91        brepn = yaml.load(StringIO(baseline__), Loader=yaml.SafeLoader)
92    except yaml.scanner.ScannerError, err:
93        print "ERROR parsing baseline results: %s" % str(err)
94        print ""
95        print "OUTPUT:"
96        print baseline__
97        raise IOError, "YAML parse error"
98    #print "Comparing ..."
99    recursive_compare(brepn, orepn, tolerance)
100   
101
102def _failIfCooprResultsDiffer(self, cmd=None, baseline=None, cwd=None):
103    if cwd is not None:
104        oldpwd = os.getcwd()
105        os.chdir(cwd)
106    output = pyutilib.subprocess.run(cmd)
107    if output[0] != 0:
108        self.fail("Command terminated with nonzero status: '%s'" % cmd)
109    results = extract_results( re.split('\n',output[1]) )
110    if os.path.exists(baseline):
111        INPUT = open(baseline, 'r')
112        baseline = "\n".join(INPUT.readlines())
113        INPUT.close()
114    try:
115        compare_results(results, baseline)
116    except IOError, err:
117        self.fail("Command failed to generate results that can be compared with the baseline: '%s'" % str(err))
118    except ValueError, err:
119        self.fail("Difference between results and baseline: '%s'" % str(err))
120
121
122class TestCase(pyutilib.th.TestCase):
123
124    def __init__(self, methodName='runTest'):
125        pyutilib.th.TestCase.__init__(self, methodName)
126
127    def failIfCooprResultsDiffer(self, cmd, baseline, cwd=None):
128        _failIfCooprresultsDiffer(self, cmd=cmd, baseline=baseline, cwd=cwd)
129
130    def add_coopr_results_test(name=None, cmd=None, fn=None, baseline=None):
131        if cmd is None and fn is None:
132            print "ERROR: must specify either the 'cmd' or 'fn' option to define how the output file is generated"
133            return
134        if name is None and baseline is None:
135            print "ERROR: must specify a baseline comparison file, or the test name"
136            return
137        if baseline is None:
138            baseline=name+".txt"
139        tmp = name.replace("/","_")
140        tmp = tmp.replace("\\","_")
141        tmp = tmp.replace(".","_")
142        #
143        # Create an explicit function so we can assign it a __name__ attribute.
144        # This is needed by the 'nose' package
145        #
146        if fn is None:
147            currdir=os.getcwd()
148            func = lambda self,c1=currdir,c2=cmd,c3=tmp+".out",c4=baseline: _failIfCooprResultsDiffer(self,cwd=c1,cmd=c2,baseline=c4)
149        else:
150            # This option isn't implemented...
151            sys.exit(1)
152            func = lambda self,c1=fn,c2=tmp,c3=baseline: _failIfCooprResultsDiffer(self,fn=c1,name=c2,baseline=c3)
153        func.__name__ = "test_"+tmp
154        setattr(TestCase, "test_"+tmp, func)
155    add_coopr_results_test=staticmethod(add_coopr_results_test)
156
157
158
Note: See TracBrowser for help on using the repository browser.