source: branches/testScripts/NBcheckResult.py @ 720

Last change on this file since 720 was 720, checked in by jpfasano, 12 years ago

Commit is to copy file from my window to linux box.
Starting to clean up code which checks for success of "make test" and unitTest

  • Property svn:eol-style set to native
File size: 6.2 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import re 
9import os 
10
11import NBprojectConfig
12
13#------------------------------------------------------------------------
14# Determine if a projects "make test" or unitTest ran successfully.
15# Since projects are not consistent in how they report success,
16# this function has specialized code for some projects.
17#------------------------------------------------------------------------
18
19def rc0( result, project ) :
20  retVal = None
21  # If the return code is not 0, then failure
22  if result['returnCode'] != 0 :
23    retVal = "Non-zero return code of "+str(result['returnCode'])
24
25def standardSuccessMessage(result,project) :
26  retVal = None
27  # Is the success message contained in the output?
28  if result['stderr'].rfind("All tests completed successfully") == -1 and \
29     result['stdout'].rfind("All tests completed successfully") == -1 :
30    # Success message not found, assume test failed
31    retVal = "The output does not contain the messages: 'All tests completed successfully'"
32
33
34
35def didTestFail( result, project, buildStep ) :
36  retVal = None
37
38  # If the return code is not 0, then failure
39  retVal=rc0(result,project)
40  if retVal : return retVal
41
42  # Many tests write a "Success" message.
43  # For test that do this, check for the success message
44  if NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS.has_key(project) : 
45    if buildStep in NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS[project] :
46      retVal= standardSucessMessage(result,project)
47      if retVal : return retVal
48
49  #---------------------------------------------------------------------
50  # Some (project,buildStep) pairs require further checking
51  # to determine if they were successful
52  #---------------------------------------------------------------------
53  # Clp's "./clp -unitTest dirNetlib=_NETLIBDIR_ -netlib"
54  if project=='Clp' and buildStep==NBprojectConfig.UNITTEST_CMD['Clp'] :
55    # Check that last netlib test case ran by looking for message of form
56    # '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
57    reexp = r"(.|\n)*(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
58    msgTail = result['stdout'][-200:]
59    if not re.compile(reexp).match(msgTail,1) :
60      # message not found, assume test failed
61      retVal = "Did not complete the woodw testcase"
62     
63  # Cbc's "make test"
64  elif project=='Cbc' and buildStep=='make test' :
65    # Check that last the last few lines are of the form
66    # 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'
67    reexp=r"(.|\n)*cbc_clp solved 2 out of 2 and took (\d*\.\d*) seconds."
68    msgTail = result['stdout'][-300:]
69    if not re.compile(reexp).match(msgTail,1) :
70      # message not found, assume test failed
71      retVal = "Did not display message 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'" 
72
73  # Cbc's "./cbc -unitTest dirNetlib=_MIPLIB3DIR_ -miplib"
74  elif project=='Cbc' and buildStep==NBprojectConfig.UNITTEST_CMD['Cbc'] :
75    if result['returnCode']>=0 and result['returnCode']<=2 :
76      # return code is between 0 and 2.
77      # Return code between 1 and 44 is the number of test cases that
78      # ended because maxnodes limit reached.  John Forrest says if this
79      # is less than 3, the OK.
80      retVal=None
81    else :
82      retVal = "Return code of "+str(result['returnCode'])+" which is > 2."
83
84  # DyLP's "make test"
85  # DyLP's "./unitTest -testOsiSolverInterface -netlibDir=_NETLIBDIR_"
86  # Osi's "make test"
87  # Osi's "./unitTest -testOsiSolverInterface -netlibDir=_NETLIBDIR_"
88  elif project=='DyLP' and buildStep=='make test' or \
89       project=='DyLP' and buildStep==NBprojectConfig.UNITTEST_CMD['Osi'] or \
90       project=='Osi'  and buildStep=='make test' or \
91       project=='Osi'  and buildStep==NBprojectConfig.UNITTEST_CMD['Osi'] :
92    # Messages should not contain:
93    # "*** xxxSolverInterface testing issue: whatever the problem is"
94    reexp=r'.*\*\*.+SolverInterface testing issue:.*'
95    if re.compile(reexp).match(result['stderr'],1) :
96      # message found, assume test failed
97      retVal = "Issued message: 'SolverInterface tessting issue:'"
98    if re.compile(reexp).match(result['stdout'],1) :
99      # message found, assume test failed
100      retVal = "Issued message: 'SolverInterface tessting issue:'"
101
102    if project=='Osi'  and buildStep==NBprojectConfig.UNITTEST_CMD['Osi'] :
103
104      # Look for pattern "<solver> solved NN out of 90 and took nnn.xx seconds"
105      r=r'((.+) solved (\d+) out of 90 and took (\d*\.\d*) seconds)'
106      osisSummaryResult=re.findall(r,result['stdout'][-800:])
107      expectedOsis=['clp','sym','dylp','cbcclp']
108      for osi in osisSummaryResult :
109        if osi[1] in expectedOsis: expectedOsis.remove(osi[1])
110        numSolved = int(osi[2])
111        # Sym only solves 89 of the 90
112        if osi[1]=='sym':
113          if numSolved<89 :
114            retVal=osi[1]+\
115                 " only solved "\
116                 +osi[2]\
117                 +" out of 90 in "\
118                 +osi[3]+" seconds"
119        elif numSolved<90 :
120          retVal=osi[1]+\
121                 " only solved "\
122                 +osi[2]+\
123                 " out of 90 in "\
124                 +osi[3]+" seconds"
125      if len(expectedOsis)!=0 :
126        retVal="Osi "+expectedOsis[0]+" did not report number solved"
127
128  return retVal
129
130#-------------------------------------------------------------------------
131#
132# Determine if config needs to be run.
133# If there is a config.log file and it indicates that it ran successfully
134# then config should not need to be rerun.
135#
136#-------------------------------------------------------------------------
137def didConfigRunOK( ) :
138  retVal=1
139  logFileName='config.log'
140  if not os.path.isfile(logFileName) :
141    retVal=0
142  else :
143    # Read logfile
144    logFilePtr = open(logFileName,'r')
145    logfile = logFilePtr.read()
146    logFilePtr.close()
147
148    # If logfile does not contain "configure: exit 0" then assume
149    # that configure needs to be rerun
150    if logfile.rfind("configure: exit 0") == -1 :
151      retVal=0
152
153  return retVal
Note: See TracBrowser for help on using the repository browser.