Ignore:
Timestamp:
Nov 2, 2007 9:55:56 AM (13 years ago)
Author:
jpfasano
Message:

Mofified NBcheckResult so that it is a bunch of small functions that check for one specific thing.
NBprojectConfig now defines a dictionary where the key is the project.
The value is a list of functions that need to be run to check the success of a particular test run.
The code still needs to be cleaned up, but it seems to be running and was time to do a check-in so I can move it to other platforms for testing.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/testScripts/NBcheckResult.py

    r722 r730  
    1717#------------------------------------------------------------------------
    1818
     19# Assume result is always correct
     20def anythingGoes( result, project ) :
     21  retVal = None
     22  return retVal
     23
     24# Is the rc=0?
    1925def rc0( result, project ) :
    2026  retVal = None
     
    2228  if result['returnCode'] != 0 :
    2329    retVal = "Non-zero return code of "+str(result['returnCode'])
     30  return retVal 
     31   
     32# Is 0<=rc<=2?
     33def rc0to2( result, project ) :
     34  retVal = None
     35  # If the return code is not 0, then failure
     36  if 0>result['returnCode'] or result['returnCode']>2 :
     37    retVal = "Return code out of range. Expected: 0<=rc<=2. rc="+str(result['returnCode'])
     38  return retVal
    2439
     40# Was the standard success message written?
    2541def standardSuccessMessage(result,project) :
    2642  retVal = None
     
    3046    # Success message not found, assume test failed
    3147    retVal = "The output does not contain the messages: 'All tests completed successfully'"
     48  return retVal 
    3249
     50# Was woodw the last netlib problem run?
     51# Check that last netlib test case ran by looking for message of form
     52# '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
     53def endWithWoodw(result,project) :
     54  retVal = None
     55  reexp = r"(.|\n)*(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
     56  msgTail = result['stdout'][-200:]
     57  if not re.compile(reexp).match(msgTail,1) :
     58    # message not found, assume test failed
     59    retVal = "Did not complete the woodw testcase"
     60  return retVal
    3361
     62# Did Cbc 'make test' write its success message?
     63# Check that last the last few lines are of the form
     64# 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'
     65def cbcMakeTestSuccessMessage(result,project) :
     66  retVal=None
     67  reexp=r"(.|\n)*cbc_clp solved 2 out of 2 and took (\d*\.\d*) seconds."
     68  msgTail = result['stdout'][-300:]
     69  if not re.compile(reexp).match(msgTail,1) :
     70    # message not found, assume test failed
     71    retVal = "Did not display message 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'"
     72  return retVal
    3473
    35 def didTestFail( result, project, buildStep ) :
     74# Messages must not contain:
     75# "*** xxxSolverInterface testing issue: whatever the problem is"
     76def noSolverInterfaceTestingIssueMessage(result,project):
    3677  retVal = None
     78  reexp=r'.*\*\*.+SolverInterface testing issue:.*'
     79  if re.compile(reexp).match(result['stderr'],1) :
     80    # message found, assume test failed
     81    retVal = "Issued message: 'SolverInterface tessting issue:'"
     82  if re.compile(reexp).match(result['stdout'],1) :
     83    # message found, assume test failed
     84    retVal = "Issued message: 'SolverInterface tessting issue:'"
     85  return retVal
     86     
    3787
    38   # If the return code is not 0, then failure
    39   retVal=rc0(result,project)
    40   if retVal : return retVal
     88# Look for pattern "<solver> solved NN out of 90 and took nnn.xx seconds"
     89def OsiUnitTestSuccessMessages(result,project):
     90  retVal = None
     91  # Look for pattern "<solver> solved NN out of 90 and took nnn.xx seconds"
     92  r=r'((.+) solved (\d+) out of 90 and took (\d*\.\d*) seconds)'
     93  osisSummaryResult=re.findall(r,result['stdout'][-800:])
     94  expectedOsis=['clp','sym','dylp','cbcclp']
     95  for osi in osisSummaryResult :
     96    if osi[1] in expectedOsis: expectedOsis.remove(osi[1])
     97    numSolved = int(osi[2])
     98    # Sym only solves 89 of the 90
     99    if osi[1]=='sym':
     100      if numSolved<89 :
     101        retVal=osi[1]+\
     102               " only solved "\
     103               +osi[2]\
     104               +" out of 90 in "\
     105               +osi[3]+" seconds"
     106    elif numSolved<90 :
     107      retVal=osi[1]+\
     108               " only solved "\
     109               +osi[2]+\
     110               " out of 90 in "\
     111               +osi[3]+" seconds"
     112  if len(expectedOsis)!=0 :
     113        retVal="Osi "+expectedOsis[0]+" did not report number solved"
     114  return retVal     
    41115
    42   # Many tests write a "Success" message.
    43   # For test that do this, check for the success message
    44   if NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS.has_key(project) :
    45     if buildStep in NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS[project] :
    46       retVal= standardSuccessMessage(result,project)
    47       if retVal : return retVal
    48 
    49   #---------------------------------------------------------------------
    50   # Some (project,buildStep) pairs require further checking
    51   # to determine if they were successful
    52   #---------------------------------------------------------------------
    53   # Clp's "./clp -unitTest dirNetlib=_NETLIBDIR_ -netlib"
    54   if project=='Clp' and buildStep==NBprojectConfig.UNITTEST_CMD['Clp'] :
    55     # Check that last netlib test case ran by looking for message of form
    56     # '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
    57     reexp = r"(.|\n)*(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
    58     msgTail = result['stdout'][-200:]
    59     if not re.compile(reexp).match(msgTail,1) :
    60       # message not found, assume test failed
    61       retVal = "Did not complete the woodw testcase"
    62      
    63   # Cbc's "make test"
    64   elif project=='Cbc' and buildStep=='make test' :
    65     # Check that last the last few lines are of the form
    66     # 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'
    67     reexp=r"(.|\n)*cbc_clp solved 2 out of 2 and took (\d*\.\d*) seconds."
    68     msgTail = result['stdout'][-300:]
    69     if not re.compile(reexp).match(msgTail,1) :
    70       # message not found, assume test failed
    71       retVal = "Did not display message 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'"
    72 
    73   # Cbc's "./cbc -unitTest dirNetlib=_MIPLIB3DIR_ -miplib"
    74   elif project=='Cbc' and buildStep==NBprojectConfig.UNITTEST_CMD['Cbc'] :
    75     if result['returnCode']>=0 and result['returnCode']<=2 :
    76       # return code is between 0 and 2.
    77       # Return code between 1 and 44 is the number of test cases that
    78       # ended because maxnodes limit reached.  John Forrest says if this
    79       # is less than 3, the OK.
    80       retVal=None
    81     else :
    82       retVal = "Return code of "+str(result['returnCode'])+" which is > 2."
    83 
    84   # DyLP's "make test"
    85   # DyLP's "./unitTest -testOsiSolverInterface -netlibDir=_NETLIBDIR_"
    86   # Osi's "make test"
    87   # Osi's "./unitTest -testOsiSolverInterface -netlibDir=_NETLIBDIR_"
    88   elif project=='DyLP' and buildStep=='make test' or \
    89        project=='DyLP' and buildStep==NBprojectConfig.UNITTEST_CMD['Osi'] or \
    90        project=='Osi'  and buildStep=='make test' or \
    91        project=='Osi'  and buildStep==NBprojectConfig.UNITTEST_CMD['Osi'] :
    92     # Messages should not contain:
    93     # "*** xxxSolverInterface testing issue: whatever the problem is"
    94     reexp=r'.*\*\*.+SolverInterface testing issue:.*'
    95     if re.compile(reexp).match(result['stderr'],1) :
    96       # message found, assume test failed
    97       retVal = "Issued message: 'SolverInterface tessting issue:'"
    98     if re.compile(reexp).match(result['stdout'],1) :
    99       # message found, assume test failed
    100       retVal = "Issued message: 'SolverInterface tessting issue:'"
    101 
    102     if project=='Osi'  and buildStep==NBprojectConfig.UNITTEST_CMD['Osi'] :
    103 
    104       # Look for pattern "<solver> solved NN out of 90 and took nnn.xx seconds"
    105       r=r'((.+) solved (\d+) out of 90 and took (\d*\.\d*) seconds)'
    106       osisSummaryResult=re.findall(r,result['stdout'][-800:])
    107       expectedOsis=['clp','sym','dylp','cbcclp']
    108       for osi in osisSummaryResult :
    109         if osi[1] in expectedOsis: expectedOsis.remove(osi[1])
    110         numSolved = int(osi[2])
    111         # Sym only solves 89 of the 90
    112         if osi[1]=='sym':
    113           if numSolved<89 :
    114             retVal=osi[1]+\
    115                  " only solved "\
    116                  +osi[2]\
    117                  +" out of 90 in "\
    118                  +osi[3]+" seconds"
    119         elif numSolved<90 :
    120           retVal=osi[1]+\
    121                  " only solved "\
    122                  +osi[2]+\
    123                  " out of 90 in "\
    124                  +osi[3]+" seconds"
    125       if len(expectedOsis)!=0 :
    126         retVal="Osi "+expectedOsis[0]+" did not report number solved"
    127 
    128   return retVal
    129116
    130117#-------------------------------------------------------------------------
Note: See TracChangeset for help on using the changeset viewer.