source: branches/testScripts/nightlyBuild.py @ 630

Last change on this file since 630 was 630, checked in by jpfasano, 12 years ago

Moved code that runs an svn command from nightlyBuild.py to its own file.

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 11.0 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5import re 
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13
14# TODO:
15#   -After "svn co" then get all 3rd party packages.
16#   -Get some information about the platform and put this in email failure message.
17#   -Implement Kipp's vpath (delete vpath instead of 'make distclean').
18#   Break this file up into multiple files so it is manageable.
19#   Don't do build if 'svn update' doesn't change anything and prior test was OK.
20#     (no need to re-run if nothing has changed since prior run)
21#   Build both trunk and latest stable
22#   Build both optimized and debug (or have a set of config-site scripts to test?)
23#   Check the testing of the success criteria of each projects "make test"
24#   Implement "cbc -miplib" test for successful run.  JohnF sent JP the criteria
25#     to test on in an email dated 10/12/2007 12:01pm
26
27
28
29#------------------------------------------------------------------------
30# Function to Check Return Code from unitTest
31#------------------------------------------------------------------------
32def didTestFail( result, project, buildStep ) :
33  retVal = 0
34
35  # If the return code is not 0, then failure
36  if result['returnCode'] != 0 :
37    retVal = 1
38
39  # Many tests write a "Success" message.
40  # For test that do this, check for the success message
41  if NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS.has_key(project) : 
42    if buildStep in NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS[project] :
43      # Is the success message contained in the output?
44      if result['stderr'].rfind("All tests completed successfully") == -1 and \
45         result['stdout'].rfind("All tests completed successfully") == -1 :
46        # Success message not found, assume test failed
47        retVal = 1
48
49  #---------------------------------------------------------------------
50  # Some (project,buildStep) pairs require further checking
51  # to determine if they were successful
52  #---------------------------------------------------------------------
53  # Clp's "./clp -unitTest dirNetlib=_NETLIBDIR_ -netlib"
54  if project=='Clp' and buildStep==NBprojectConfig.UNITTEST_CMD['Clp'] :
55    # Check that last netlib test case ran by looking for message of form
56    # '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
57    reexp = r"(.|\n)*(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
58    msgTail = result['stdout'][-200:]
59    if not re.compile(reexp).match(msgTail,1) :
60      # message not found, assume test failed
61      retVal = 1
62     
63  # Cbc's "make test"
64  elif project=='Cbc' and buildStep=='make test' :
65    # Check that last the last few lines are of the form
66    # 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'
67    reexp=r"(.|\n)*cbc_clp solved 2 out of 2 and took (\d*\.\d*) seconds."
68    msgTail = result['stdout'][-300:]
69    if not re.compile(reexp).match(msgTail,1) :
70      # message not found, assume test failed
71      retVal = 1
72
73  # Cbc's "./cbc -unitTest dirNetlib=_MIPLIB3DIR_ -miplib"
74  elif project=='Cbc' and buildStep==NBprojectConfig.UNITTEST_CMD['Cbc'] :
75    if result['returnCode']>=0 and result['returnCode']<=2 :
76      # return code is between 0 and 2.
77      # Return code between 1 and 44 is the number of test cases that
78      # ended because maxnodes limit reached.  John Forrest says if this
79      # is less than 3, the OK.
80      retVal=0
81    else :
82      retVal=1
83
84  return retVal
85
86#------------------------------------------------------------------------
87#  Main Program Starts Here 
88#------------------------------------------------------------------------
89
90#------------------------------------------------------------------------
91#  If needed create the top level directory
92#------------------------------------------------------------------------
93if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
94  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
95os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
96
97#------------------------------------------------------------------------
98#  Get the data directories if they don't already exist
99#------------------------------------------------------------------------
100dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
101if not os.path.isdir(dataBaseDir) :
102  os.makedirs(dataBaseDir)
103dataDirs=['Netlib','miplib3']
104for d in dataDirs :
105  dataDir=os.path.join(dataBaseDir,d)
106  if not os.path.isdir(dataDir) :
107    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
108    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
109      sys.exit(1)
110    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
111netlibDir=os.path.join(dataBaseDir,'Netlib')
112miplib3Dir=os.path.join(dataBaseDir,'miplib3')
113
114#------------------------------------------------------------------------
115# Loop once for each project
116#------------------------------------------------------------------------
117for p in NBuserConfig.PROJECTS:
118  NBlogMessages.writeMessage( p )
119
120  #---------------------------------------------------------------------
121  # svn checkout or update the project
122  #---------------------------------------------------------------------
123  projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
124  projectCheckOutDir=os.path.join(projectBaseDir,'trunk')
125  if not os.path.isdir(projectBaseDir) :
126    os.makedirs(projectBaseDir)
127  if not os.path.isdir(projectCheckOutDir) :
128    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/'+p+'/trunk trunk'
129    if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
130      continue
131  else :
132    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
133    if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
134      continue
135
136  #---------------------------------------------------------------------
137  # Should probably run make 'distclean' to do a build from scrath
138  # or delete the VPATH directory when there is one
139  #---------------------------------------------------------------------
140
141
142  #---------------------------------------------------------------------
143  # Run configure part of buid
144  #---------------------------------------------------------------------
145  os.chdir(projectCheckOutDir)
146  configCmd = os.path.join('.','configure -C')
147  NBlogMessages.writeMessage('  '+configCmd)
148  result=NBosCommand.run(configCmd)
149 
150  # Check if configure worked
151  if result['returnCode'] != 0 :
152    error_msg = result
153    # Add contents of log file to message
154    logFileName = 'config.log'
155    if os.path.isfile(logFileName) :
156      logFilePtr = open(logFileName,'r')
157      error_msg['config.log']  = "config.log contains: \n" 
158      error_msg['config.log'] += logFilePtr.read()
159      logFilePtr.close()
160    NBemail.sendCmdMsgs(p,error_msg,configCmd)
161    continue
162
163  #---------------------------------------------------------------------
164  # Run make part of build
165  #---------------------------------------------------------------------
166  NBlogMessages.writeMessage( '  make' )
167  result=NBosCommand.run('make')
168 
169  # Check if make worked
170  if result['returnCode'] != 0 :
171    NBemail.sendCmdMsgs(p,result,'make')
172    continue
173
174  #---------------------------------------------------------------------
175  # Run 'make test' part of build
176  #---------------------------------------------------------------------
177  NBlogMessages.writeMessage( '  make test' )
178  result=NBosCommand.run('make test')
179 
180  # Check if 'make test' worked
181  if didTestFail(result,p,"make test") :
182    NBemail.sendCmdMsgs(p,result,"make test")
183    continue
184
185  #---------------------------------------------------------------------
186  # Run unitTest if available and different from 'make test'
187  #---------------------------------------------------------------------
188  if NBprojectConfig.UNITTEST_CMD.has_key(p) :
189    unitTestPath = os.path.join(projectCheckOutDir,NBprojectConfig.UNITTEST_DIR[p])
190    os.chdir(unitTestPath)
191
192    unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
193    unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
194    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
195
196    NBlogMessages.writeMessage( '  '+unitTestCmd )
197    result=NBosCommand.run(unitTestCmd)
198 
199    if didTestFail(result,p,unitTestCmdTemplate) :
200      NBemail.sendCmdMsgs(p,result,unitTestCmd)
201      continue
202
203  # For testing purposes only do first successful project
204  #break
205
206
207NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
208
209sys.exit(0)
210
211
212# START KIPP
213#----------------------------------------------------------------------
214# CONFIG FILE PATH:
215#   path to the config file for the build
216#   done. If the directory does not exist, it will be created.
217#   this should have all of the user specific data
218#   it should have values for
219#   NIGHTLY_BUILD_ROOT
220#   NBuserConfig.SMTP_SERVER_NAME
221#   NBuserConfig.SMTP_SERVER_PORT
222#   NBuserConfig.SMTP_SSL_SERVER
223#   NBuserConfig.SMTP_USER_NAME
224#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
225#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
226#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
227#   
228#----------------------------------------------------------------------
229
230CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
231CONFIG_FILENAME = 'config.txt'
232
233
234# Get configFile data
235
236configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
237                                 os.path.basename(CONFIG_FILENAME ))
238if os.path.isfile(  configFile) :
239  pwFilePtr = open(configFile ,'r')
240  d = pwFilePtr.readlines()
241  # do pwFilePtr.read() to get a string object
242  # we have a list object
243  print d[0]
244  print d[1]
245  # make a dictionary
246  config_dic = {}
247
248  #smtppass  = pwFilePtr.read().strip()
249  pwFilePtr.close()
250else :
251  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
252  #print cmdMsgs
253  sys.exit( 1)
254sys.exit( 0)
255
256
257
258# START KIPP
259#----------------------------------------------------------------------
260#   path to the config file for the build
261#   get the user dependent variables
262# CONFIG FILE PATH:
263#   it should have values for
264#   NIGHTLY_BUILD_ROOT
265#   NBuserConfig.SMTP_SERVER_NAME
266#   NBuserConfig.SMTP_SERVER_PORT
267#   NBuserConfig.SMTP_SSL_SERVER
268#   NBuserConfig.SMTP_USER_NAME
269#   NBuserConfig.SMTP_PASSWORD_FILENAME
270#   NBuserConfig.SENDER_EMAIL_ADDR
271#   NBuserConfig.MY_EMAIL_ADDR
272#   
273#----------------------------------------------------------------------
274
275CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
276CONFIG_FILENAME = 'config.txt'
277
278
279# Get configFile data
280
281configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
282                                 os.path.basename(CONFIG_FILENAME ))
283if os.path.isfile(  configFile) :
284  pwFilePtr = open(configFile ,'r')
285  d = pwFilePtr.readlines()
286  # do pwFilePtr.read() to get a string object
287  # we have a list object
288  print d[0]
289  print d[1]
290  # make a dictionary
291  config_dic = {}
292
293  #smtppass  = pwFilePtr.read().strip()
294  pwFilePtr.close()
295else :
296  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
297  #print cmdMsgs
298  sys.exit( 1)
299sys.exit( 0)
300
301# END KIPP
Note: See TracBrowser for help on using the repository browser.