source: branches/testScripts/nightlyBuild.py @ 618

Last change on this file since 618 was 617, checked in by jpfasano, 13 years ago

Moved email functions to their own file

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 10.7 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5import commands
6import smtplib
7import re 
8
9import NBuserConfig
10import NBprojectConfig
11import NBlogMessages
12import NBemail
13
14# TODO:
15#   -After "svn co" then get all 3rd party packages.
16#   -Get some information about the platform and put this in email failure message.
17#   -Implement Kipp's vpath (delete vpath instead of 'make distclean').
18#   Break this file up into multiple files so it is manageable.
19#   Don't do build if 'svn update' doesn't change anything and prior test was OK.
20#     (no need to re-run if nothing has changed since prior run)
21#   Build both trunk and latest stable
22#   Build both optimized and debug (or have a set of config-site scripts to test?)
23#   Check the testing of the success criteria of each projects "make test"
24#   Implement "cbc -miplib" test for successful run.  JohnF sent JP the criteria
25#     to test on in an email dated 10/12/2007 12:01pm
26
27
28#------------------------------------------------------------------------
29# Function to Check Return Code from unitTest
30#------------------------------------------------------------------------
31def didTestFail( rc, project, buildStep ) :
32  retVal = 0
33
34  # If the return code is not 0, then failure
35  if rc[0] != 0 :
36    retVal = 1
37
38  # Many tests write a "Success" message.
39  # For test that do this, check for the success message
40  if NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS.has_key(project) : 
41    if buildStep in NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS[project] :
42      # Is the success message contained in the output?
43      if rc[1].rfind("All tests completed successfully") == -1 :
44        # Success message not found, assume test failed
45        retVal = 1
46
47  #---------------------------------------------------------------------
48  # Some (project,buildStep) pairs require further checking
49  # to determine if they were successful
50  #---------------------------------------------------------------------
51  # Clp's "./clp -unitTest -netlib dirNetlib=_NETLIBDIR_"
52  if project=='Clp' and buildStep==NBprojectConfig.UNITTEST_CMD['Clp'] :
53    # Check that last netlib test case ran by looking for message of form
54    # '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
55    reexp = r"(.|\n)*\.\.(\\|/)\.\.(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
56    msgTail = rc[1][-200:]
57    if not re.compile(reexp).match(msgTail,1) :
58      # message not found, assume test failed
59      retVal = 1
60  elif project=='Cbc' and buildStep=='make test' :
61    # Check that last the last few lines are of the form
62    # 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'
63    reexp=r"(.|\n)*cbc_clp solved 2 out of 2 and took (\d*\.\d*) seconds."
64    msgTail = rc[1][-300:]
65    if not re.compile(reexp).match(msgTail,1) :
66      # message not found, assume test failed
67      retVal = 1
68
69  return retVal
70
71#------------------------------------------------------------------------
72# Function for executing svn commands
73#------------------------------------------------------------------------
74def issueSvnCmd(svnCmd,dir,project) :
75  retVal='OK'
76  os.chdir(dir)
77  NBlogMessages.writeMessage('  '+svnCmd)
78  rc=commands.getstatusoutput(svnCmd)
79  if rc[0] != 0 :
80    NBemail.sendCmdMsgs(project,rc[1],svnCmd)
81    retVal='Error'
82  return retVal
83
84#------------------------------------------------------------------------
85#  Main Program Starts Here 
86#------------------------------------------------------------------------
87
88#------------------------------------------------------------------------
89#  If needed create the top level directory
90#------------------------------------------------------------------------
91# rc=commands.getstatusoutput(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
92if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
93  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
94os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
95
96#------------------------------------------------------------------------
97#  Get the data directories if they don't already exist
98#------------------------------------------------------------------------
99dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
100if not os.path.isdir(dataBaseDir) :
101  os.makedirs(dataBaseDir)
102dataDirs=['Netlib','miplib3']
103for d in dataDirs :
104  dataDir=os.path.join(dataBaseDir,d)
105  if not os.path.isdir(dataDir) :
106    svnCmd='svn checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
107    if issueSvnCmd(svnCmd,dataBaseDir,'Data')!='OK' :
108      sys.exit(1)
109    rc=commands.getstatusoutput('find '+d+' -name \*.gz -print | xargs gzip -d')
110netlibDir=os.path.join(dataBaseDir,'Netlib')
111miplib3Dir=os.path.join(dataBaseDir,'miplib3')
112
113#------------------------------------------------------------------------
114# Loop once for each project
115#------------------------------------------------------------------------
116for p in NBuserConfig.PROJECTS:
117  NBlogMessages.writeMessage( p )
118  rc = [0]
119
120  #---------------------------------------------------------------------
121  # svn checkout or update the project
122  #---------------------------------------------------------------------
123  projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
124  projectCheckOutDir=os.path.join(projectBaseDir,'trunk')
125  if not os.path.isdir(projectBaseDir) :
126    os.makedirs(projectBaseDir)
127  if not os.path.isdir(projectCheckOutDir) :
128    svnCmd='svn checkout https://projects.coin-or.org/svn/'+p+'/trunk trunk'
129    if issueSvnCmd(svnCmd,projectBaseDir,p)!='OK' :
130      continue
131  else :
132    svnCmd='svn update'
133    if issueSvnCmd(svnCmd,projectCheckOutDir,p)!='OK' :
134      continue
135
136  #---------------------------------------------------------------------
137  # Should probably run make 'distclean' to do a build from scrath
138  # or delete the VPATH directory when there is one
139  #---------------------------------------------------------------------
140
141
142  #---------------------------------------------------------------------
143  # Run configure part of buid
144  #---------------------------------------------------------------------
145  os.chdir(projectCheckOutDir)
146  configCmd = os.path.join('.','configure -C')
147  NBlogMessages.writeMessage('  '+configCmd)
148  rc=commands.getstatusoutput(configCmd)
149 
150  # Check if configure worked
151  if rc[0] != 0 :
152    error_msg = rc[1] + '\n\n'
153    # Add contents of log file to message
154    logFileName = 'config.log'
155    if os.path.isfile(logFileName) :
156      logFilePtr = open(logFileName,'r')
157      error_msg += "config.log contains: \n" 
158      error_msg += logFilePtr.read()
159      logFilePtr.close()
160    NBemail.sendCmdMsgs(p,error_msg,configCmd)
161    continue
162
163  #---------------------------------------------------------------------
164  # Run make part of buid
165  #---------------------------------------------------------------------
166  NBlogMessages.writeMessage( '  make' )
167  rc=commands.getstatusoutput('make')
168 
169  # Check if make worked
170  if rc[0] != 0 :
171    NBemail.sendCmdMsgs(p,rc[1],'make')
172    continue
173
174  #---------------------------------------------------------------------
175  # Run 'make test' part of buid
176  #---------------------------------------------------------------------
177  NBlogMessages.writeMessage( '  make test' )
178  rc=commands.getstatusoutput('make test')
179 
180  # Check if 'make test' worked
181  if didTestFail(rc,p,"make test") :
182    NBemail.sendCmdMsgs(p,rc[1],"make test")
183    continue
184
185  #---------------------------------------------------------------------
186  # Run unitTest if available and different from 'make test'
187  #---------------------------------------------------------------------
188  if NBprojectConfig.UNITTEST_CMD.has_key(p) :
189    unitTestPath = os.path.join(projectCheckOutDir,NBprojectConfig.UNITTEST_DIR[p])
190    os.chdir(unitTestPath)
191
192    unitTestCmd=NBprojectConfig.UNITTEST_CMD[p]
193    unitTestCmd=unitTestCmd.replace('_NETLIBDIR_',netlibDir)
194    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
195
196    NBlogMessages.writeMessage( '  '+unitTestCmd )
197    rc=commands.getstatusoutput(unitTestCmd)
198 
199    if didTestFail(rc,p,unitTestCmd) :
200      NBemail.sendCmdMsgs(p,rc[1],unitTestCmd)
201      continue
202
203  # For testing purposes only do first successful project
204  #break
205
206
207NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
208
209sys.exit(0)
210
211
212# START KIPP
213#----------------------------------------------------------------------
214# CONFIG FILE PATH:
215#   path to the config file for the build
216#   done. If the directory does not exist, it will be created.
217#   this should have all of the user specific data
218#   it should have values for
219#   NIGHTLY_BUILD_ROOT
220#   NBuserConfig.SMTP_SERVER_NAME
221#   NBuserConfig.SMTP_SERVER_PORT
222#   NBuserConfig.SMTP_SSL_SERVER
223#   NBuserConfig.SMTP_USER_NAME
224#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
225#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
226#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
227#   
228#----------------------------------------------------------------------
229
230CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
231CONFIG_FILENAME = 'config.txt'
232
233
234# Get configFile data
235
236configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
237                                 os.path.basename(CONFIG_FILENAME ))
238if os.path.isfile(  configFile) :
239  pwFilePtr = open(configFile ,'r')
240  d = pwFilePtr.readlines()
241  # do pwFilePtr.read() to get a string object
242  # we have a list object
243  print d[0]
244  print d[1]
245  # make a dictionary
246  config_dic = {}
247
248  #smtppass  = pwFilePtr.read().strip()
249  pwFilePtr.close()
250else :
251  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
252  #print cmdMsgs
253  sys.exit( 1)
254sys.exit( 0)
255
256
257
258# START KIPP
259#----------------------------------------------------------------------
260#   path to the config file for the build
261#   get the user dependent variables
262# CONFIG FILE PATH:
263#   it should have values for
264#   NIGHTLY_BUILD_ROOT
265#   NBuserConfig.SMTP_SERVER_NAME
266#   NBuserConfig.SMTP_SERVER_PORT
267#   NBuserConfig.SMTP_SSL_SERVER
268#   NBuserConfig.SMTP_USER_NAME
269#   NBuserConfig.SMTP_PASSWORD_FILENAME
270#   NBuserConfig.SENDER_EMAIL_ADDR
271#   NBuserConfig.MY_EMAIL_ADDR
272#   
273#----------------------------------------------------------------------
274
275CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
276CONFIG_FILENAME = 'config.txt'
277
278
279# Get configFile data
280
281configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
282                                 os.path.basename(CONFIG_FILENAME ))
283if os.path.isfile(  configFile) :
284  pwFilePtr = open(configFile ,'r')
285  d = pwFilePtr.readlines()
286  # do pwFilePtr.read() to get a string object
287  # we have a list object
288  print d[0]
289  print d[1]
290  # make a dictionary
291  config_dic = {}
292
293  #smtppass  = pwFilePtr.read().strip()
294  pwFilePtr.close()
295else :
296  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
297  #print cmdMsgs
298  sys.exit( 1)
299sys.exit( 0)
300
301# END KIPP
Note: See TracBrowser for help on using the repository browser.