source: branches/testScripts/nightlyBuild.py @ 614

Last change on this file since 614 was 614, checked in by jpfasano, 14 years ago

Started process of breaking code up into several files(modules in python speak)
Project and User variables are now defined in their own files.

File size: 13.5 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5import commands
6import smtplib
7import re 
8import time
9
10import NBuserConfig
11import NBprojectConfig
12
13# TODO:
14#   -After "svn co" then get all 3rd party packages.
15#   -Get some information about the platform and put this in email failure message.
16#   -Implement Kipp's vpath (delete vpath instead of 'make distclean').
17#   Break this file up into multiple files so it is manageable.
18#   Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   Build both trunk and latest stable
21#   Build both optimized and debug (or have a set of config-site scripts to test?)
22#   Check the testing of the success criteria of each projects "make test"
23#   Implement "cbc -miplib" test for successful run.  JohnF sent JP the criteria
24#     to test on in an email dated 10/12/2007 12:01pm
25
26
27#------------------------------------------------------------------------
28# Send email typically about an error.
29#  project: coin project name
30#  cmd: command being exucuted. perhaps: "svn update", "./configure",
31#       "make".
32#  cmdMsgs: the messages generated by cmd.  This will typically contain
33#       errors issued by cmd.
34#------------------------------------------------------------------------
35def sendEmailCmdMsgs(project,cmdMsgs,cmd):
36  curDir = os.getcwd()
37 
38  toAddrs = [unscrambleEmailAddress(NBuserConfig.MY_EMAIL_ADDR)]
39  if NBprojectConfig.PROJECT_EMAIL_ADDRS.has_key(project) and NBuserConfig.SEND_MAIL_TO_PROJECT_MANAGER:
40    toAddrs.append(unscrambleEmailAddress(NBprojectConfig.PROJECT_EMAIL_ADDRS[project]))
41
42  subject = project + " build problem when running '" + cmd +"'"
43
44  emailMsg  = "'" + cmd + "' from directory " + curDir + " failed.\n\n"
45  emailMsg += "Operating System: "+os.name
46  emailMsg += "'" + cmd + "' messages are:\n" 
47  emailMsg += cmdMsgs
48  sendEmail(toAddrs,subject,emailMsg)
49  writeLogMessage( "  email sent regarding "+project+" running '"+cmd+"'" )
50
51#------------------------------------------------------------------------
52# Send email
53#------------------------------------------------------------------------
54def sendEmail(toAddrs,subject,message):
55
56  sender = unscrambleEmailAddress(NBuserConfig.SENDER_EMAIL_ADDR) 
57  msgWHeader = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n"
58       % (sender, ", ".join(toAddrs), subject))
59  msgWHeader += message
60 
61  # Get smpt server password
62  if os.path.isfile(NBuserConfig.SMTP_PASSWORD_FILENAME) :
63    pwFilePtr = open(NBuserConfig.SMTP_PASSWORD_FILENAME,'r')
64    smtppass  = pwFilePtr.read().strip()
65    pwFilePtr.close()
66  else :
67    writeLogMessage( "Failure reading pwFileName=" + NBuserConfig.SMTP_PASSWORD_FILENAME )
68    sys.exit(1)
69   
70  session = smtplib.SMTP(NBuserConfig.SMTP_SERVER_NAME,NBuserConfig.SMTP_SERVER_PORT)
71  #session.set_debuglevel(1)
72  if NBuserConfig.SMTP_SSL_SERVER==1 :
73    session.ehlo('x')
74    session.starttls()
75    session.ehlo('x') 
76  session.login(unscrambleEmailAddress(NBuserConfig.SMTP_USER_NAME),smtppass)
77
78  rc = session.sendmail(sender,toAddrs,msgWHeader)
79  if rc!={} :
80    writeLogMessage( 'session.sendmail rc='  )
81    writeLogMessage( rc )
82  session.quit()
83
84#------------------------------------------------------------------------
85# Decrypt email address
86#------------------------------------------------------------------------
87def unscrambleEmailAddress( scrambledEmailAddress ) :
88  retVal = scrambledEmailAddress
89  retVal = retVal.replace(' _AT_ ','@')
90  retVal = retVal.replace(' _DOT_ ','.')
91  return retVal
92
93#------------------------------------------------------------------------
94# Function to Check Return Code from unitTest
95#------------------------------------------------------------------------
96def didTestFail( rc, project, buildStep ) :
97  retVal = 0
98
99  # If the return code is not 0, then failure
100  if rc[0] != 0 :
101    retVal = 1
102
103  # Many tests write a "Success" message.
104  # For test that do this, check for the success message
105  if NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS.has_key(project) : 
106    if buildStep in NBprojectConfig.ALL_TESTS_COMPLETED_SUCCESSFULLY_CMDS[project] :
107      # Is the success message contained in the output?
108      if rc[1].rfind("All tests completed successfully") == -1 :
109        # Success message not found, assume test failed
110        retVal = 1
111
112  #---------------------------------------------------------------------
113  # Some (project,buildStep) pairs require further checking
114  # to determine if they were successful
115  #---------------------------------------------------------------------
116  # Clp's "./clp -unitTest -netlib dirNetlib=_NETLIBDIR_"
117  if project=='Clp' and buildStep==NBprojectConfig.UNITTEST_CMD['Clp'] :
118    # Check that last netlib test case ran by looking for message of form
119    # '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
120    reexp = r"(.|\n)*\.\.(\\|/)\.\.(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
121    msgTail = rc[1][-200:]
122    if not re.compile(reexp).match(msgTail,1) :
123      # message not found, assume test failed
124      retVal = 1
125  elif project=='Cbc' and buildStep=='make test' :
126    # Check that last the last few lines are of the form
127    # 'cbc_clp solved 2 out of 2 and took XX.XX seconds.'
128    reexp=r"(.|\n)*cbc_clp solved 2 out of 2 and took (\d*\.\d*) seconds."
129    msgTail = rc[1][-300:]
130    if not re.compile(reexp).match(msgTail,1) :
131      # message not found, assume test failed
132      retVal = 1
133
134  return retVal
135
136#------------------------------------------------------------------------
137# Function for executing svn commands
138#------------------------------------------------------------------------
139def issueSvnCmd(svnCmd,dir,project) :
140  retVal='OK'
141  os.chdir(dir)
142  writeLogMessage('  '+svnCmd)
143  rc=commands.getstatusoutput(svnCmd)
144  if rc[0] != 0 :
145    sendEmailCmdMsgs(project,rc[1],svnCmd)
146    retVal='Error'
147  return retVal
148
149#------------------------------------------------------------------------
150# Function to write log messages
151#------------------------------------------------------------------------
152def writeLogMessage( msg ) :
153  logMsg = time.ctime(time.time())+': '
154  logMsg += msg
155  print logMsg
156
157#------------------------------------------------------------------------
158#  Main Program Starts Here 
159#------------------------------------------------------------------------
160
161#------------------------------------------------------------------------
162#  If needed create the top level directory
163#------------------------------------------------------------------------
164# rc=commands.getstatusoutput(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
165if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
166  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
167os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
168
169#------------------------------------------------------------------------
170#  Get the data directories if they don't already exist
171#------------------------------------------------------------------------
172dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
173if not os.path.isdir(dataBaseDir) :
174  os.makedirs(dataBaseDir)
175dataDirs=['Netlib','miplib3']
176for d in dataDirs :
177  dataDir=os.path.join(dataBaseDir,d)
178  if not os.path.isdir(dataDir) :
179    svnCmd='svn checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
180    if issueSvnCmd(svnCmd,dataBaseDir,'Data')!='OK' :
181      sys.exit(1)
182    rc=commands.getstatusoutput('find '+d+' -name \*.gz -print | xargs gzip -d')
183netlibDir=os.path.join(dataBaseDir,'Netlib')
184miplib3Dir=os.path.join(dataBaseDir,'miplib3')
185
186#------------------------------------------------------------------------
187# Loop once for each project
188#------------------------------------------------------------------------
189for p in NBuserConfig.PROJECTS:
190  writeLogMessage( p )
191  rc = [0]
192
193  #---------------------------------------------------------------------
194  # svn checkout or update the project
195  #---------------------------------------------------------------------
196  projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
197  projectCheckOutDir=os.path.join(projectBaseDir,'trunk')
198  if not os.path.isdir(projectBaseDir) :
199    os.makedirs(projectBaseDir)
200  if not os.path.isdir(projectCheckOutDir) :
201    svnCmd='svn checkout https://projects.coin-or.org/svn/'+p+'/trunk trunk'
202    if issueSvnCmd(svnCmd,projectBaseDir,p)!='OK' :
203      continue
204  else :
205    svnCmd='svn update'
206    if issueSvnCmd(svnCmd,projectCheckOutDir,p)!='OK' :
207      continue
208
209  #---------------------------------------------------------------------
210  # Should probably run make 'distclean' to do a build from scrath
211  # or delete the VPATH directory when there is one
212  #---------------------------------------------------------------------
213
214
215  #---------------------------------------------------------------------
216  # Run configure part of buid
217  #---------------------------------------------------------------------
218  os.chdir(projectCheckOutDir)
219  configCmd = os.path.join('.','configure -C')
220  writeLogMessage('  '+configCmd)
221  rc=commands.getstatusoutput(configCmd)
222 
223  # Check if configure worked
224  if rc[0] != 0 :
225    error_msg = rc[1] + '\n\n'
226    # Add contents of log file to message
227    logFileName = 'config.log'
228    if os.path.isfile(logFileName) :
229      logFilePtr = open(logFileName,'r')
230      error_msg += "config.log contains: \n" 
231      error_msg += logFilePtr.read()
232      logFilePtr.close()
233    sendEmailCmdMsgs(p,error_msg,configCmd)
234    continue
235
236  #---------------------------------------------------------------------
237  # Run make part of buid
238  #---------------------------------------------------------------------
239  writeLogMessage( '  make' )
240  rc=commands.getstatusoutput('make')
241 
242  # Check if make worked
243  if rc[0] != 0 :
244    sendEmailCmdMsgs(p,rc[1],'make')
245    continue
246
247  #---------------------------------------------------------------------
248  # Run 'make test' part of buid
249  #---------------------------------------------------------------------
250  writeLogMessage( '  make test' )
251  rc=commands.getstatusoutput('make test')
252 
253  # Check if 'make test' worked
254  if didTestFail(rc,p,"make test") :
255    sendEmailCmdMsgs(p,rc[1],"make test")
256    continue
257
258  #---------------------------------------------------------------------
259  # Run unitTest if available and different from 'make test'
260  #---------------------------------------------------------------------
261  if NBprojectConfig.UNITTEST_CMD.has_key(p) :
262    unitTestPath = os.path.join(projectCheckOutDir,NBprojectConfig.UNITTEST_DIR[p])
263    os.chdir(unitTestPath)
264
265    unitTestCmd=NBprojectConfig.UNITTEST_CMD[p]
266    unitTestCmd=unitTestCmd.replace('_NETLIBDIR_',netlibDir)
267    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
268
269    writeLogMessage( '  '+unitTestCmd )
270    rc=commands.getstatusoutput(unitTestCmd)
271 
272    if didTestFail(rc,p,unitTestCmd) :
273      sendEmailCmdMsgs(p,rc[1],unitTestCmd)
274      continue
275
276  # For testing purposes only do first successful project
277  #break
278
279
280writeLogMessage( "nightlyBuild.py Finished" )
281
282sys.exit(0)
283
284
285# START KIPP
286#----------------------------------------------------------------------
287# CONFIG FILE PATH:
288#   path to the config file for the build
289#   done. If the directory does not exist, it will be created.
290#   this should have all of the user specific data
291#   it should have values for
292#   NIGHTLY_BUILD_ROOT
293#   NBuserConfig.SMTP_SERVER_NAME
294#   NBuserConfig.SMTP_SERVER_PORT
295#   NBuserConfig.SMTP_SSL_SERVER
296#   NBuserConfig.SMTP_USER_NAME
297#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
298#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
299#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
300#   
301#----------------------------------------------------------------------
302
303CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
304CONFIG_FILENAME = 'config.txt'
305
306
307# Get configFile data
308
309configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
310                                 os.path.basename(CONFIG_FILENAME ))
311if os.path.isfile(  configFile) :
312  pwFilePtr = open(configFile ,'r')
313  d = pwFilePtr.readlines()
314  # do pwFilePtr.read() to get a string object
315  # we have a list object
316  print d[0]
317  print d[1]
318  # make a dictionary
319  config_dic = {}
320
321  #smtppass  = pwFilePtr.read().strip()
322  pwFilePtr.close()
323else :
324  #writeLogMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
325  #print cmdMsgs
326  sys.exit( 1)
327sys.exit( 0)
328
329
330
331# START KIPP
332#----------------------------------------------------------------------
333#   path to the config file for the build
334#   get the user dependent variables
335# CONFIG FILE PATH:
336#   it should have values for
337#   NIGHTLY_BUILD_ROOT
338#   NBuserConfig.SMTP_SERVER_NAME
339#   NBuserConfig.SMTP_SERVER_PORT
340#   NBuserConfig.SMTP_SSL_SERVER
341#   NBuserConfig.SMTP_USER_NAME
342#   NBuserConfig.SMTP_PASSWORD_FILENAME
343#   NBuserConfig.SENDER_EMAIL_ADDR
344#   NBuserConfig.MY_EMAIL_ADDR
345#   
346#----------------------------------------------------------------------
347
348CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
349CONFIG_FILENAME = 'config.txt'
350
351
352# Get configFile data
353
354configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
355                                 os.path.basename(CONFIG_FILENAME ))
356if os.path.isfile(  configFile) :
357  pwFilePtr = open(configFile ,'r')
358  d = pwFilePtr.readlines()
359  # do pwFilePtr.read() to get a string object
360  # we have a list object
361  print d[0]
362  print d[1]
363  # make a dictionary
364  config_dic = {}
365
366  #smtppass  = pwFilePtr.read().strip()
367  pwFilePtr.close()
368else :
369  #writeLogMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
370  #print cmdMsgs
371  sys.exit( 1)
372sys.exit( 0)
373
374# END KIPP
Note: See TracBrowser for help on using the repository browser.