source: branches/testScripts/nightlyBuild.py @ 610

Last change on this file since 610 was 610, checked in by kmartin, 14 years ago

edit documentation relating to config file

File size: 17.2 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5import commands
6import smtplib
7import re 
8import time
9
10# TODO:
11#   -After "svn co" then get all 3rd party packages.
12#   -Get some information about the platform and put this in email failure message.
13#   -Implement Kipp's vpath (delete vpath instead of 'make distclean').
14#   Break this file up into multiple files so it is manageable.
15#   Don't do build if 'svn update' doesn't change anything and prior test was OK.
16#     (no need to re-run if nothing has changed since prior run)
17#   Build both trunk and latest stable
18#   Build both optimized and debug (or have a set of config-site scripts to test?)
19#   Check the testing of the success criteria of each projects "make test"
20#   Implement "cbc -miplib" test for successful run.  JohnF sent JP the criteria
21#     to test on in an email dated 10/12/2007 12:01pm
22
23#----------------------------------------------------------------------
24# NIGHTLY_BUILD_ROOT_DIR:
25#   directory where code will be checked out and builds
26#   done. If the directory does not exist, it will be created.
27#----------------------------------------------------------------------
28NIGHTLY_BUILD_ROOT_DIR = '/home/jp/COIN'
29
30#----------------------------------------------------------------------
31# Values for sending mail:
32#  SMTP_SERVER_NAME: name of smtp server. For gmail server
33#                 this is smtp.gmail.com
34#  SMTP_SERVER_PORT: port number of the smtp server. This is typically 25,
35#                 but for gmail server it is 587.
36#  SMTP_SSL_SERVER: 0 or 1. If 1 then SMTP uses SSL (sometimes called startltls).
37#                 For gmail this is 1.
38#  SMTP_USER_NAME: name of authorized user on server. If using gmail server
39#                 this is gmail_userid@gmail.com which is coded as
40#                 'gmail_userid _AT_ gmail _DOT_ com. 
41#  SMTP_PASSWORD_FILENAME: name of file containing smtp user's password
42#  SENDER_EMAIL_ADDR: email sent by this script will be from this address
43#  MY_EMAIL_ADDR: All problems detected by the script will be sent to
44#                 this email address. The intention is for this to be
45#                 the email address of the person running this script
46#  SEND_MAIL_TO_PROJECT_MANAGER: 0 or 1. If 1 then any problems
47#                 detected are sent to MY_EMAIL_ADDRESS and the
48#                 project manager.
49#----------------------------------------------------------------------
50#SMTP_SERVER_NAME = 'outgoing.verizon.net'
51#SMTP_SERVER_PORT = 25
52#SMTP_SSL_SERVER = 0
53#SMTP_USER_NAME = 'jpfasano'
54
55SMTP_SERVER_NAME = 'smtp.gmail.com'
56SMTP_SERVER_PORT = 587
57SMTP_SSL_SERVER = 1
58SMTP_USER_NAME = 'jpfasano _AT_ gmail _DOT_ com'
59SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
60
61SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
62MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
63SEND_MAIL_TO_PROJECT_MANAGER=0
64#SMTP_SERVER_NAME = 'gsbims.chicagogsb.edu'
65
66
67#----------------------------------------------------------------------
68# List of Projects to be processed by script
69#----------------------------------------------------------------------
70PROJECTS = ['CoinUtils','DyLP','Clp','SYMPHONY','Vol','Osi','Cgl','Cbc','Ipopt','OS','CppAD']
71#PROJECTS = ['CppAD']
72
73#----------------------------------------------------------------------
74PROJECT_EMAIL_ADDRS = {}
75UNITTEST_DIR = {}
76UNITTEST_CMD = {}
77ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS = {} 
78
79#----------------------------------------------------------------------
80PROJECT_EMAIL_ADDRS['CoinUtils'] = 'ladanyi _AT_ us _DOT_ ibm _DOT_ com'
81ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['CoinUtils'] = ['make test']
82
83#----------------------------------------------------------------------
84PROJECT_EMAIL_ADDRS['DyLP'] = 'lou _AT_ cs _DOT_ sfu _DOT_ ca'
85UNITTEST_DIR['DyLP'] = os.path.join('Osi','test')
86UNITTEST_CMD['DyLP'] = './unitTest -testOsiSolverInterface -netlibDir=_NETLIBDIR_ -cerr2cout' 
87ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['DyLP'] = ['make test']
88
89#----------------------------------------------------------------------
90PROJECT_EMAIL_ADDRS['Clp'] = 'jjforre _AT_ us _DOT_ ibm _DOT_ com'
91UNITTEST_DIR['Clp'] = os.path.join('Clp','src')
92UNITTEST_CMD['Clp'] = './clp -unitTest -netlib dirNetlib=_NETLIBDIR_' 
93ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['Clp'] = ['make test',UNITTEST_CMD['Clp']]
94
95#----------------------------------------------------------------------
96PROJECT_EMAIL_ADDRS['SYMPHONY'] = 'tkr2 _AT_ lehigh _DOT_ edu'
97ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['SYMPHONY'] = ['make test']
98
99#----------------------------------------------------------------------
100PROJECT_EMAIL_ADDRS['Vol'] = 'barahon _AT_ us _DOT_ ibm _DOT_ com'
101
102#----------------------------------------------------------------------
103PROJECT_EMAIL_ADDRS['Osi'] = 'mjs _AT_ ces _DOT_ clemson _DOT_ edu'
104UNITTEST_DIR['Osi'] = os.path.join('Osi','test')
105UNITTEST_CMD['Osi'] = './unitTest -testOsiSolverInterface' 
106UNITTEST_CMD['Osi'] = './unitTest -testOsiSolverInterface -netlibDir=_NETLIBDIR_ -cerr2cout' 
107ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['Osi'] = ['make test',UNITTEST_CMD['Osi']]
108
109#----------------------------------------------------------------------
110PROJECT_EMAIL_ADDRS['Cgl'] = 'robinlh _AT_ us _DOT_ ibm _DOT_ com'
111ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['Cgl'] = ['make test']
112
113#----------------------------------------------------------------------
114PROJECT_EMAIL_ADDRS['Cbc'] = 'jjforre _AT_ us _DOT_ ibm _DOT_ com'
115ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS['Cbc'] = ['make test']
116
117#----------------------------------------------------------------------
118PROJECT_EMAIL_ADDRS['Ipopt'] = 'andreasw _AT_ us _DOT_ ibm _DOT_ com'
119
120#----------------------------------------------------------------------
121PROJECT_EMAIL_ADDRS['OS'] = 'kipp _DOT_ martin _AT_ chicagogsb _DOT_ edu'
122
123#----------------------------------------------------------------------
124PROJECT_EMAIL_ADDRS['CppAD'] = 'bradbell _AT_ washington _DOT_ edu'
125
126#------------------------------------------------------------------------
127# Send email typically about an error.
128#  project: coin project name
129#  cmd: command being exucuted. perhaps: "svn update", "./configure",
130#       "make".
131#  cmdMsgs: the messages generated by cmd.  This will typically contain
132#       errors issued by cmd.
133#------------------------------------------------------------------------
134def sendEmailCmdMsgs(project,cmdMsgs,cmd):
135  curDir = os.getcwd()
136 
137  toAddrs = [unscrambleEmailAddress(MY_EMAIL_ADDR)]
138  if PROJECT_EMAIL_ADDRS.has_key(project) and SEND_MAIL_TO_PROJECT_MANAGER:
139    toAddrs.append(unscrambleEmailAddress(PROJECT_EMAIL_ADDRS[project]))
140
141  subject = project + " build problem when running '" + cmd +"'"
142
143  emailMsg  = "'" + cmd + "' from directory " + curDir + " failed.\n\n"
144  emailMsg += "'" + cmd + "' messages are:\n" 
145  emailMsg += cmdMsgs
146  sendEmail(toAddrs,subject,emailMsg)
147  writeLogMessage( "  email sent regarding "+project+" running '"+cmd+"'" )
148
149#------------------------------------------------------------------------
150# Send email
151#------------------------------------------------------------------------
152def sendEmail(toAddrs,subject,message):
153
154  sender = unscrambleEmailAddress(SENDER_EMAIL_ADDR) 
155  msgWHeader = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n"
156       % (sender, ", ".join(toAddrs), subject))
157  msgWHeader += message
158 
159  # Get smpt server password
160  if os.path.isfile(SMTP_PASSWORD_FILENAME) :
161    pwFilePtr = open(SMTP_PASSWORD_FILENAME,'r')
162    smtppass  = pwFilePtr.read().strip()
163    pwFilePtr.close()
164  else :
165    writeLogMessage( "Failure reading pwFileName=" + SMTP_PASSWORD_FILENAME )
166    print cmdMsgs
167    sys.exit(1)
168   
169  session = smtplib.SMTP(SMTP_SERVER_NAME,SMTP_SERVER_PORT)
170  #session.set_debuglevel(1)
171  if SMTP_SSL_SERVER==1 :
172    session.ehlo('x')
173    session.starttls()
174    session.ehlo('x') 
175  session.login(unscrambleEmailAddress(SMTP_USER_NAME),smtppass)
176
177  rc = session.sendmail(sender,toAddrs,msgWHeader)
178  if rc!={} :
179    writeLogMessage( 'session.sendmail rc='  )
180    writeLogMessage( rc )
181  session.quit()
182
183#------------------------------------------------------------------------
184# Decrypt email address
185#------------------------------------------------------------------------
186def unscrambleEmailAddress( scrambledEmailAddress ) :
187  retVal = scrambledEmailAddress
188  retVal = retVal.replace(' _AT_ ','@')
189  retVal = retVal.replace(' _DOT_ ','.')
190  return retVal
191
192#------------------------------------------------------------------------
193# Function to Check Return Code from unitTest
194#------------------------------------------------------------------------
195def didTestFail( rc, project, buildStep ) :
196  retVal = 0
197
198  # If the return code is not 0, then failure
199  if rc[0] != 0 :
200    retVal = 1
201
202  # Many tests write a "Success" message.
203  # For test that do this, check for the success message
204  if ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS.has_key(project) : 
205    if buildStep in ALL_TEST_COMPLETED_SUCCESSFULLY_CMDS[project] :
206      # Is the success message contained in the output?
207      if rc[1].rfind("All tests completed successfully") == -1 :
208        # Success message not found, assume test failed
209        retVal = 1
210
211  # Some (project,buildStep) pairs require further checking
212  if project=='Clp' and buildStep==UNITTEST_CMD['Clp'] :
213    # Build Step is './clp -unitTest -netlib'
214    # Check that last netlib test case ran by looking for message of form
215    # '../../Data/Netlib/woodw took 0.47 seconds using algorithm either'
216    reexp = r"(.|\n)*\.\.(\\|/)\.\.(\\|/)Data(\\|/)Netlib(\\|/)woodw took (\d*\.\d*) seconds using algorithm either(.|\n)*"
217    msgTail = rc[1][len(rc[1])-200:]
218    if not re.compile(reexp).match(msgTail,1) :
219      # message not found, assume test failed
220      retVal = 1
221
222  return retVal
223
224#------------------------------------------------------------------------
225# Function for executing svn commands
226#------------------------------------------------------------------------
227def issueSvnCmd(svnCmd,dir,project) :
228  retVal='OK'
229  os.chdir(dir)
230  writeLogMessage('  '+svnCmd)
231  rc=commands.getstatusoutput(svnCmd)
232  if rc[0] != 0 :
233    sendEmailCmdMsgs(project,rc[1],svnCmd)
234    retVal='Error'
235  return retVal
236
237#------------------------------------------------------------------------
238# Function to write log messages
239#------------------------------------------------------------------------
240def writeLogMessage( msg ) :
241  logMsg = time.ctime(time.time())+': '
242  logMsg += msg
243  print logMsg
244
245#------------------------------------------------------------------------
246#  Main Program Starts Here 
247#------------------------------------------------------------------------
248
249#------------------------------------------------------------------------
250#  If needed create the top level directory
251#------------------------------------------------------------------------
252# rc=commands.getstatusoutput(NIGHTLY_BUILD_ROOT_DIR)
253if not os.path.isdir(NIGHTLY_BUILD_ROOT_DIR) :
254  os.makedirs(NIGHTLY_BUILD_ROOT_DIR)
255os.chdir(NIGHTLY_BUILD_ROOT_DIR)
256
257#------------------------------------------------------------------------
258#  Get the data directories if they don't already exist
259#------------------------------------------------------------------------
260dataBaseDir=os.path.join(NIGHTLY_BUILD_ROOT_DIR,'Data')
261if not os.path.isdir(dataBaseDir) :
262  os.makedirs(dataBaseDir)
263dataDirs=['Netlib','miplib3']
264for d in dataDirs :
265  dataDir=os.path.join(dataBaseDir,d)
266  if not os.path.isdir(dataDir) :
267    svnCmd='svn checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
268    if issueSvnCmd(svnCmd,dataBaseDir,'Data')!='OK' :
269      sys.exit(1)
270    rc=commands.getstatusoutput('find '+d+' -name \*.gz -print | xargs gzip -d')
271netlibDir=os.path.join(dataBaseDir,'Netlib')
272miplib3Dir=os.path.join(dataBaseDir,'miplib3')
273
274#------------------------------------------------------------------------
275# Loop once for each project
276#------------------------------------------------------------------------
277for p in PROJECTS:
278  writeLogMessage( p )
279  rc = [0]
280
281  #---------------------------------------------------------------------
282  # svn checkout or update the project
283  #---------------------------------------------------------------------
284  projectBaseDir=os.path.join(NIGHTLY_BUILD_ROOT_DIR,p)
285  projectCheckOutDir=os.path.join(projectBaseDir,'trunk')
286  if not os.path.isdir(projectBaseDir) :
287    os.makedirs(projectBaseDir)
288  if not os.path.isdir(projectCheckOutDir) :
289    svnCmd='svn checkout https://projects.coin-or.org/svn/'+p+'/trunk trunk'
290    if issueSvnCmd(svnCmd,projectBaseDir,p)!='OK' :
291      continue
292  else :
293    svnCmd='svn update'
294    if issueSvnCmd(svnCmd,projectCheckOutDir,p)!='OK' :
295      continue
296
297  #---------------------------------------------------------------------
298  # Should probably run make 'distclean' to do a build from scrath
299  # or delete the VPATH directory when there is one
300  #---------------------------------------------------------------------
301
302
303  #---------------------------------------------------------------------
304  # Run configure part of buid
305  #---------------------------------------------------------------------
306  os.chdir(projectCheckOutDir)
307  configCmd = os.path.join('.','configure -C')
308  writeLogMessage('  '+configCmd)
309  rc=commands.getstatusoutput(configCmd)
310 
311  # Check if configure worked
312  if rc[0] != 0 :
313    error_msg = rc[1] + '\n\n'
314    # Add contents of log file to message
315    logFileName = 'config.log'
316    if os.path.isfile(logFileName) :
317      logFilePtr = open(logFileName,'r')
318      error_msg += "config.log contains: \n" 
319      error_msg += logFilePtr.read()
320      logFilePtr.close()
321    sendEmailCmdMsgs(p,error_msg,configCmd)
322    continue
323
324  #---------------------------------------------------------------------
325  # Run make part of buid
326  #---------------------------------------------------------------------
327  writeLogMessage( '  make' )
328  rc=commands.getstatusoutput('make')
329 
330  # Check if make worked
331  if rc[0] != 0 :
332    sendEmailCmdMsgs(p,rc[1],'make')
333    continue
334
335  #---------------------------------------------------------------------
336  # Run 'make test' part of buid
337  #---------------------------------------------------------------------
338  writeLogMessage( '  make test' )
339  rc=commands.getstatusoutput('make test')
340 
341  # Check if 'make test' worked
342  if didTestFail(rc,p,"make test") :
343    sendEmailCmdMsgs(p,rc[1],"make test")
344    continue
345
346  #---------------------------------------------------------------------
347  # Run unitTest if available and different from 'make test'
348  #---------------------------------------------------------------------
349  if UNITTEST_CMD.has_key(p) :
350    unitTestPath = os.path.join(projectCheckOutDir,UNITTEST_DIR[p])
351    os.chdir(unitTestPath)
352
353    unitTestCmd=UNITTEST_CMD[p]
354    unitTestCmd=unitTestCmd.replace('_NETLIBDIR_',netlibDir)
355    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
356
357    writeLogMessage( '  '+unitTestCmd )
358    rc=commands.getstatusoutput(unitTestCmd)
359 
360    if didTestFail(rc,p,unitTestCmd) :
361      sendEmailCmdMsgs(p,rc[1],unitTestCmd)
362      continue
363
364  # For testing purposes only do first successful project
365  #break
366
367
368writeLogMessage( "nightlyBuild.py Finished" )
369
370sys.exit(0)
371
372
373# START KIPP
374#----------------------------------------------------------------------
375# CONFIG FILE PATH:
376#   path to the config file for the build
377#   done. If the directory does not exist, it will be created.
378#   this should have all of the user specific data
379#   it should have values for
380#   NIGHTLY_BUILD_ROOT
381#   SMTP_SERVER_NAME
382#   SMTP_SERVER_PORT
383#   SMTP_SSL_SERVER
384#   SMTP_USER_NAME
385#   SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
386#   SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
387#   MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
388#   
389#----------------------------------------------------------------------
390
391CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
392CONFIG_FILENAME = 'config.txt'
393
394
395# Get configFile data
396
397configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
398                                 os.path.basename(CONFIG_FILENAME ))
399if os.path.isfile(  configFile) :
400  pwFilePtr = open(configFile ,'r')
401  d = pwFilePtr.readlines()
402  # do pwFilePtr.read() to get a string object
403  # we have a list object
404  print d[0]
405  print d[1]
406  # make a dictionary
407  config_dic = {}
408
409  #smtppass  = pwFilePtr.read().strip()
410  pwFilePtr.close()
411else :
412  #writeLogMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
413  #print cmdMsgs
414  sys.exit( 1)
415sys.exit( 0)
416
417
418
419# START KIPP
420#----------------------------------------------------------------------
421#   path to the config file for the build
422#   get the user dependent variables
423# CONFIG FILE PATH:
424#   it should have values for
425#   NIGHTLY_BUILD_ROOT
426#   SMTP_SERVER_NAME
427#   SMTP_SERVER_PORT
428#   SMTP_SSL_SERVER
429#   SMTP_USER_NAME
430#   SMTP_PASSWORD_FILENAME
431#   SENDER_EMAIL_ADDR
432#   MY_EMAIL_ADDR
433#   
434#----------------------------------------------------------------------
435
436CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
437CONFIG_FILENAME = 'config.txt'
438
439
440# Get configFile data
441
442configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
443                                 os.path.basename(CONFIG_FILENAME ))
444if os.path.isfile(  configFile) :
445  pwFilePtr = open(configFile ,'r')
446  d = pwFilePtr.readlines()
447  # do pwFilePtr.read() to get a string object
448  # we have a list object
449  print d[0]
450  print d[1]
451  # make a dictionary
452  config_dic = {}
453
454  #smtppass  = pwFilePtr.read().strip()
455  pwFilePtr.close()
456else :
457  #writeLogMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
458  #print cmdMsgs
459  sys.exit( 1)
460sys.exit( 0)
461
462# END KIPP
Note: See TracBrowser for help on using the repository browser.