source: branches/testScripts/nightlyBuild.py @ 669

Last change on this file since 669 was 663, checked in by stefan, 12 years ago

use stat to check file dates instead of ls --fulltime and regex

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 12.8 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Skip make of project that depend on a project which make had been failing. (Similar for configure.)
21#    But also tell project managers of skiped project that their project was skipped.
22#   -Store output of get.XXX calls, make, make test, configures... in some files.
23#   -Allow a fine configuration of which build tests to run per project.
24#   -Move actual [get project, configure, make, tests] part into an extra file.
25
26
27#------------------------------------------------------------------------
28#  Main Program Starts Here 
29#------------------------------------------------------------------------
30
31#------------------------------------------------------------------------
32#  If needed create the top level directory
33#------------------------------------------------------------------------
34if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
35  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
36os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
37
38#------------------------------------------------------------------------
39#  Get the data directories if they don't already exist
40#------------------------------------------------------------------------
41dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
42if not os.path.isdir(dataBaseDir) :
43  os.makedirs(dataBaseDir)
44dataDirs=['Netlib','miplib3']
45for d in dataDirs :
46  dataDir=os.path.join(dataBaseDir,d)
47  if not os.path.isdir(dataDir) :
48    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
49    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
50      sys.exit(1)
51    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
52netlibDir=os.path.join(dataBaseDir,'Netlib')
53miplib3Dir=os.path.join(dataBaseDir,'miplib3')
54
55#------------------------------------------------------------------------
56# Loop once for each project
57#------------------------------------------------------------------------
58for p in NBuserConfig.PROJECTS:
59  NBlogMessages.writeMessage( p )
60
61  #---------------------------------------------------------------------
62  # Loop once for each version of the project to be checked out.
63  # The supported types are trunk & latestStable
64  #---------------------------------------------------------------------
65  projectVersions=[]
66  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
67    lsv = NBsvnCommand.latestStableVersion(p)
68    projectVersions.append(['stable'+lsv,'stable/'+lsv])
69  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
70    projectVersions.append(['trunk','trunk'])
71  for projectVersion in projectVersions :
72    #---------------------------------------------------------------------
73    # svn checkout or update the project
74    #---------------------------------------------------------------------
75    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
76    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
77    if not os.path.isdir(projectBaseDir) :
78      os.makedirs(projectBaseDir)
79    if not os.path.isdir(projectCheckOutDir) :
80      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
81             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
82      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
83        continue
84    else :
85      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
86      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
87        continue
88
89    #---------------------------------------------------------------------
90    # If there are third party apps, then get these apps
91    #---------------------------------------------------------------------
92    if NBuserConfig.DOWNLOAD_3RD_PARTY :
93      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
94      if os.path.isdir(thirdPartyBaseDir) :
95        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
96        for d in thirdPartyDirs :
97          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
98          install3rdPartyCmd=os.path.join(".","get."+d)
99          os.chdir(thirdPartyDir)
100          # If the install command has been updated since the last
101          # install, then do a new install
102          if os.path.isfile('NBinstalldone') :
103            if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
104              os.remove('NBinstalldone')
105          if not os.path.isfile('NBinstalldone') :
106            if os.path.isfile(install3rdPartyCmd) :
107              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
108              NBosCommand.run(install3rdPartyCmd)
109              f=open('NBinstalldone','w')
110              f.close()
111          else :
112            NBlogMessages.writeMessage('  skipped anew download of '+d)
113   
114    #---------------------------------------------------------------------
115    # Loop once for each type of build to be done.
116    # Debug, use third party code, ...
117    # vpath and options to configure must be set for the buildType
118    #---------------------------------------------------------------------
119    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
120
121    print buildtypes
122   
123    for buildType, configLine in buildtypes.iteritems() :
124   
125      #---------------------------------------------------------------------
126      # Setup the directory where the build will be done and the configure
127      # command options
128      #---------------------------------------------------------------------
129      vpathDir=projectVersion[0]+'-'+buildType
130      configOptions='-C '+NBuserConfig.CONFIGURE_FLAGS+' '+configLine
131
132#      print 'HERE ARE THE CONFIG OPTIONS SO FAR'
133#      print configOptions
134
135
136#      if "ThirdParty" in buildType :
137#        vpathDir += "ThirdParty"
138#      else :
139#        vpathDir += "NoThirdParty"
140#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
141#        if os.path.isdir(thirdPartyBaseDir) :
142#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
143#          skipOptions=''
144#          for d in thirdPartyDirs :
145#            skipOptions+=' ThirdParty/'+d
146#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
147
148# Added by Kipp -- Sunday, Oct 21
149#      print buildType
150#      print configLine
151      if "NoThirdParty" in buildType :
152        skipOptions=''
153        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
154        if os.path.isdir(thirdPartyBaseDir) :
155          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
156          for d in thirdPartyDirs :
157            skipOptions+=' ThirdParty/'+d
158        configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
159# End Kipp
160
161
162      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
163      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
164      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
165      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
166
167      #---------------------------------------------------------------------
168      # Run configure part of build (only if config has not previously
169      # ran successfully).
170      #---------------------------------------------------------------------
171      os.chdir(fullVpathDir)
172      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
173      if NBcheckResult.didConfigRunOK() :
174        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
175      else :
176        NBlogMessages.writeMessage('  '+configCmd)
177        result=NBosCommand.run(configCmd)
178     
179        # Check if configure worked
180        if result['returnCode'] != 0 :
181          error_msg = result
182          # Add contents of log file to message
183          logFileName = 'config.log'
184          if os.path.isfile(logFileName) :
185            logFilePtr = open(logFileName,'r')
186            error_msg['config.log']  = "config.log contains: \n" 
187            error_msg['config.log'] += logFilePtr.read()
188            logFilePtr.close()
189          NBemail.sendCmdMsgs(p,error_msg,configCmd)
190          continue
191
192      #---------------------------------------------------------------------
193      # Run make part of build
194      #---------------------------------------------------------------------
195      NBlogMessages.writeMessage( '  make' )
196      result=NBosCommand.run('make')
197     
198      # Check if make worked
199      if result['returnCode'] != 0 :
200        NBemail.sendCmdMsgs(p,result,'make')
201        continue
202
203      #---------------------------------------------------------------------
204      # Run 'make test' part of build
205      #---------------------------------------------------------------------
206      NBlogMessages.writeMessage( '  make test' )
207      result=NBosCommand.run('make test')
208     
209      # Check if 'make test' worked
210      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
211      if didMakeTestFail :
212        result['make test']=didMakeTestFail
213        NBemail.sendCmdMsgs(p,result,"make test")
214        continue
215
216      #---------------------------------------------------------------------
217      # Run unitTest if available and different from 'make test'
218      #---------------------------------------------------------------------
219      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
220        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
221        os.chdir(unitTestPath)
222
223        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
224        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
225        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
226
227        NBlogMessages.writeMessage( '  '+unitTestCmd )
228        result=NBosCommand.run(unitTestCmd)
229     
230        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
231        if didUnitTestFail :
232          result['unitTest']=didUnitTestFail
233          NBemail.sendCmdMsgs(p,result,unitTestCmd)
234          continue
235
236NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
237
238sys.exit(0)
239
240
241# START KIPP
242#----------------------------------------------------------------------
243# CONFIG FILE PATH:
244#   path to the config file for the build
245#   done. If the directory does not exist, it will be created.
246#   this should have all of the user specific data
247#   it should have values for
248#   NIGHTLY_BUILD_ROOT
249#   NBuserConfig.SMTP_SERVER_NAME
250#   NBuserConfig.SMTP_SERVER_PORT
251#   NBuserConfig.SMTP_SSL_SERVER
252#   NBuserConfig.SMTP_USER_NAME
253#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
254#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
255#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
256#   
257#----------------------------------------------------------------------
258
259CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
260CONFIG_FILENAME = 'config.txt'
261
262
263# Get configFile data
264
265configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
266                                 os.path.basename(CONFIG_FILENAME ))
267if os.path.isfile(  configFile) :
268  pwFilePtr = open(configFile ,'r')
269  d = pwFilePtr.readlines()
270  # do pwFilePtr.read() to get a string object
271  # we have a list object
272  print d[0]
273  print d[1]
274  # make a dictionary
275  config_dic = {}
276
277  #smtppass  = pwFilePtr.read().strip()
278  pwFilePtr.close()
279else :
280  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
281  #print cmdMsgs
282  sys.exit( 1)
283sys.exit( 0)
284
285
286
287# START KIPP
288#----------------------------------------------------------------------
289#   path to the config file for the build
290#   get the user dependent variables
291# CONFIG FILE PATH:
292#   it should have values for
293#   NIGHTLY_BUILD_ROOT
294#   NBuserConfig.SMTP_SERVER_NAME
295#   NBuserConfig.SMTP_SERVER_PORT
296#   NBuserConfig.SMTP_SSL_SERVER
297#   NBuserConfig.SMTP_USER_NAME
298#   NBuserConfig.SMTP_PASSWORD_FILENAME
299#   NBuserConfig.SENDER_EMAIL_ADDR
300#   NBuserConfig.MY_EMAIL_ADDR
301#   
302#----------------------------------------------------------------------
303
304CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
305CONFIG_FILENAME = 'config.txt'
306
307
308# Get configFile data
309
310configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
311                                 os.path.basename(CONFIG_FILENAME ))
312if os.path.isfile(  configFile) :
313  pwFilePtr = open(configFile ,'r')
314  d = pwFilePtr.readlines()
315  # do pwFilePtr.read() to get a string object
316  # we have a list object
317  print d[0]
318  print d[1]
319  # make a dictionary
320  config_dic = {}
321
322  #smtppass  = pwFilePtr.read().strip()
323  pwFilePtr.close()
324else :
325  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
326  #print cmdMsgs
327  sys.exit( 1)
328sys.exit( 0)
329
330# END KIPP
Note: See TracBrowser for help on using the repository browser.