source: branches/testScripts/nightlyBuild.py @ 671

Last change on this file since 671 was 671, checked in by kmartin, 12 years ago

put in temporary kludge to skip Ipopt if ThirdParty? is skipped

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 12.8 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Skip make of project that depend on a project which make had been failing. (Similar for configure.)
21#    But also tell project managers of skiped project that their project was skipped.
22#   -Store output of get.XXX calls, make, make test, configures... in some files.
23#   -Allow a fine configuration of which build tests to run per project.
24#   -Move actual [get project, configure, make, tests] part into an extra file.
25
26
27#------------------------------------------------------------------------
28#  Main Program Starts Here 
29#------------------------------------------------------------------------
30
31#------------------------------------------------------------------------
32#  If needed create the top level directory
33#------------------------------------------------------------------------
34if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
35  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
36os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
37
38#------------------------------------------------------------------------
39#  Get the data directories if they don't already exist
40#------------------------------------------------------------------------
41dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
42if not os.path.isdir(dataBaseDir) :
43  os.makedirs(dataBaseDir)
44dataDirs=['Netlib','miplib3']
45for d in dataDirs :
46  dataDir=os.path.join(dataBaseDir,d)
47  if not os.path.isdir(dataDir) :
48    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
49    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
50      sys.exit(1)
51    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
52netlibDir=os.path.join(dataBaseDir,'Netlib')
53miplib3Dir=os.path.join(dataBaseDir,'miplib3')
54
55#------------------------------------------------------------------------
56# Loop once for each project
57#------------------------------------------------------------------------
58for p in NBuserConfig.PROJECTS:
59  NBlogMessages.writeMessage( p )
60
61  #---------------------------------------------------------------------
62  # Loop once for each version of the project to be checked out.
63  # The supported types are trunk & latestStable
64  #---------------------------------------------------------------------
65  projectVersions=[]
66  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
67    lsv = NBsvnCommand.latestStableVersion(p)
68    projectVersions.append(['stable'+lsv,'stable/'+lsv])
69  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
70    projectVersions.append(['trunk','trunk'])
71  for projectVersion in projectVersions :
72    #---------------------------------------------------------------------
73    # svn checkout or update the project
74    #---------------------------------------------------------------------
75    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
76    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
77    if not os.path.isdir(projectBaseDir) :
78      os.makedirs(projectBaseDir)
79    if not os.path.isdir(projectCheckOutDir) :
80      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
81             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
82      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
83        continue
84    else :
85      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
86      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
87        continue
88
89    #---------------------------------------------------------------------
90    # If there are third party apps, then get these apps
91    #---------------------------------------------------------------------
92    if NBuserConfig.DOWNLOAD_3RD_PARTY :
93      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
94      if os.path.isdir(thirdPartyBaseDir) :
95        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
96        for d in thirdPartyDirs :
97          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
98          install3rdPartyCmd=os.path.join(".","get."+d)
99          os.chdir(thirdPartyDir)
100          # If the install command has been updated since the last
101          # install, then do a new install
102          if os.path.isfile('NBinstalldone') :
103            if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
104              os.remove('NBinstalldone')
105          if not os.path.isfile('NBinstalldone') :
106            if os.path.isfile(install3rdPartyCmd) :
107              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
108              NBosCommand.run(install3rdPartyCmd)
109              f=open('NBinstalldone','w')
110              f.close()
111          else :
112            NBlogMessages.writeMessage('  skipped anew download of '+d)
113   
114    #---------------------------------------------------------------------
115    # Loop once for each type of build to be done.
116    # Debug, use third party code, ...
117    # vpath and options to configure must be set for the buildType
118    #---------------------------------------------------------------------
119    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
120
121    print buildtypes
122   
123    for buildType, configLine in buildtypes.iteritems() :
124   
125      #---------------------------------------------------------------------
126      # Setup the directory where the build will be done and the configure
127      # command options
128      #---------------------------------------------------------------------
129      vpathDir=projectVersion[0]+'-'+buildType
130      configOptions='-C '+NBuserConfig.CONFIGURE_FLAGS+' '+configLine
131
132#      print 'HERE ARE THE CONFIG OPTIONS SO FAR'
133#      print configOptions
134
135
136#      if "ThirdParty" in buildType :
137#        vpathDir += "ThirdParty"
138#      else :
139#        vpathDir += "NoThirdParty"
140#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
141#        if os.path.isdir(thirdPartyBaseDir) :
142#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
143#          skipOptions=''
144#          for d in thirdPartyDirs :
145#            skipOptions+=' ThirdParty/'+d
146#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
147
148# Added by Kipp -- Sunday, Oct 21
149#      print buildType
150#      print configLine
151      if "NoThirdParty" in buildType :
152        skipOptions=''
153        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
154        if os.path.isdir(thirdPartyBaseDir) :
155          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
156          for d in thirdPartyDirs :
157            skipOptions+=' ThirdParty/'+d
158          skipOptions+=' Ipopt' 
159        configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
160# End Kipp
161
162
163      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
164      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
165      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
166      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
167
168      #---------------------------------------------------------------------
169      # Run configure part of build (only if config has not previously
170      # ran successfully).
171      #---------------------------------------------------------------------
172      os.chdir(fullVpathDir)
173      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
174      if NBcheckResult.didConfigRunOK() :
175        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
176      else :
177        NBlogMessages.writeMessage('  '+configCmd)
178        result=NBosCommand.run(configCmd)
179     
180        # Check if configure worked
181        if result['returnCode'] != 0 :
182          error_msg = result
183          # Add contents of log file to message
184          logFileName = 'config.log'
185          if os.path.isfile(logFileName) :
186            logFilePtr = open(logFileName,'r')
187            error_msg['config.log']  = "config.log contains: \n" 
188            error_msg['config.log'] += logFilePtr.read()
189            logFilePtr.close()
190          NBemail.sendCmdMsgs(p,error_msg,configCmd)
191          continue
192
193      #---------------------------------------------------------------------
194      # Run make part of build
195      #---------------------------------------------------------------------
196      NBlogMessages.writeMessage( '  make' )
197      result=NBosCommand.run('make')
198     
199      # Check if make worked
200      if result['returnCode'] != 0 :
201        NBemail.sendCmdMsgs(p,result,'make')
202        continue
203
204      #---------------------------------------------------------------------
205      # Run 'make test' part of build
206      #---------------------------------------------------------------------
207      NBlogMessages.writeMessage( '  make test' )
208      result=NBosCommand.run('make test')
209     
210      # Check if 'make test' worked
211      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
212      if didMakeTestFail :
213        result['make test']=didMakeTestFail
214        NBemail.sendCmdMsgs(p,result,"make test")
215        continue
216
217      #---------------------------------------------------------------------
218      # Run unitTest if available and different from 'make test'
219      #---------------------------------------------------------------------
220      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
221        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
222        os.chdir(unitTestPath)
223
224        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
225        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
226        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
227
228        NBlogMessages.writeMessage( '  '+unitTestCmd )
229        result=NBosCommand.run(unitTestCmd)
230     
231        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
232        if didUnitTestFail :
233          result['unitTest']=didUnitTestFail
234          NBemail.sendCmdMsgs(p,result,unitTestCmd)
235          continue
236
237NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
238
239sys.exit(0)
240
241
242# START KIPP
243#----------------------------------------------------------------------
244# CONFIG FILE PATH:
245#   path to the config file for the build
246#   done. If the directory does not exist, it will be created.
247#   this should have all of the user specific data
248#   it should have values for
249#   NIGHTLY_BUILD_ROOT
250#   NBuserConfig.SMTP_SERVER_NAME
251#   NBuserConfig.SMTP_SERVER_PORT
252#   NBuserConfig.SMTP_SSL_SERVER
253#   NBuserConfig.SMTP_USER_NAME
254#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
255#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
256#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
257#   
258#----------------------------------------------------------------------
259
260CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
261CONFIG_FILENAME = 'config.txt'
262
263
264# Get configFile data
265
266configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
267                                 os.path.basename(CONFIG_FILENAME ))
268if os.path.isfile(  configFile) :
269  pwFilePtr = open(configFile ,'r')
270  d = pwFilePtr.readlines()
271  # do pwFilePtr.read() to get a string object
272  # we have a list object
273  print d[0]
274  print d[1]
275  # make a dictionary
276  config_dic = {}
277
278  #smtppass  = pwFilePtr.read().strip()
279  pwFilePtr.close()
280else :
281  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
282  #print cmdMsgs
283  sys.exit( 1)
284sys.exit( 0)
285
286
287
288# START KIPP
289#----------------------------------------------------------------------
290#   path to the config file for the build
291#   get the user dependent variables
292# CONFIG FILE PATH:
293#   it should have values for
294#   NIGHTLY_BUILD_ROOT
295#   NBuserConfig.SMTP_SERVER_NAME
296#   NBuserConfig.SMTP_SERVER_PORT
297#   NBuserConfig.SMTP_SSL_SERVER
298#   NBuserConfig.SMTP_USER_NAME
299#   NBuserConfig.SMTP_PASSWORD_FILENAME
300#   NBuserConfig.SENDER_EMAIL_ADDR
301#   NBuserConfig.MY_EMAIL_ADDR
302#   
303#----------------------------------------------------------------------
304
305CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
306CONFIG_FILENAME = 'config.txt'
307
308
309# Get configFile data
310
311configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
312                                 os.path.basename(CONFIG_FILENAME ))
313if os.path.isfile(  configFile) :
314  pwFilePtr = open(configFile ,'r')
315  d = pwFilePtr.readlines()
316  # do pwFilePtr.read() to get a string object
317  # we have a list object
318  print d[0]
319  print d[1]
320  # make a dictionary
321  config_dic = {}
322
323  #smtppass  = pwFilePtr.read().strip()
324  pwFilePtr.close()
325else :
326  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
327  #print cmdMsgs
328  sys.exit( 1)
329sys.exit( 0)
330
331# END KIPP
Note: See TracBrowser for help on using the repository browser.