source: branches/testScripts/nightlyBuild.py @ 672

Last change on this file since 672 was 672, checked in by stefan, 12 years ago

add some defaults to userParametersEmpty.py to make clear that '0' is not the same as 0

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 12.8 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Skip make of project that depend on a project which make had been failing. (Similar for configure.)
21#    But also tell project managers of skiped project that their project was skipped.
22#   -Store output of get.XXX calls, make, make test, configures... in some files.
23#   -Allow a fine configuration of which build tests to run per project.
24#   -Move actual [get project, configure, make, tests] part into an extra file.
25
26
27#------------------------------------------------------------------------
28#  Main Program Starts Here 
29#------------------------------------------------------------------------
30
31#------------------------------------------------------------------------
32#  If needed create the top level directory
33#------------------------------------------------------------------------
34if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
35  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
36os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
37
38#------------------------------------------------------------------------
39#  Get the data directories if they don't already exist
40#------------------------------------------------------------------------
41dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
42if not os.path.isdir(dataBaseDir) :
43  os.makedirs(dataBaseDir)
44dataDirs=['Netlib','miplib3']
45for d in dataDirs :
46  dataDir=os.path.join(dataBaseDir,d)
47  if not os.path.isdir(dataDir) :
48    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
49    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
50      sys.exit(1)
51    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
52netlibDir=os.path.join(dataBaseDir,'Netlib')
53miplib3Dir=os.path.join(dataBaseDir,'miplib3')
54
55#------------------------------------------------------------------------
56# Loop once for each project
57#------------------------------------------------------------------------
58for p in NBuserConfig.PROJECTS:
59  NBlogMessages.writeMessage( p )
60
61  #---------------------------------------------------------------------
62  # Loop once for each version of the project to be checked out.
63  # The supported types are trunk & latestStable
64  #---------------------------------------------------------------------
65  projectVersions=[]
66  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
67    lsv = NBsvnCommand.latestStableVersion(p)
68    projectVersions.append(['stable'+lsv,'stable/'+lsv])
69  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
70    projectVersions.append(['trunk','trunk'])
71  for projectVersion in projectVersions :
72    #---------------------------------------------------------------------
73    # svn checkout or update the project
74    #---------------------------------------------------------------------
75    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
76    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
77    if not os.path.isdir(projectBaseDir) :
78      os.makedirs(projectBaseDir)
79    if not os.path.isdir(projectCheckOutDir) :
80      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
81             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
82      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
83        continue
84    else :
85      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
86      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
87        continue
88
89    #---------------------------------------------------------------------
90    # If there are third party apps, then get these apps
91    #---------------------------------------------------------------------
92    if NBuserConfig.DOWNLOAD_3RD_PARTY :
93      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
94      if os.path.isdir(thirdPartyBaseDir) :
95        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
96        for d in thirdPartyDirs :
97          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
98          install3rdPartyCmd=os.path.join(".","get."+d)
99          os.chdir(thirdPartyDir)
100          # If the install command has been updated since the last
101          # install, then do a new install
102          if os.path.isfile('NBinstalldone') :
103            if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
104              os.remove('NBinstalldone')
105          if not os.path.isfile('NBinstalldone') :
106            if os.path.isfile(install3rdPartyCmd) :
107              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
108              NBosCommand.run(install3rdPartyCmd)
109              f=open('NBinstalldone','w')
110              f.close()
111          else :
112            NBlogMessages.writeMessage('  skipped anew download of '+d)
113   
114    #---------------------------------------------------------------------
115    # Loop once for each type of build to be done.
116    # Debug, use third party code, ...
117    # vpath and options to configure must be set for the buildType
118    #---------------------------------------------------------------------
119    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
120
121#    print buildtypes
122   
123    for buildType, configLine in buildtypes.iteritems() :
124   
125      #---------------------------------------------------------------------
126      # Setup the directory where the build will be done and the configure
127      # command options
128      #---------------------------------------------------------------------
129      vpathDir=projectVersion[0]+'-'+buildType
130      configOptions='-C '+NBuserConfig.CONFIGURE_FLAGS+' '+configLine
131
132#      print 'HERE ARE THE CONFIG OPTIONS SO FAR'
133#      print configOptions
134
135
136#      if "ThirdParty" in buildType :
137#        vpathDir += "ThirdParty"
138#      else :
139#        vpathDir += "NoThirdParty"
140#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
141#        if os.path.isdir(thirdPartyBaseDir) :
142#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
143#          skipOptions=''
144#          for d in thirdPartyDirs :
145#            skipOptions+=' ThirdParty/'+d
146#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
147
148# Added by Kipp -- Sunday, Oct 21
149#      print buildType
150#      print configLine
151      if "NoThirdParty" in buildType :
152        skipOptions=''
153        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
154        if os.path.isdir(thirdPartyBaseDir) :
155          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
156          for d in thirdPartyDirs :
157            skipOptions+=' ThirdParty/'+d
158          skipOptions+=' Ipopt' 
159        configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
160# End Kipp
161
162
163      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
164      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
165      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
166      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
167
168      #---------------------------------------------------------------------
169      # Run configure part of build (only if config has not previously
170      # ran successfully).
171      #---------------------------------------------------------------------
172      os.chdir(fullVpathDir)
173      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
174      if NBcheckResult.didConfigRunOK() :
175        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
176      else :
177        NBlogMessages.writeMessage('  '+configCmd)
178        result=NBosCommand.run(configCmd)
179     
180        # Check if configure worked
181        if result['returnCode'] != 0 :
182          error_msg = result
183          # Add contents of log file to message
184          logFileName = 'config.log'
185          if os.path.isfile(logFileName) :
186            logFilePtr = open(logFileName,'r')
187            error_msg['config.log']  = "config.log contains: \n" 
188            error_msg['config.log'] += logFilePtr.read()
189            logFilePtr.close()
190          NBemail.sendCmdMsgs(p,error_msg,configCmd)
191          continue
192
193      #---------------------------------------------------------------------
194      # Run make part of build
195      #---------------------------------------------------------------------
196      NBlogMessages.writeMessage( '  make' )
197      result=NBosCommand.run('make')
198     
199      # Check if make worked
200      if result['returnCode'] != 0 :
201        NBemail.sendCmdMsgs(p,result,'make')
202        continue
203
204      #---------------------------------------------------------------------
205      # Run 'make test' part of build
206      #---------------------------------------------------------------------
207      NBlogMessages.writeMessage( '  make test' )
208      result=NBosCommand.run('make test')
209     
210      # Check if 'make test' worked
211      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
212      if didMakeTestFail :
213        result['make test']=didMakeTestFail
214        NBemail.sendCmdMsgs(p,result,"make test")
215        continue
216
217      #---------------------------------------------------------------------
218      # Run unitTest if available and different from 'make test'
219      #---------------------------------------------------------------------
220      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
221        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
222        os.chdir(unitTestPath)
223
224        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
225        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
226        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
227
228        NBlogMessages.writeMessage( '  '+unitTestCmd )
229        result=NBosCommand.run(unitTestCmd)
230     
231        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
232        if didUnitTestFail :
233          result['unitTest']=didUnitTestFail
234          NBemail.sendCmdMsgs(p,result,unitTestCmd)
235          continue
236
237NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
238
239sys.exit(0)
240
241
242# START KIPP
243#----------------------------------------------------------------------
244# CONFIG FILE PATH:
245#   path to the config file for the build
246#   done. If the directory does not exist, it will be created.
247#   this should have all of the user specific data
248#   it should have values for
249#   NIGHTLY_BUILD_ROOT
250#   NBuserConfig.SMTP_SERVER_NAME
251#   NBuserConfig.SMTP_SERVER_PORT
252#   NBuserConfig.SMTP_SSL_SERVER
253#   NBuserConfig.SMTP_USER_NAME
254#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
255#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
256#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
257#   
258#----------------------------------------------------------------------
259
260CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
261CONFIG_FILENAME = 'config.txt'
262
263
264# Get configFile data
265
266configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
267                                 os.path.basename(CONFIG_FILENAME ))
268if os.path.isfile(  configFile) :
269  pwFilePtr = open(configFile ,'r')
270  d = pwFilePtr.readlines()
271  # do pwFilePtr.read() to get a string object
272  # we have a list object
273  print d[0]
274  print d[1]
275  # make a dictionary
276  config_dic = {}
277
278  #smtppass  = pwFilePtr.read().strip()
279  pwFilePtr.close()
280else :
281  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
282  #print cmdMsgs
283  sys.exit( 1)
284sys.exit( 0)
285
286
287
288# START KIPP
289#----------------------------------------------------------------------
290#   path to the config file for the build
291#   get the user dependent variables
292# CONFIG FILE PATH:
293#   it should have values for
294#   NIGHTLY_BUILD_ROOT
295#   NBuserConfig.SMTP_SERVER_NAME
296#   NBuserConfig.SMTP_SERVER_PORT
297#   NBuserConfig.SMTP_SSL_SERVER
298#   NBuserConfig.SMTP_USER_NAME
299#   NBuserConfig.SMTP_PASSWORD_FILENAME
300#   NBuserConfig.SENDER_EMAIL_ADDR
301#   NBuserConfig.MY_EMAIL_ADDR
302#   
303#----------------------------------------------------------------------
304
305CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
306CONFIG_FILENAME = 'config.txt'
307
308
309# Get configFile data
310
311configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
312                                 os.path.basename(CONFIG_FILENAME ))
313if os.path.isfile(  configFile) :
314  pwFilePtr = open(configFile ,'r')
315  d = pwFilePtr.readlines()
316  # do pwFilePtr.read() to get a string object
317  # we have a list object
318  print d[0]
319  print d[1]
320  # make a dictionary
321  config_dic = {}
322
323  #smtppass  = pwFilePtr.read().strip()
324  pwFilePtr.close()
325else :
326  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
327  #print cmdMsgs
328  sys.exit( 1)
329sys.exit( 0)
330
331# END KIPP
Note: See TracBrowser for help on using the repository browser.