source: branches/testScripts/nightlyBuild.py @ 650

Last change on this file since 650 was 650, checked in by stefan, 14 years ago

make it possible to define project dependent build types (configure call parameters)

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 11.4 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Figure out how to run configure with no third party code
21
22
23
24
25#------------------------------------------------------------------------
26#  Main Program Starts Here 
27#------------------------------------------------------------------------
28
29#------------------------------------------------------------------------
30#  If needed create the top level directory
31#------------------------------------------------------------------------
32if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
33  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
34os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
35
36#------------------------------------------------------------------------
37#  Get the data directories if they don't already exist
38#------------------------------------------------------------------------
39dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
40if not os.path.isdir(dataBaseDir) :
41  os.makedirs(dataBaseDir)
42dataDirs=['Netlib','miplib3']
43for d in dataDirs :
44  dataDir=os.path.join(dataBaseDir,d)
45  if not os.path.isdir(dataDir) :
46    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
47    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
48      sys.exit(1)
49    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
50netlibDir=os.path.join(dataBaseDir,'Netlib')
51miplib3Dir=os.path.join(dataBaseDir,'miplib3')
52
53#------------------------------------------------------------------------
54# Loop once for each project
55#------------------------------------------------------------------------
56for p in NBuserConfig.PROJECTS:
57  NBlogMessages.writeMessage( p )
58
59  #---------------------------------------------------------------------
60  # Loop once for each version of the project to be checked out.
61  # The supported types are trunk & latestStable
62  #---------------------------------------------------------------------
63  projectVersions=[]
64  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
65    lsv = NBsvnCommand.latestStableVersion(p)
66    projectVersions.append(['stable'+lsv,'stable/'+lsv])
67  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
68    projectVersions.append(['trunk','trunk'])
69  for projectVersion in projectVersions :
70    #---------------------------------------------------------------------
71    # svn checkout or update the project
72    #---------------------------------------------------------------------
73    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
74    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
75    if not os.path.isdir(projectBaseDir) :
76      os.makedirs(projectBaseDir)
77    if not os.path.isdir(projectCheckOutDir) :
78      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
79             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
80      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
81        continue
82    else :
83      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
84      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
85        continue
86
87    #---------------------------------------------------------------------
88    # If there are third party apps, then get these apps
89    #---------------------------------------------------------------------
90    if NBuserConfig.DOWNLOAD_3RD_PARTY :
91      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
92      if os.path.isdir(thirdPartyBaseDir) :
93        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
94        for d in thirdPartyDirs :
95          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
96          install3rdPartyCmd=os.path.join(".","get."+d)
97          os.chdir(thirdPartyDir)
98          if os.path.isfile(install3rdPartyCmd) :
99            NBlogMessages.writeMessage('  '+install3rdPartyCmd)
100            NBosCommand.run(install3rdPartyCmd)
101   
102    #---------------------------------------------------------------------
103    # Loop once for each type of build to be done.
104    # Debug, use third party code, ...
105    # vpath and options to configure must be set for the buildType
106    #---------------------------------------------------------------------
107    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
108    for buildType, configLine in buildtypes.iteritems() :
109   
110      #---------------------------------------------------------------------
111      # Setup the directory where the build will be done and the configure
112      # command options
113      #---------------------------------------------------------------------
114      vpathDir=projectVersion[0]+'-'+buildType
115      configOptions='-C '+configLine
116
117#      if "ThirdParty" in buildType :
118#        vpathDir += "ThirdParty"
119#      else :
120#        vpathDir += "NoThirdParty"
121#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
122#        if os.path.isdir(thirdPartyBaseDir) :
123#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
124#          skipOptions=''
125#          for d in thirdPartyDirs :
126#            skipOptions+=' ThirdParty/'+d
127#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
128     
129      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
130      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
131      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
132      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
133
134      #---------------------------------------------------------------------
135      # Run configure part of build (only if config has not previously
136      # ran successfully).
137      #---------------------------------------------------------------------
138      os.chdir(fullVpathDir)
139      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
140      if NBcheckResult.didConfigRunOK() :
141        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
142      else :
143        NBlogMessages.writeMessage('  '+configCmd)
144        result=NBosCommand.run(configCmd)
145     
146        # Check if configure worked
147        if result['returnCode'] != 0 :
148          error_msg = result
149          # Add contents of log file to message
150          logFileName = 'config.log'
151          if os.path.isfile(logFileName) :
152            logFilePtr = open(logFileName,'r')
153            error_msg['config.log']  = "config.log contains: \n" 
154            error_msg['config.log'] += logFilePtr.read()
155            logFilePtr.close()
156          NBemail.sendCmdMsgs(p,error_msg,configCmd)
157          continue
158
159      #---------------------------------------------------------------------
160      # Run make part of build
161      #---------------------------------------------------------------------
162      NBlogMessages.writeMessage( '  make' )
163      result=NBosCommand.run('make')
164     
165      # Check if make worked
166      if result['returnCode'] != 0 :
167        NBemail.sendCmdMsgs(p,result,'make')
168        continue
169
170      #---------------------------------------------------------------------
171      # Run 'make test' part of build
172      #---------------------------------------------------------------------
173      NBlogMessages.writeMessage( '  make test' )
174      result=NBosCommand.run('make test')
175     
176      # Check if 'make test' worked
177      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
178      if didMakeTestFail :
179        result['make test']=didMakeTestFail
180        NBemail.sendCmdMsgs(p,result,"make test")
181        continue
182
183      #---------------------------------------------------------------------
184      # Run unitTest if available and different from 'make test'
185      #---------------------------------------------------------------------
186      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
187        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
188        os.chdir(unitTestPath)
189
190        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
191        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
192        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
193
194        NBlogMessages.writeMessage( '  '+unitTestCmd )
195        result=NBosCommand.run(unitTestCmd)
196     
197        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
198        if didUnitTestFail :
199          result['unitTest']=didUnitTestFail
200          NBemail.sendCmdMsgs(p,result,unitTestCmd)
201          continue
202
203NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
204
205sys.exit(0)
206
207
208# START KIPP
209#----------------------------------------------------------------------
210# CONFIG FILE PATH:
211#   path to the config file for the build
212#   done. If the directory does not exist, it will be created.
213#   this should have all of the user specific data
214#   it should have values for
215#   NIGHTLY_BUILD_ROOT
216#   NBuserConfig.SMTP_SERVER_NAME
217#   NBuserConfig.SMTP_SERVER_PORT
218#   NBuserConfig.SMTP_SSL_SERVER
219#   NBuserConfig.SMTP_USER_NAME
220#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
221#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
222#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
223#   
224#----------------------------------------------------------------------
225
226CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
227CONFIG_FILENAME = 'config.txt'
228
229
230# Get configFile data
231
232configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
233                                 os.path.basename(CONFIG_FILENAME ))
234if os.path.isfile(  configFile) :
235  pwFilePtr = open(configFile ,'r')
236  d = pwFilePtr.readlines()
237  # do pwFilePtr.read() to get a string object
238  # we have a list object
239  print d[0]
240  print d[1]
241  # make a dictionary
242  config_dic = {}
243
244  #smtppass  = pwFilePtr.read().strip()
245  pwFilePtr.close()
246else :
247  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
248  #print cmdMsgs
249  sys.exit( 1)
250sys.exit( 0)
251
252
253
254# START KIPP
255#----------------------------------------------------------------------
256#   path to the config file for the build
257#   get the user dependent variables
258# CONFIG FILE PATH:
259#   it should have values for
260#   NIGHTLY_BUILD_ROOT
261#   NBuserConfig.SMTP_SERVER_NAME
262#   NBuserConfig.SMTP_SERVER_PORT
263#   NBuserConfig.SMTP_SSL_SERVER
264#   NBuserConfig.SMTP_USER_NAME
265#   NBuserConfig.SMTP_PASSWORD_FILENAME
266#   NBuserConfig.SENDER_EMAIL_ADDR
267#   NBuserConfig.MY_EMAIL_ADDR
268#   
269#----------------------------------------------------------------------
270
271CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
272CONFIG_FILENAME = 'config.txt'
273
274
275# Get configFile data
276
277configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
278                                 os.path.basename(CONFIG_FILENAME ))
279if os.path.isfile(  configFile) :
280  pwFilePtr = open(configFile ,'r')
281  d = pwFilePtr.readlines()
282  # do pwFilePtr.read() to get a string object
283  # we have a list object
284  print d[0]
285  print d[1]
286  # make a dictionary
287  config_dic = {}
288
289  #smtppass  = pwFilePtr.read().strip()
290  pwFilePtr.close()
291else :
292  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
293  #print cmdMsgs
294  sys.exit( 1)
295sys.exit( 0)
296
297# END KIPP
Note: See TracBrowser for help on using the repository browser.