source: branches/testScripts/nightlyBuild.py @ 657

Last change on this file since 657 was 657, checked in by jpfasano, 12 years ago

Updated nightlyBuild to get 3rd party code, if the script for getting the
3rd party code has been updated since the 3rd party code had previously
been obtained.

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 11.8 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Figure out how to run configure with no third party code (done, or?)
21
22
23
24
25#------------------------------------------------------------------------
26#  Main Program Starts Here 
27#------------------------------------------------------------------------
28
29#------------------------------------------------------------------------
30#  If needed create the top level directory
31#------------------------------------------------------------------------
32if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
33  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
34os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
35
36#------------------------------------------------------------------------
37#  Get the data directories if they don't already exist
38#------------------------------------------------------------------------
39dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
40if not os.path.isdir(dataBaseDir) :
41  os.makedirs(dataBaseDir)
42dataDirs=['Netlib','miplib3']
43for d in dataDirs :
44  dataDir=os.path.join(dataBaseDir,d)
45  if not os.path.isdir(dataDir) :
46    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
47    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
48      sys.exit(1)
49    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
50netlibDir=os.path.join(dataBaseDir,'Netlib')
51miplib3Dir=os.path.join(dataBaseDir,'miplib3')
52
53#------------------------------------------------------------------------
54# Loop once for each project
55#------------------------------------------------------------------------
56for p in NBuserConfig.PROJECTS:
57  NBlogMessages.writeMessage( p )
58
59  #---------------------------------------------------------------------
60  # Loop once for each version of the project to be checked out.
61  # The supported types are trunk & latestStable
62  #---------------------------------------------------------------------
63  projectVersions=[]
64  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
65    lsv = NBsvnCommand.latestStableVersion(p)
66    projectVersions.append(['stable'+lsv,'stable/'+lsv])
67  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
68    projectVersions.append(['trunk','trunk'])
69  for projectVersion in projectVersions :
70    #---------------------------------------------------------------------
71    # svn checkout or update the project
72    #---------------------------------------------------------------------
73    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
74    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
75    if not os.path.isdir(projectBaseDir) :
76      os.makedirs(projectBaseDir)
77    if not os.path.isdir(projectCheckOutDir) :
78      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
79             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
80      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
81        continue
82    else :
83      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
84      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
85        continue
86
87    #---------------------------------------------------------------------
88    # If there are third party apps, then get these apps
89    #---------------------------------------------------------------------
90    if NBuserConfig.DOWNLOAD_3RD_PARTY :
91      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
92      if os.path.isdir(thirdPartyBaseDir) :
93        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
94        for d in thirdPartyDirs :
95          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
96          install3rdPartyCmd=os.path.join(".","get."+d)
97          os.chdir(thirdPartyDir)
98          # If the install command has been updated since the last
99          # install, then do a new install
100          if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
101            os.remove('NBinstalldone')
102          if not os.path.isfile('NBinstalldone') :
103            if os.path.isfile(install3rdPartyCmd) :
104              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
105              NBosCommand.run(install3rdPartyCmd)
106              f=open('NBinstalldone','w')
107              f.close()
108          else :
109            NBlogMessages.writeMessage('  skipped anew download of '+d)
110   
111    #---------------------------------------------------------------------
112    # Loop once for each type of build to be done.
113    # Debug, use third party code, ...
114    # vpath and options to configure must be set for the buildType
115    #---------------------------------------------------------------------
116    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
117    for buildType, configLine in buildtypes.iteritems() :
118   
119      #---------------------------------------------------------------------
120      # Setup the directory where the build will be done and the configure
121      # command options
122      #---------------------------------------------------------------------
123      vpathDir=projectVersion[0]+'-'+buildType
124      configOptions='-C '+NBuserConfig.CONFIGURE_FLAGS+' '+configLine
125
126#      if "ThirdParty" in buildType :
127#        vpathDir += "ThirdParty"
128#      else :
129#        vpathDir += "NoThirdParty"
130#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
131#        if os.path.isdir(thirdPartyBaseDir) :
132#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
133#          skipOptions=''
134#          for d in thirdPartyDirs :
135#            skipOptions+=' ThirdParty/'+d
136#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
137     
138      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
139      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
140      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
141      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
142
143      #---------------------------------------------------------------------
144      # Run configure part of build (only if config has not previously
145      # ran successfully).
146      #---------------------------------------------------------------------
147      os.chdir(fullVpathDir)
148      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
149      if NBcheckResult.didConfigRunOK() :
150        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
151      else :
152        NBlogMessages.writeMessage('  '+configCmd)
153        result=NBosCommand.run(configCmd)
154     
155        # Check if configure worked
156        if result['returnCode'] != 0 :
157          error_msg = result
158          # Add contents of log file to message
159          logFileName = 'config.log'
160          if os.path.isfile(logFileName) :
161            logFilePtr = open(logFileName,'r')
162            error_msg['config.log']  = "config.log contains: \n" 
163            error_msg['config.log'] += logFilePtr.read()
164            logFilePtr.close()
165          NBemail.sendCmdMsgs(p,error_msg,configCmd)
166          continue
167
168      #---------------------------------------------------------------------
169      # Run make part of build
170      #---------------------------------------------------------------------
171      NBlogMessages.writeMessage( '  make' )
172      result=NBosCommand.run('make')
173     
174      # Check if make worked
175      if result['returnCode'] != 0 :
176        NBemail.sendCmdMsgs(p,result,'make')
177        continue
178
179      #---------------------------------------------------------------------
180      # Run 'make test' part of build
181      #---------------------------------------------------------------------
182      NBlogMessages.writeMessage( '  make test' )
183      result=NBosCommand.run('make test')
184     
185      # Check if 'make test' worked
186      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
187      if didMakeTestFail :
188        result['make test']=didMakeTestFail
189        NBemail.sendCmdMsgs(p,result,"make test")
190        continue
191
192      #---------------------------------------------------------------------
193      # Run unitTest if available and different from 'make test'
194      #---------------------------------------------------------------------
195      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
196        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
197        os.chdir(unitTestPath)
198
199        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
200        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
201        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
202
203        NBlogMessages.writeMessage( '  '+unitTestCmd )
204        result=NBosCommand.run(unitTestCmd)
205     
206        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
207        if didUnitTestFail :
208          result['unitTest']=didUnitTestFail
209          NBemail.sendCmdMsgs(p,result,unitTestCmd)
210          continue
211
212NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
213
214sys.exit(0)
215
216
217# START KIPP
218#----------------------------------------------------------------------
219# CONFIG FILE PATH:
220#   path to the config file for the build
221#   done. If the directory does not exist, it will be created.
222#   this should have all of the user specific data
223#   it should have values for
224#   NIGHTLY_BUILD_ROOT
225#   NBuserConfig.SMTP_SERVER_NAME
226#   NBuserConfig.SMTP_SERVER_PORT
227#   NBuserConfig.SMTP_SSL_SERVER
228#   NBuserConfig.SMTP_USER_NAME
229#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
230#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
231#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
232#   
233#----------------------------------------------------------------------
234
235CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
236CONFIG_FILENAME = 'config.txt'
237
238
239# Get configFile data
240
241configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
242                                 os.path.basename(CONFIG_FILENAME ))
243if os.path.isfile(  configFile) :
244  pwFilePtr = open(configFile ,'r')
245  d = pwFilePtr.readlines()
246  # do pwFilePtr.read() to get a string object
247  # we have a list object
248  print d[0]
249  print d[1]
250  # make a dictionary
251  config_dic = {}
252
253  #smtppass  = pwFilePtr.read().strip()
254  pwFilePtr.close()
255else :
256  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
257  #print cmdMsgs
258  sys.exit( 1)
259sys.exit( 0)
260
261
262
263# START KIPP
264#----------------------------------------------------------------------
265#   path to the config file for the build
266#   get the user dependent variables
267# CONFIG FILE PATH:
268#   it should have values for
269#   NIGHTLY_BUILD_ROOT
270#   NBuserConfig.SMTP_SERVER_NAME
271#   NBuserConfig.SMTP_SERVER_PORT
272#   NBuserConfig.SMTP_SSL_SERVER
273#   NBuserConfig.SMTP_USER_NAME
274#   NBuserConfig.SMTP_PASSWORD_FILENAME
275#   NBuserConfig.SENDER_EMAIL_ADDR
276#   NBuserConfig.MY_EMAIL_ADDR
277#   
278#----------------------------------------------------------------------
279
280CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
281CONFIG_FILENAME = 'config.txt'
282
283
284# Get configFile data
285
286configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
287                                 os.path.basename(CONFIG_FILENAME ))
288if os.path.isfile(  configFile) :
289  pwFilePtr = open(configFile ,'r')
290  d = pwFilePtr.readlines()
291  # do pwFilePtr.read() to get a string object
292  # we have a list object
293  print d[0]
294  print d[1]
295  # make a dictionary
296  config_dic = {}
297
298  #smtppass  = pwFilePtr.read().strip()
299  pwFilePtr.close()
300else :
301  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
302  #print cmdMsgs
303  sys.exit( 1)
304sys.exit( 0)
305
306# END KIPP
Note: See TracBrowser for help on using the repository browser.