source: branches/testScripts/nightlyBuild.py @ 658

Last change on this file since 658 was 658, checked in by jpfasano, 12 years ago

fixed bug in logic (i made) regarding getting 3rd party code

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 11.9 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Figure out how to run configure with no third party code (done, or?)
21
22
23
24
25#------------------------------------------------------------------------
26#  Main Program Starts Here 
27#------------------------------------------------------------------------
28
29#------------------------------------------------------------------------
30#  If needed create the top level directory
31#------------------------------------------------------------------------
32if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
33  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
34os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
35
36#------------------------------------------------------------------------
37#  Get the data directories if they don't already exist
38#------------------------------------------------------------------------
39dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
40if not os.path.isdir(dataBaseDir) :
41  os.makedirs(dataBaseDir)
42dataDirs=['Netlib','miplib3']
43for d in dataDirs :
44  dataDir=os.path.join(dataBaseDir,d)
45  if not os.path.isdir(dataDir) :
46    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
47    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
48      sys.exit(1)
49    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
50netlibDir=os.path.join(dataBaseDir,'Netlib')
51miplib3Dir=os.path.join(dataBaseDir,'miplib3')
52
53#------------------------------------------------------------------------
54# Loop once for each project
55#------------------------------------------------------------------------
56for p in NBuserConfig.PROJECTS:
57  NBlogMessages.writeMessage( p )
58
59  #---------------------------------------------------------------------
60  # Loop once for each version of the project to be checked out.
61  # The supported types are trunk & latestStable
62  #---------------------------------------------------------------------
63  projectVersions=[]
64  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
65    lsv = NBsvnCommand.latestStableVersion(p)
66    projectVersions.append(['stable'+lsv,'stable/'+lsv])
67  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
68    projectVersions.append(['trunk','trunk'])
69  for projectVersion in projectVersions :
70    #---------------------------------------------------------------------
71    # svn checkout or update the project
72    #---------------------------------------------------------------------
73    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
74    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
75    if not os.path.isdir(projectBaseDir) :
76      os.makedirs(projectBaseDir)
77    if not os.path.isdir(projectCheckOutDir) :
78      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
79             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
80      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
81        continue
82    else :
83      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
84      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
85        continue
86
87    #---------------------------------------------------------------------
88    # If there are third party apps, then get these apps
89    #---------------------------------------------------------------------
90    if NBuserConfig.DOWNLOAD_3RD_PARTY :
91      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
92      if os.path.isdir(thirdPartyBaseDir) :
93        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
94        for d in thirdPartyDirs :
95          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
96          install3rdPartyCmd=os.path.join(".","get."+d)
97          os.chdir(thirdPartyDir)
98          # If the install command has been updated since the last
99          # install, then do a new install
100          if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
101            if os.path.isfile('NBinstalldone') :
102              os.remove('NBinstalldone')
103          if not os.path.isfile('NBinstalldone') :
104            if os.path.isfile(install3rdPartyCmd) :
105              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
106              NBosCommand.run(install3rdPartyCmd)
107              f=open('NBinstalldone','w')
108              f.close()
109          else :
110            NBlogMessages.writeMessage('  skipped anew download of '+d)
111   
112    #---------------------------------------------------------------------
113    # Loop once for each type of build to be done.
114    # Debug, use third party code, ...
115    # vpath and options to configure must be set for the buildType
116    #---------------------------------------------------------------------
117    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
118    for buildType, configLine in buildtypes.iteritems() :
119   
120      #---------------------------------------------------------------------
121      # Setup the directory where the build will be done and the configure
122      # command options
123      #---------------------------------------------------------------------
124      vpathDir=projectVersion[0]+'-'+buildType
125      configOptions='-C '+NBuserConfig.CONFIGURE_FLAGS+' '+configLine
126
127#      if "ThirdParty" in buildType :
128#        vpathDir += "ThirdParty"
129#      else :
130#        vpathDir += "NoThirdParty"
131#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
132#        if os.path.isdir(thirdPartyBaseDir) :
133#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
134#          skipOptions=''
135#          for d in thirdPartyDirs :
136#            skipOptions+=' ThirdParty/'+d
137#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
138     
139      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
140      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
141      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
142      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
143
144      #---------------------------------------------------------------------
145      # Run configure part of build (only if config has not previously
146      # ran successfully).
147      #---------------------------------------------------------------------
148      os.chdir(fullVpathDir)
149      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
150      if NBcheckResult.didConfigRunOK() :
151        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
152      else :
153        NBlogMessages.writeMessage('  '+configCmd)
154        result=NBosCommand.run(configCmd)
155     
156        # Check if configure worked
157        if result['returnCode'] != 0 :
158          error_msg = result
159          # Add contents of log file to message
160          logFileName = 'config.log'
161          if os.path.isfile(logFileName) :
162            logFilePtr = open(logFileName,'r')
163            error_msg['config.log']  = "config.log contains: \n" 
164            error_msg['config.log'] += logFilePtr.read()
165            logFilePtr.close()
166          NBemail.sendCmdMsgs(p,error_msg,configCmd)
167          continue
168
169      #---------------------------------------------------------------------
170      # Run make part of build
171      #---------------------------------------------------------------------
172      NBlogMessages.writeMessage( '  make' )
173      result=NBosCommand.run('make')
174     
175      # Check if make worked
176      if result['returnCode'] != 0 :
177        NBemail.sendCmdMsgs(p,result,'make')
178        continue
179
180      #---------------------------------------------------------------------
181      # Run 'make test' part of build
182      #---------------------------------------------------------------------
183      NBlogMessages.writeMessage( '  make test' )
184      result=NBosCommand.run('make test')
185     
186      # Check if 'make test' worked
187      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
188      if didMakeTestFail :
189        result['make test']=didMakeTestFail
190        NBemail.sendCmdMsgs(p,result,"make test")
191        continue
192
193      #---------------------------------------------------------------------
194      # Run unitTest if available and different from 'make test'
195      #---------------------------------------------------------------------
196      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
197        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
198        os.chdir(unitTestPath)
199
200        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
201        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
202        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
203
204        NBlogMessages.writeMessage( '  '+unitTestCmd )
205        result=NBosCommand.run(unitTestCmd)
206     
207        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
208        if didUnitTestFail :
209          result['unitTest']=didUnitTestFail
210          NBemail.sendCmdMsgs(p,result,unitTestCmd)
211          continue
212
213NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
214
215sys.exit(0)
216
217
218# START KIPP
219#----------------------------------------------------------------------
220# CONFIG FILE PATH:
221#   path to the config file for the build
222#   done. If the directory does not exist, it will be created.
223#   this should have all of the user specific data
224#   it should have values for
225#   NIGHTLY_BUILD_ROOT
226#   NBuserConfig.SMTP_SERVER_NAME
227#   NBuserConfig.SMTP_SERVER_PORT
228#   NBuserConfig.SMTP_SSL_SERVER
229#   NBuserConfig.SMTP_USER_NAME
230#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
231#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
232#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
233#   
234#----------------------------------------------------------------------
235
236CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
237CONFIG_FILENAME = 'config.txt'
238
239
240# Get configFile data
241
242configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
243                                 os.path.basename(CONFIG_FILENAME ))
244if os.path.isfile(  configFile) :
245  pwFilePtr = open(configFile ,'r')
246  d = pwFilePtr.readlines()
247  # do pwFilePtr.read() to get a string object
248  # we have a list object
249  print d[0]
250  print d[1]
251  # make a dictionary
252  config_dic = {}
253
254  #smtppass  = pwFilePtr.read().strip()
255  pwFilePtr.close()
256else :
257  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
258  #print cmdMsgs
259  sys.exit( 1)
260sys.exit( 0)
261
262
263
264# START KIPP
265#----------------------------------------------------------------------
266#   path to the config file for the build
267#   get the user dependent variables
268# CONFIG FILE PATH:
269#   it should have values for
270#   NIGHTLY_BUILD_ROOT
271#   NBuserConfig.SMTP_SERVER_NAME
272#   NBuserConfig.SMTP_SERVER_PORT
273#   NBuserConfig.SMTP_SSL_SERVER
274#   NBuserConfig.SMTP_USER_NAME
275#   NBuserConfig.SMTP_PASSWORD_FILENAME
276#   NBuserConfig.SENDER_EMAIL_ADDR
277#   NBuserConfig.MY_EMAIL_ADDR
278#   
279#----------------------------------------------------------------------
280
281CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
282CONFIG_FILENAME = 'config.txt'
283
284
285# Get configFile data
286
287configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
288                                 os.path.basename(CONFIG_FILENAME ))
289if os.path.isfile(  configFile) :
290  pwFilePtr = open(configFile ,'r')
291  d = pwFilePtr.readlines()
292  # do pwFilePtr.read() to get a string object
293  # we have a list object
294  print d[0]
295  print d[1]
296  # make a dictionary
297  config_dic = {}
298
299  #smtppass  = pwFilePtr.read().strip()
300  pwFilePtr.close()
301else :
302  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
303  #print cmdMsgs
304  sys.exit( 1)
305sys.exit( 0)
306
307# END KIPP
Note: See TracBrowser for help on using the repository browser.