source: branches/testScripts/nightlyBuild.py @ 656

Last change on this file since 656 was 654, checked in by stefan, 12 years ago

allow additional usergiven configure flags

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 11.7 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Figure out how to run configure with no third party code (done, or?)
21#   -Remove markerfile in ThirdParty dir if svn update updated the get... script of the ThirdParty project
22
23
24
25
26#------------------------------------------------------------------------
27#  Main Program Starts Here 
28#------------------------------------------------------------------------
29
30#------------------------------------------------------------------------
31#  If needed create the top level directory
32#------------------------------------------------------------------------
33if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
34  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
35os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
36
37#------------------------------------------------------------------------
38#  Get the data directories if they don't already exist
39#------------------------------------------------------------------------
40dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
41if not os.path.isdir(dataBaseDir) :
42  os.makedirs(dataBaseDir)
43dataDirs=['Netlib','miplib3']
44for d in dataDirs :
45  dataDir=os.path.join(dataBaseDir,d)
46  if not os.path.isdir(dataDir) :
47    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
48    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
49      sys.exit(1)
50    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
51netlibDir=os.path.join(dataBaseDir,'Netlib')
52miplib3Dir=os.path.join(dataBaseDir,'miplib3')
53
54#------------------------------------------------------------------------
55# Loop once for each project
56#------------------------------------------------------------------------
57for p in NBuserConfig.PROJECTS:
58  NBlogMessages.writeMessage( p )
59
60  #---------------------------------------------------------------------
61  # Loop once for each version of the project to be checked out.
62  # The supported types are trunk & latestStable
63  #---------------------------------------------------------------------
64  projectVersions=[]
65  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
66    lsv = NBsvnCommand.latestStableVersion(p)
67    projectVersions.append(['stable'+lsv,'stable/'+lsv])
68  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
69    projectVersions.append(['trunk','trunk'])
70  for projectVersion in projectVersions :
71    #---------------------------------------------------------------------
72    # svn checkout or update the project
73    #---------------------------------------------------------------------
74    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
75    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
76    if not os.path.isdir(projectBaseDir) :
77      os.makedirs(projectBaseDir)
78    if not os.path.isdir(projectCheckOutDir) :
79      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
80             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
81      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
82        continue
83    else :
84      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
85      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
86        continue
87
88    #---------------------------------------------------------------------
89    # If there are third party apps, then get these apps
90    #---------------------------------------------------------------------
91    if NBuserConfig.DOWNLOAD_3RD_PARTY :
92      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
93      if os.path.isdir(thirdPartyBaseDir) :
94        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
95        for d in thirdPartyDirs :
96          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
97          install3rdPartyCmd=os.path.join(".","get."+d)
98          os.chdir(thirdPartyDir)
99          if not os.path.isfile('NBinstalldone') :
100            if os.path.isfile(install3rdPartyCmd) :
101              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
102              NBosCommand.run(install3rdPartyCmd)
103              f=open('NBinstalldone','w')
104              f.close()
105          else :
106            NBlogMessages.writeMessage('  skipped anew download of '+d)
107   
108    #---------------------------------------------------------------------
109    # Loop once for each type of build to be done.
110    # Debug, use third party code, ...
111    # vpath and options to configure must be set for the buildType
112    #---------------------------------------------------------------------
113    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
114    for buildType, configLine in buildtypes.iteritems() :
115   
116      #---------------------------------------------------------------------
117      # Setup the directory where the build will be done and the configure
118      # command options
119      #---------------------------------------------------------------------
120      vpathDir=projectVersion[0]+'-'+buildType
121      configOptions='-C '+NBuserConfig.CONFIGURE_FLAGS+' '+configLine
122
123#      if "ThirdParty" in buildType :
124#        vpathDir += "ThirdParty"
125#      else :
126#        vpathDir += "NoThirdParty"
127#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
128#        if os.path.isdir(thirdPartyBaseDir) :
129#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
130#          skipOptions=''
131#          for d in thirdPartyDirs :
132#            skipOptions+=' ThirdParty/'+d
133#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
134     
135      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
136      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
137      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
138      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
139
140      #---------------------------------------------------------------------
141      # Run configure part of build (only if config has not previously
142      # ran successfully).
143      #---------------------------------------------------------------------
144      os.chdir(fullVpathDir)
145      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
146      if NBcheckResult.didConfigRunOK() :
147        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
148      else :
149        NBlogMessages.writeMessage('  '+configCmd)
150        result=NBosCommand.run(configCmd)
151     
152        # Check if configure worked
153        if result['returnCode'] != 0 :
154          error_msg = result
155          # Add contents of log file to message
156          logFileName = 'config.log'
157          if os.path.isfile(logFileName) :
158            logFilePtr = open(logFileName,'r')
159            error_msg['config.log']  = "config.log contains: \n" 
160            error_msg['config.log'] += logFilePtr.read()
161            logFilePtr.close()
162          NBemail.sendCmdMsgs(p,error_msg,configCmd)
163          continue
164
165      #---------------------------------------------------------------------
166      # Run make part of build
167      #---------------------------------------------------------------------
168      NBlogMessages.writeMessage( '  make' )
169      result=NBosCommand.run('make')
170     
171      # Check if make worked
172      if result['returnCode'] != 0 :
173        NBemail.sendCmdMsgs(p,result,'make')
174        continue
175
176      #---------------------------------------------------------------------
177      # Run 'make test' part of build
178      #---------------------------------------------------------------------
179      NBlogMessages.writeMessage( '  make test' )
180      result=NBosCommand.run('make test')
181     
182      # Check if 'make test' worked
183      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
184      if didMakeTestFail :
185        result['make test']=didMakeTestFail
186        NBemail.sendCmdMsgs(p,result,"make test")
187        continue
188
189      #---------------------------------------------------------------------
190      # Run unitTest if available and different from 'make test'
191      #---------------------------------------------------------------------
192      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
193        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
194        os.chdir(unitTestPath)
195
196        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
197        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
198        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
199
200        NBlogMessages.writeMessage( '  '+unitTestCmd )
201        result=NBosCommand.run(unitTestCmd)
202     
203        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
204        if didUnitTestFail :
205          result['unitTest']=didUnitTestFail
206          NBemail.sendCmdMsgs(p,result,unitTestCmd)
207          continue
208
209NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
210
211sys.exit(0)
212
213
214# START KIPP
215#----------------------------------------------------------------------
216# CONFIG FILE PATH:
217#   path to the config file for the build
218#   done. If the directory does not exist, it will be created.
219#   this should have all of the user specific data
220#   it should have values for
221#   NIGHTLY_BUILD_ROOT
222#   NBuserConfig.SMTP_SERVER_NAME
223#   NBuserConfig.SMTP_SERVER_PORT
224#   NBuserConfig.SMTP_SSL_SERVER
225#   NBuserConfig.SMTP_USER_NAME
226#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
227#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
228#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
229#   
230#----------------------------------------------------------------------
231
232CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
233CONFIG_FILENAME = 'config.txt'
234
235
236# Get configFile data
237
238configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
239                                 os.path.basename(CONFIG_FILENAME ))
240if os.path.isfile(  configFile) :
241  pwFilePtr = open(configFile ,'r')
242  d = pwFilePtr.readlines()
243  # do pwFilePtr.read() to get a string object
244  # we have a list object
245  print d[0]
246  print d[1]
247  # make a dictionary
248  config_dic = {}
249
250  #smtppass  = pwFilePtr.read().strip()
251  pwFilePtr.close()
252else :
253  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
254  #print cmdMsgs
255  sys.exit( 1)
256sys.exit( 0)
257
258
259
260# START KIPP
261#----------------------------------------------------------------------
262#   path to the config file for the build
263#   get the user dependent variables
264# CONFIG FILE PATH:
265#   it should have values for
266#   NIGHTLY_BUILD_ROOT
267#   NBuserConfig.SMTP_SERVER_NAME
268#   NBuserConfig.SMTP_SERVER_PORT
269#   NBuserConfig.SMTP_SSL_SERVER
270#   NBuserConfig.SMTP_USER_NAME
271#   NBuserConfig.SMTP_PASSWORD_FILENAME
272#   NBuserConfig.SENDER_EMAIL_ADDR
273#   NBuserConfig.MY_EMAIL_ADDR
274#   
275#----------------------------------------------------------------------
276
277CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
278CONFIG_FILENAME = 'config.txt'
279
280
281# Get configFile data
282
283configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
284                                 os.path.basename(CONFIG_FILENAME ))
285if os.path.isfile(  configFile) :
286  pwFilePtr = open(configFile ,'r')
287  d = pwFilePtr.readlines()
288  # do pwFilePtr.read() to get a string object
289  # we have a list object
290  print d[0]
291  print d[1]
292  # make a dictionary
293  config_dic = {}
294
295  #smtppass  = pwFilePtr.read().strip()
296  pwFilePtr.close()
297else :
298  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
299  #print cmdMsgs
300  sys.exit( 1)
301sys.exit( 0)
302
303# END KIPP
Note: See TracBrowser for help on using the repository browser.