source: branches/testScripts/NBbuildConfig.py @ 771

Last change on this file since 771 was 771, checked in by stefan, 13 years ago

run configure only if last configure run was not ok OR svn update updated a configure or a *.in file

  • Property svn:eol-style set to native
File size: 16.8 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10import re
11
12import NBlogMessages
13import NBemail
14import NBosCommand
15import NBsvnCommand
16import NBcheckResult
17
18#---------------------------------------------------------------------
19# Keep history so same project is not repeatedly getting code from
20# subversion repository.
21#---------------------------------------------------------------------
22SVN_HISTORY = {}
23THIRD_PARTY_HISTORY = []
24
25
26#---------------------------------------------------------------------
27# cleanUpName:
28# File and directory names are generated and may contain
29# undesireable characters.
30# Remove these characters from the name
31#---------------------------------------------------------------------
32def cleanUpName(messedUpName) :
33  cleanedUpName=messedUpName
34  cleanedUpName=cleanedUpName.replace('-','')
35  cleanedUpName=cleanedUpName.replace('/','-')
36  cleanedUpName=cleanedUpName.replace('\\','-')
37  cleanedUpName=cleanedUpName.replace(' ','')
38  cleanedUpName=cleanedUpName.replace('"','')
39  cleanedUpName=cleanedUpName.replace("'",'')
40  cleanedUpName=cleanedUpName.replace("=",'-')
41  cleanedUpName=cleanedUpName.replace(":",'')
42  cleanedUpName=cleanedUpName.replace('--enable','')
43  return cleanedUpName
44
45
46#---------------------------------------------------------------------
47# writeResults:
48# After running a command write stdout and stderr to a file
49#---------------------------------------------------------------------
50def writeResults(result,filenameSuffix) :
51  cleanedUpSuffix=cleanUpName(filenameSuffix)
52  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
53  stdoutfile.write(result['stdout'])
54  stdoutfile.close()
55  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
56  stderrfile.write(result['stderr'])
57  stderrfile.close()
58 
59#------------------------------------------------------------------------
60#  Given a configuration, build and test it.
61#
62#  configuration['project']= name of project.
63#   examples: "Clp", "Ipopt"
64#
65#  configuration['rootDir']= root directory of nightlyBuild.
66#   This is where the project will be checked out from svn, and
67#   where the code will be compiled.  This directory must already
68#   exist.  If the testing requires, it needs to contain Netlib & miplib3
69#
70#  configuration['svnVersion']= svn version to be built.
71#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
72#
73#  configuration['buildMethod']= Defines method for building code.
74#  Choices are:
75#    msSoln: use microsoft compiler with a solution (sln) file.
76#    unixConfig: use sequence "./configure", "make", "make test"
77#
78#  configuration['noThirdParty']=True or False (optional). If False
79#   then 3rd party code will be used. If not specified then 3rd part
80#   code will be skipped.
81#   Only used if configuration['buildMethod']=='unixConfig'
82#
83#  configuration['configOptions']: Parameters to be passed to configure.
84#   The -C option and the options for skipping 3rd party code do not
85#   need to be specified.  These will be generated by this function.
86#   There are two types of configOptions to be specified.
87#   Only used if configuration['buildMethod']=='unixConfig'
88#  configuration['configOptions']['unique']= These are options that
89#   distinguish different build configurations.  These options are used
90#   to generate the vpath directory name where the code will be built.
91#   Examples are: "", "--enable-debug" "--enable-parrallel"
92#  configuration['configOptions']['invariant']= These are options that
93#   that are the same for every build configuration so they don't need
94#   to be part of the vpath directory name.
95#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
96
97#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
98#    from the build.
99#    examples: "Ipopt", "Ipopt DyLP"
100#
101#  configuration['slnFile']= path and name of solution file if it is not
102#    in the standard location.
103#    Only used if configuration['buildMethod']=='unixConfig'
104#
105#  configuration['test']=vector of triples indicating tests that
106#    are to be run after building the project. Each triple consists
107#    of:
108#    'dir': directory where test command is to be issued.
109#    'cmd': command to be run with any parameters.
110#    'check': vector of functions to be called which will check the
111#           results from running 'cmd' to determine if an error occurred
112#------------------------------------------------------------------------
113def run(configuration) :
114  NBlogMessages.clearMessages()
115  NBlogMessages.writeMessage( configuration['project'] )
116
117  # Create svn checkout target directory name
118  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
119
120  #---------------------------------------------------------------------
121  # Create names of directory where source is located and
122  # and were object, libs and executables are located (build directory)
123  # To compute build directory, the ./configure options need to be
124  # generated.
125  #---------------------------------------------------------------------
126  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
127  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
128
129  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
130                 configuration['project']+'/'+\
131                 configuration['svnVersion']
132
133  buildDir=svnVersionFlattened
134
135  if configuration['buildMethod']=='unixConfig' :
136    buildDir+=configuration['configOptions']['unique']
137    if 'SkipProjects' in configuration :
138      buildDir+="No"+configuration['SkipProjects']
139    if 'noThirdParty' in configuration : 
140      if configuration['noThirdParty'] :
141        buildDir+='-NoThirdParty'
142    buildDir=cleanUpName(buildDir)
143    if buildDir==svnVersionFlattened : buildDir+='-default'
144
145  fullBuildDir = os.path.join(projectBaseDir,buildDir)
146
147  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
148  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
149  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
150 
151  #for a list of commands that have been executed
152  commandHistory = []
153
154  #---------------------------------------------------------------------
155  # If nothing has changed and the prior run tested OK, then there
156  # is no need to do anything.
157  #---------------------------------------------------------------------
158  if os.path.isdir(fullBuildDir) :
159    os.chdir(fullBuildDir)
160    if os.path.isfile('NBallTestsPassed') : 
161      msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
162      if not msg:
163        # Previous run ran fine, and nothing has changed.
164        NBlogMessages.writeMessage('  No changes since previous successfull run')
165        return
166      NBlogMessages.writeMessage('  '+msg)
167     
168      # Must remove file NBallTestsPassed from all build directories that
169      # use projectCheckoutDir for their source code. This is to ensure
170      # that make will be run in all the build dirs after "svn update"
171      dirs = os.listdir("..")
172      for d in dirs :
173        if d.startswith(configuration['svnVersion']) :
174          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
175          if os.path.isfile(fileToBeRemoved) :
176            os.remove(fileToBeRemoved)
177    else :
178      NBlogMessages.writeMessage('  No record of all tests having passed')
179  else :
180    NBlogMessages.writeMessage('  Targets have not yet been built')
181
182
183  #---------------------------------------------------------------------
184  # svn checkout or update the project
185  #---------------------------------------------------------------------
186  # Don't get source from subversion if previously done
187  if not SVN_HISTORY.has_key(projectCheckOutDir) :
188    if not os.path.isdir(projectBaseDir) :
189      os.makedirs(projectBaseDir)
190    if not os.path.isdir(projectCheckOutDir) :
191      svnCmd='svn ' +\
192           'checkout ' +\
193           svnCheckOutUrl +\
194           ' '+svnVersionFlattened
195      commandHistory+=[ svnCmd ]
196      svnResult=NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])
197      if svnResult['returnCode'] != 0 :
198        return
199      runConfigure = True
200    else :
201      svnCmd='svn update'
202      commandHistory+=[ svnCmd ]
203      svnResult=NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])
204      if svnResult['returnCode'] != 0 :
205        return
206      #check whether a *.in or configure file was updated
207      r=r'(\S+\.in\s)|(configure\s)'
208      findResult=re.findall(r,svnResult['stdout'])
209      if len(findResult)!=0:
210        runConfigure = True
211      else :
212        runConfigure = False
213
214    SVN_HISTORY[projectCheckOutDir]=runConfigure
215  else :
216    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
217    runConfigure=SVN_HISTORY[projectCheckOutDir]
218
219  #---------------------------------------------------------------------
220  # If there are third party apps, then get these apps
221  #---------------------------------------------------------------------
222  if 'noThirdParty' in configuration :
223    if not configuration['noThirdParty'] :
224      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
225      if os.path.isdir(thirdPartyBaseDir) :
226        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
227          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
228          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
229          for d in thirdPartyDirs :
230            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
231            install3rdPartyCmd=os.path.join(".","get."+d)
232            os.chdir(thirdPartyDir)
233            # If the install command has been updated since the last
234            # install, then do a new install
235            if os.path.isfile('NBinstalldone') :
236              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
237                os.remove('NBinstalldone')
238            if not os.path.isfile('NBinstalldone') :
239              if os.path.isfile(install3rdPartyCmd) :
240                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
241                commandHistory+=[ install3rdPartyCmd ]
242                installReturn = NBosCommand.run(install3rdPartyCmd)
243                if installReturn['returnCode'] :
244                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
245                else :
246                  f=open('NBinstalldone','w')
247                  f.close()
248                writeResults(installReturn,install3rdPartyCmd)
249            else :
250              NBlogMessages.writeMessage('  skipped a new download of '+d)
251        else :
252          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
253
254  #---------------------------------------------------------------------
255  # Create the build directory if it doesn't exist
256  # and remove file that indicates prior run tested ok.
257  #---------------------------------------------------------------------
258  if not os.path.isdir(fullBuildDir) : 
259    os.makedirs(fullBuildDir)
260
261  #---------------------------------------------------------------------
262  # Source is now available, so now it is time to run config
263  #---------------------------------------------------------------------
264  if configuration['buildMethod']=='unixConfig' :
265    skipOptions=''
266
267    if 'SkipProjects' in configuration :
268      skipOptions+=configuration['SkipProjects']
269
270    # If needed create option for skipping 3rd party code
271    needSkip3PartySkipOptions=False
272    if 'noThirdParty' not in configuration : 
273      needSkip3PartySkipOptions=True
274    elif configuration['noThirdParty'] :
275      needSkip3PartySkipOptions=True
276    if needSkip3PartySkipOptions :
277      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
278      if os.path.isdir(thirdPartyBaseDir) :
279        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
280        for d in thirdPartyDirs :
281          skipOptions+=' ThirdParty/'+d
282
283    if skipOptions!='' :
284      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
285
286    os.chdir(fullBuildDir)
287    NBlogMessages.writeMessage('  cd '+fullBuildDir)
288
289    # Assemble all config options together and create config command
290    configOptions ="-C "+configuration['configOptions']['unique']
291    configOptions+=configuration['configOptions']['invariant']
292    configOptions+=skipOptions
293    configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
294
295    # If config was previously run, then no need to run again.
296    if (not runConfigure) and NBcheckResult.didConfigRunOK() :
297      NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
298    else :
299      NBlogMessages.writeMessage("  "+configCmd)
300      commandHistory+=[ configCmd ]
301
302      # Finally run config
303      result=NBosCommand.run(configCmd)
304      writeResults(result,'config') 
305
306      # Check if configure worked
307      if result['returnCode'] != 0 :
308        error_msg = result
309        error_msg['configure flags']=configOptions
310        error_msg['svn version']=configuration['svnVersion']
311        error_msg['command history']=commandHistory
312        # Add contents of log file to message
313        logFileName = 'config.log'
314        if os.path.isfile(logFileName) :
315          logFilePtr = open(logFileName,'r')
316          error_msg['config.log'] = logFilePtr.read()
317          logFilePtr.close()
318        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
319        return
320
321    #---------------------------------------------------------------------
322    # Run make part of build
323    #---------------------------------------------------------------------
324    NBlogMessages.writeMessage( '  make' )
325    commandHistory+=[ 'make' ]
326    result=NBosCommand.run('make')
327    writeResults(result,'make') 
328
329    # Check if make worked
330    if result['returnCode'] != 0 :
331      result['configure flags']=configOptions
332      result['svn version']=configuration['svnVersion']
333      result['command history']=commandHistory
334      NBemail.sendCmdMsgs(configuration['project'],result,'make')
335      return
336
337  if configuration['buildMethod']=='msSln' :
338    #---------------------------------------------------------------------
339    # Source is now available, so now it is time to run vcbuild
340    #---------------------------------------------------------------------
341   
342    os.chdir(projectCheckOutDir)
343    NBlogMessages.writeMessage('  cd '+projectCheckOutDir)
344
345    if configuration.has_key('slnFile') :
346      slnFileName = os.path.join(projectCheckOutDir,configuration['slnFile'])
347    else :
348      slnFileName = os.path.join(projectCheckOutDir,\
349                          configuration['project'],\
350                          'MSVisualStudio',\
351                          'v8',\
352                          configuration['project']) +\
353                    '.sln'
354    if not os.path.isfile(slnFileName) :
355      NBlogMessages.writeMessage("  Solution file does not exist: "+slnFileName)
356      return
357
358    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
359             
360
361    NBlogMessages.writeMessage("  "+vcbuild)
362    commandHistory+=[ vcbuild ]
363
364    # Finally run vcbuild
365    result=NBosCommand.run(vcbuild)
366    writeResults(result,'vcbuild') 
367
368    # Check if vcbuild worked
369    if result['returnCode'] != 0 :
370        error_msg = result
371        error_msg['svn version']=configuration['svnVersion']
372        error_msg['command history']=commandHistory
373        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
374        return
375
376  #---------------------------------------------------------------------
377  # Run all test executables
378  #---------------------------------------------------------------------
379  if "test" in configuration :
380    for t in range( len(configuration['test']) ) :
381      testRelDir=configuration['test'][t]['dir']
382      testDir = os.path.join(fullBuildDir,testRelDir)
383      testCmd=configuration['test'][t]['cmd']
384      if not os.path.isdir(testDir) :
385        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
386        NBlogMessages.writeMessage('    Intended directory: '+testDir)
387        NBlogMessages.writeMessage('    Intended command: '+testCmd)
388        continue
389      os.chdir(testDir)
390      NBlogMessages.writeMessage('  cd '+testDir)
391
392      NBlogMessages.writeMessage( '  '+testCmd )
393      commandHistory+=[ testCmd ]
394      result=NBosCommand.run(testCmd)
395      writeResults(result,testCmd) 
396       
397      for testFunc in configuration['test'][t]['check'] :
398        testResultFail=testFunc(result,configuration['project'])
399        if testResultFail :
400          result['svn version']=configuration['svnVersion']
401          result['test']=testResultFail
402          result['command history']=commandHistory
403          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
404          return
405
406
407  #---------------------------------------------------------------------
408  # Everything build and all tests passed.
409  #---------------------------------------------------------------------
410  os.chdir(fullBuildDir)
411  f=open('NBallTestsPassed','w')
412  f.close()
Note: See TracBrowser for help on using the repository browser.