source: branches/testScripts/NBbuildConfig.py @ 786

Last change on this file since 786 was 786, checked in by jpfasano, 12 years ago

Added some more control on when the configurations run

  • Property svn:eol-style set to native
File size: 18.4 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10import re
11import shutil
12
13import NBlogMessages
14import NBemail
15import NBosCommand
16import NBsvnCommand
17import NBcheckResult
18
19#---------------------------------------------------------------------
20# Keep history so same project is not repeatedly getting code from
21# subversion repository.
22#---------------------------------------------------------------------
23SVN_HISTORY = {}
24THIRD_PARTY_HISTORY = []
25
26
27#---------------------------------------------------------------------
28# cleanUpName:
29# File and directory names are generated and may contain
30# undesireable characters.
31# Remove these characters from the name
32#---------------------------------------------------------------------
33def cleanUpName(messedUpName) :
34  cleanedUpName=messedUpName
35  cleanedUpName=cleanedUpName.replace('-','')
36  cleanedUpName=cleanedUpName.replace('/','-')
37  cleanedUpName=cleanedUpName.replace('\\','-')
38  cleanedUpName=cleanedUpName.replace(' ','')
39  cleanedUpName=cleanedUpName.replace('"','')
40  cleanedUpName=cleanedUpName.replace("'",'')
41  cleanedUpName=cleanedUpName.replace("=",'-')
42  cleanedUpName=cleanedUpName.replace(":",'')
43  cleanedUpName=cleanedUpName.replace('--enable','')
44  return cleanedUpName
45
46
47#---------------------------------------------------------------------
48# writeResults:
49# After running a command write stdout and stderr to a file
50#---------------------------------------------------------------------
51def writeResults(result,filenameSuffix) :
52  cleanedUpSuffix=cleanUpName(filenameSuffix)
53  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
54  stdoutfile.write(result['stdout'])
55  stdoutfile.close()
56  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
57  stderrfile.write(result['stderr'])
58  stderrfile.close()
59 
60#------------------------------------------------------------------------
61#  Given a configuration, build and test it.
62#
63#  configuration['project']= name of project.
64#   examples: "Clp", "Ipopt"
65#
66#  configuration['rootDir']= root directory of nightlyBuild.
67#   This is where the project will be checked out from svn, and
68#   where the code will be compiled.  This directory must already
69#   exist.  If the testing requires, it needs to contain Netlib & miplib3
70#
71#  configuration['svnVersion']= svn version to be built.
72#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
73#
74#  configuration['buildMethod']= Defines method for building code.
75#  Choices are:
76#    msSoln: use microsoft compiler with a solution (sln) file.
77#    unixConfig: use sequence "./configure", "make", "make test"
78#
79#  configuration['noThirdParty']=True or False (optional). If False
80#   then 3rd party code will be used. If not specified then 3rd part
81#   code will be skipped.
82#   Only used if configuration['buildMethod']=='unixConfig'
83#
84#  configuration['configOptions']: Parameters to be passed to configure.
85#   The -C option and the options for skipping 3rd party code do not
86#   need to be specified.  These will be generated by this function.
87#   There are two types of configOptions to be specified.
88#   Only used if configuration['buildMethod']=='unixConfig'
89#  configuration['configOptions']['unique']= These are options that
90#   distinguish different build configurations.  These options are used
91#   to generate the vpath directory name where the code will be built.
92#   Examples are: "", "--enable-debug" "--enable-parrallel"
93#  configuration['configOptions']['invariant']= These are options that
94#   that are the same for every build configuration so they don't need
95#   to be part of the vpath directory name.
96#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
97
98#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
99#    from the build.
100#    examples: "Ipopt", "Ipopt DyLP"
101#
102#  configuration['slnFile']= path and name of solution file if it is not
103#    in the standard location.
104#    Only used if configuration['buildMethod']=='unixConfig'
105#
106#  configuration['test']=vector of triples indicating tests that
107#    are to be run after building the project. Each triple consists
108#    of:
109#    'dir': directory where test command is to be issued.
110#    'cmd': command to be run with any parameters.
111#    'check': vector of functions to be called which will check the
112#           results from running 'cmd' to determine if an error occurred
113#------------------------------------------------------------------------
114def run(configuration) :
115  NBlogMessages.clearMessages()
116  NBlogMessages.writeMessage( configuration['project'] )
117
118  # Create svn checkout target directory name
119  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
120
121  #---------------------------------------------------------------------
122  # Create names of directory where source is located and
123  # and were object, libs and executables are located (build directory)
124  # To compute build directory, the ./configure options need to be
125  # generated.
126  #---------------------------------------------------------------------
127  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
128  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
129
130  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
131                 configuration['project']+'/'+\
132                 configuration['svnVersion']
133
134  buildDir=svnVersionFlattened
135
136  if configuration['buildMethod']=='unixConfig' :
137    buildDir+=configuration['configOptions']['unique']
138    if 'SkipProjects' in configuration :
139      buildDir+="No"+configuration['SkipProjects']
140    if 'noThirdParty' in configuration : 
141      if configuration['noThirdParty'] :
142        buildDir+='-NoThirdParty'
143    buildDir=cleanUpName(buildDir)
144    if buildDir==svnVersionFlattened : buildDir+='-default'
145
146  fullBuildDir = os.path.join(projectBaseDir,buildDir)
147
148  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
149  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
150  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
151 
152  #for a list of commands that have been executed
153  commandHistory = []
154
155  #---------------------------------------------------------------------
156  # Completely remove a previous build if the user indicates this
157  #---------------------------------------------------------------------
158  if configuration['clear previous build'] and os.path.isdir(fullBuildDir) :
159    NBlogMessages.writeMessage('  Remove previous build in directory '+fullBuildDir)
160    try:
161      shutil.rmtree(fullBuildDir)
162    except shutils.Error :
163      NBlogMessages.writeMessage('  Warning: removal of directory '+fullBuildDir+' failed.')
164
165  #---------------------------------------------------------------------
166  # If nothing has changed and the prior run tested OK or there is
167  # a known problem being worked on, then there
168  # is no need to do anything.
169  #---------------------------------------------------------------------
170  if os.path.isdir(fullBuildDir) :
171    os.chdir(fullBuildDir)
172    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
173
174    if os.path.isfile('NBallTestsPassed') :
175      prevRunSuccess=True
176      NBlogMessages.writeMessage('  In prior build all test passed')
177    else:
178      prevRunSuccess=False
179      NBlogMessages.writeMessage('  No record of all tests having passed')
180
181    # If new source in svn, then must remove "allTestsPassed" files
182    if msg:
183      # SVN has changed since last run
184      NBlogMessages.writeMessage('  '+msg)
185      # Must remove file NBallTestsPassed from all build directories that
186      # use projectCheckoutDir for their source code. This is to ensure
187      # that make will be run in all the build dirs after "svn update"
188      dirs = os.listdir("..")
189      for d in dirs :
190        if d.startswith(svnVersionFlattened) :
191          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
192          if os.path.isfile(fileToBeRemoved) :
193            os.remove(fileToBeRemoved)
194            NBlogMessages.writeMessage('  Removing all test passed record from directory: '+d)
195    else :
196      # SVN has not changed since last run
197      if configuration['Run']=='noSuccessOrAfterChange':
198        if prevRunSuccess :
199          NBlogMessages.writeMessage('  No changes since previous successful run')
200          return
201        else: 
202          NBlogMessages.writeMessage('  Rerunning. No changes, but no record of successful run')
203      elif configuration['Run']=='afterChange' :
204        NBlogMessages.writeMessage('  No changes since previous run')
205        return
206      else:
207        NBlogMessages.writeMessage('  No changes but run always selected')
208  else :
209    NBlogMessages.writeMessage('  Targets have not yet been built')
210
211
212  #---------------------------------------------------------------------
213  # svn checkout or update the project
214  #---------------------------------------------------------------------
215  # Don't get source from subversion if previously done
216  if not SVN_HISTORY.has_key(projectCheckOutDir) :
217    if not os.path.isdir(projectBaseDir) :
218      os.makedirs(projectBaseDir)
219    if not os.path.isdir(projectCheckOutDir) :
220      svnCmd='svn ' +\
221           'checkout ' +\
222           svnCheckOutUrl +\
223           ' '+svnVersionFlattened
224      commandHistory+=[ svnCmd ]
225      svnResult=NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])
226      if svnResult['returnCode'] != 0 :
227        return
228      runConfigure = True
229    else :
230      svnCmd='svn update'
231      commandHistory+=[ svnCmd ]
232      svnResult=NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])
233      if svnResult['returnCode'] != 0 :
234        return
235      #check whether a *.in or configure file was updated
236      r=r'(\S+\.in\s)|(configure\s)'
237      findResult=re.findall(r,svnResult['stdout'])
238      if len(findResult)!=0:
239        runConfigure = True
240      else :
241        runConfigure = False
242
243    SVN_HISTORY[projectCheckOutDir]=runConfigure
244  else :
245    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
246    runConfigure=SVN_HISTORY[projectCheckOutDir]
247
248  #---------------------------------------------------------------------
249  # If there are third party apps, then get these apps
250  #---------------------------------------------------------------------
251  if 'noThirdParty' in configuration :
252    if not configuration['noThirdParty'] :
253      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
254      if os.path.isdir(thirdPartyBaseDir) :
255        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
256          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
257          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
258          for d in thirdPartyDirs :
259            if d=='.svn' : continue
260            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
261            install3rdPartyCmd=os.path.join(".","get."+d)
262            os.chdir(thirdPartyDir)
263            # If the install command has been updated since the last
264            # install, then do a new install
265            if os.path.isfile('NBinstalldone') :
266              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
267                os.remove('NBinstalldone')
268            if not os.path.isfile('NBinstalldone') :
269              if os.path.isfile(install3rdPartyCmd) :
270                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
271                commandHistory+=[ install3rdPartyCmd ]
272                installReturn = NBosCommand.run(install3rdPartyCmd)
273                if installReturn['returnCode'] :
274                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
275                else :
276                  f=open('NBinstalldone','w')
277                  f.close()
278                writeResults(installReturn,install3rdPartyCmd)
279            else :
280              NBlogMessages.writeMessage('  skipped a new download of '+d)
281        else :
282          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
283
284   
285  #---------------------------------------------------------------------
286  # Create the build directory if it doesn't exist
287  #---------------------------------------------------------------------
288  if not os.path.isdir(fullBuildDir) : 
289    os.makedirs(fullBuildDir)
290
291  #---------------------------------------------------------------------
292  # Source is now available, so now it is time to run config
293  #---------------------------------------------------------------------
294  if configuration['buildMethod']=='unixConfig' :
295    skipOptions=''
296
297    if 'SkipProjects' in configuration :
298      skipOptions+=configuration['SkipProjects']
299
300    # If needed create option for skipping 3rd party code
301    needSkip3PartySkipOptions=False
302    if 'noThirdParty' not in configuration : 
303      needSkip3PartySkipOptions=True
304    elif configuration['noThirdParty'] :
305      needSkip3PartySkipOptions=True
306    if needSkip3PartySkipOptions :
307      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
308      if os.path.isdir(thirdPartyBaseDir) :
309        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
310        for d in thirdPartyDirs :
311          if d=='.svn' : continue
312          skipOptions+=' ThirdParty/'+d
313
314    if skipOptions!='' :
315      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
316
317    os.chdir(fullBuildDir)
318    NBlogMessages.writeMessage('  cd '+fullBuildDir)
319
320    # Assemble all config options together and create config command
321    configOptions ="-C "+configuration['configOptions']['unique']
322    configOptions+=configuration['configOptions']['invariant']
323    configOptions+=skipOptions
324    configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
325
326    # If config was previously run, then no need to run again.
327    if (not runConfigure) and NBcheckResult.didConfigRunOK() :
328      NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
329    else :
330      NBlogMessages.writeMessage("  "+configCmd)
331      commandHistory+=[ configCmd ]
332
333      # Finally run config
334      result=NBosCommand.run(configCmd)
335      writeResults(result,'config') 
336
337      # Check if configure worked
338      if result['returnCode'] != 0 :
339        error_msg = result
340        error_msg['configure flags']=configOptions
341        error_msg['svn version']=configuration['svnVersion']
342        error_msg['command history']=commandHistory
343        # Add contents of log file to message
344        logFileName = 'config.log'
345        if os.path.isfile(logFileName) :
346          logFilePtr = open(logFileName,'r')
347          error_msg['config.log'] = logFilePtr.read()
348          logFilePtr.close()
349        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
350        return
351
352    #---------------------------------------------------------------------
353    # Run make part of build
354    #---------------------------------------------------------------------
355    NBlogMessages.writeMessage( '  make' )
356    commandHistory+=[ 'make' ]
357    result=NBosCommand.run('make')
358    writeResults(result,'make') 
359
360    # Check if make worked
361    if result['returnCode'] != 0 :
362      result['configure flags']=configOptions
363      result['svn version']=configuration['svnVersion']
364      result['command history']=commandHistory
365      NBemail.sendCmdMsgs(configuration['project'],result,'make')
366      return
367
368  if configuration['buildMethod']=='msSln' :
369    #---------------------------------------------------------------------
370    # Source is now available, so now it is time to run vcbuild
371    #---------------------------------------------------------------------
372
373    if configuration.has_key('slnDir') :
374      slnFileDir = os.path.join(projectCheckOutDir,configuration['slnDir'])
375    else :
376      slnFileDir = os.path.join(projectCheckOutDir,\
377                          configuration['project'],\
378                          'MSVisualStudio',\
379                          'v8')
380    if not os.path.isdir(slnFileDir) :
381      NBlogMessages.writeMessage("  Solution file directory does not exist: "+slnFileDir)
382      return
383
384    os.chdir(slnFileDir)
385    NBlogMessages.writeMessage('  cd '+slnFileDir)
386
387    if configuration.has_key('slnFile') :
388      slnFileName = configuration['slnFile']
389    else :
390      slnFileName = configuration['project']+'.sln'
391    if not os.path.isfile(slnFileName) :
392      NBlogMessages.writeMessage("  Solution file does not exist '" \
393                                 +slnFileName \
394                                 +"' in directory " \
395                                 +slnFileDir )
396
397      return
398
399    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
400             
401    NBlogMessages.writeMessage("  "+vcbuild)
402    commandHistory+=[ vcbuild ]
403
404    # Finally run vcbuild
405    result=NBosCommand.run(vcbuild)
406    writeResults(result,'vcbuild') 
407
408    # Check if vcbuild worked
409    if result['returnCode'] != 0 :
410        error_msg = result
411        error_msg['svn version']=configuration['svnVersion']
412        error_msg['command history']=commandHistory
413        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
414        return
415
416  #---------------------------------------------------------------------
417  # Run all test executables
418  #---------------------------------------------------------------------
419  if "test" in configuration :
420    for t in range( len(configuration['test']) ) :
421      testRelDir=configuration['test'][t]['dir']
422      testDir = os.path.join(fullBuildDir,testRelDir)
423      testCmd=configuration['test'][t]['cmd']
424      if not os.path.isdir(testDir) :
425        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
426        NBlogMessages.writeMessage('    Intended directory: '+testDir)
427        NBlogMessages.writeMessage('    Intended command: '+testCmd)
428        continue
429      os.chdir(testDir)
430      NBlogMessages.writeMessage('  cd '+testDir)
431
432      NBlogMessages.writeMessage( '  '+testCmd )
433      commandHistory+=[ testCmd ]
434      result=NBosCommand.run(testCmd)
435      writeResults(result,testCmd) 
436       
437      for testFunc in configuration['test'][t]['check'] :
438        testResultFail=testFunc(result,configuration['project'])
439        if testResultFail :
440          result['svn version']=configuration['svnVersion']
441          result['test']=testResultFail
442          result['command history']=commandHistory
443          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
444          return
445
446
447  #---------------------------------------------------------------------
448  # Everything build and all tests passed.
449  #---------------------------------------------------------------------
450  os.chdir(fullBuildDir)
451  f=open('NBallTestsPassed','w')
452  f.close()
Note: See TracBrowser for help on using the repository browser.