source: branches/testScripts/NBbuildConfig.py @ 796

Last change on this file since 796 was 796, checked in by kmartin, 14 years ago

change shutils to shutil

  • Property svn:eol-style set to native
File size: 21.1 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10import re
11import shutil
12
13import NBlogMessages
14import NBemail
15import NBosCommand
16import NBsvnCommand
17import NBcheckResult
18
19#---------------------------------------------------------------------
20# Keep history so same project is not repeatedly getting code from
21# subversion repository.
22#---------------------------------------------------------------------
23SVN_HISTORY = {}
24THIRD_PARTY_HISTORY = []
25
26
27#---------------------------------------------------------------------
28# cleanUpName:
29# File and directory names are generated and may contain
30# undesireable characters.
31# Remove these characters from the name
32#---------------------------------------------------------------------
33def cleanUpName(messedUpName) :
34  cleanedUpName=messedUpName
35  cleanedUpName=cleanedUpName.replace('-','')
36  cleanedUpName=cleanedUpName.replace('/','-')
37  cleanedUpName=cleanedUpName.replace('\\','-')
38  cleanedUpName=cleanedUpName.replace(' ','')
39  cleanedUpName=cleanedUpName.replace('"','')
40  cleanedUpName=cleanedUpName.replace("'",'')
41  cleanedUpName=cleanedUpName.replace("=",'-')
42  cleanedUpName=cleanedUpName.replace(":",'')
43  cleanedUpName=cleanedUpName.replace('--enable','')
44  return cleanedUpName
45
46
47#---------------------------------------------------------------------
48# writeResults:
49# After running a command write stdout and stderr to a file
50#---------------------------------------------------------------------
51def writeResults(result,filenameSuffix) :
52  cleanedUpSuffix=cleanUpName(filenameSuffix)
53  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
54  stdoutfile.write(result['stdout'])
55  stdoutfile.close()
56  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
57  stderrfile.write(result['stderr'])
58  stderrfile.close()
59 
60#------------------------------------------------------------------------
61#  Given a configuration, build and test it.
62#
63#  configuration['project']= name of project.
64#   examples: "Clp", "Ipopt"
65#
66#  configuration['rootDir']= root directory of nightlyBuild.
67#   This is where the project will be checked out from svn, and
68#   where the code will be compiled.  This directory must already
69#   exist.  If the testing requires, it needs to contain Netlib & miplib3
70#
71#  configuration['svnVersion']= svn version to be built.
72#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
73#
74#  configuration['buildMethod']= Defines method for building code.
75#  Choices are:
76#    msSoln: use microsoft compiler with a solution (sln) file.
77#    unixConfig: use sequence "./configure", "make", "make test"
78#
79#  configuration['noThirdParty']=True or False (optional). If False
80#   then 3rd party code will be used. If not specified then 3rd part
81#   code will be skipped.
82#   Only used if configuration['buildMethod']=='unixConfig'
83#
84#  configuration['configOptions']: Parameters to be passed to configure.
85#   The -C option and the options for skipping 3rd party code do not
86#   need to be specified.  These will be generated by this function.
87#   There are two types of configOptions to be specified.
88#   Only used if configuration['buildMethod']=='unixConfig'
89#  configuration['configOptions']['unique']= These are options that
90#   distinguish different build configurations.  These options are used
91#   to generate the vpath directory name where the code will be built.
92#   Examples are: "", "--enable-debug" "--enable-parrallel"
93#  configuration['configOptions']['invariant']= These are options that
94#   that are the same for every build configuration so they don't need
95#   to be part of the vpath directory name.
96#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
97
98#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
99#    from the build.
100#    examples: "Ipopt", "Ipopt DyLP"
101#
102#  configuration['slnFile']= path and name of solution file if it is not
103#    in the standard location.
104#    Only used if configuration['buildMethod']=='unixConfig'
105#
106#  configuration['test']=vector of triples indicating tests that
107#    are to be run after building the project. Each triple consists
108#    of:
109#    'dir': directory where test command is to be issued.
110#    'cmd': command to be run with any parameters.
111#    'check': vector of functions to be called which will check the
112#           results from running 'cmd' to determine if an error occurred
113#------------------------------------------------------------------------
114def run(configuration) :
115  NBlogMessages.clearMessages()
116  NBlogMessages.writeMessage( configuration['project'] )
117
118  # Create svn checkout target directory name
119  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
120
121  #---------------------------------------------------------------------
122  # Create names of directory where source is located and
123  # and were object, libs and executables are located (build directory)
124  # To compute build directory, the ./configure options need to be
125  # generated.
126  #---------------------------------------------------------------------
127  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
128  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
129
130  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
131                 configuration['project']+'/'+\
132                 configuration['svnVersion']
133
134  buildDir=svnVersionFlattened
135
136  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw' :
137    buildDir+=configuration['configOptions']['unique']
138    if 'SkipProjects' in configuration :
139      buildDir+="No"+configuration['SkipProjects']
140    if 'noThirdParty' in configuration : 
141      if configuration['noThirdParty'] :
142        buildDir+='-NoThirdParty'
143    buildDir=cleanUpName(buildDir)
144    if buildDir==svnVersionFlattened : buildDir+='-default'
145
146  fullBuildDir = os.path.join(projectBaseDir,buildDir)
147
148  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
149  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
150  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
151 
152  #for a list of commands that have been executed
153  commandHistory = []
154
155  #---------------------------------------------------------------------
156  # Completely remove a previous build if the user indicates this
157  #---------------------------------------------------------------------
158  if configuration['clear previous build'] and os.path.isdir(fullBuildDir) :
159    NBlogMessages.writeMessage('  Remove previous build in directory '+fullBuildDir)
160    try:
161      shutil.rmtree(fullBuildDir)
162    except shutil.Error :
163      NBlogMessages.writeMessage('  Warning: removal of directory '+fullBuildDir+' failed.')
164
165  #---------------------------------------------------------------------
166  # If nothing has changed and the prior run tested OK or there is
167  # a known problem being worked on, then there
168  # is no need to do anything.
169  #---------------------------------------------------------------------
170  if os.path.isdir(fullBuildDir) :
171    os.chdir(fullBuildDir)
172    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
173
174    if os.path.isfile('NBallTestsPassed') :
175      prevRunSuccess=True
176      NBlogMessages.writeMessage('  In prior build all test passed')
177    else:
178      prevRunSuccess=False
179      NBlogMessages.writeMessage('  No record of all tests having passed')
180
181    # If new source in svn, then must remove "allTestsPassed" files
182    if msg:
183      # SVN has changed since last run
184      NBlogMessages.writeMessage('  '+msg)
185      # Must remove file NBallTestsPassed from all build directories that
186      # use projectCheckoutDir for their source code. This is to ensure
187      # that make will be run in all the build dirs after "svn update"
188      dirs = os.listdir("..")
189      for d in dirs :
190        if d.startswith(svnVersionFlattened) :
191          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
192          if os.path.isfile(fileToBeRemoved) :
193            os.remove(fileToBeRemoved)
194            NBlogMessages.writeMessage('  Removing all test passed record from directory: '+d)
195    else :
196      # SVN has not changed since last run
197      if configuration['Run']=='noSuccessOrAfterChange':
198        if prevRunSuccess :
199          NBlogMessages.writeMessage('  No changes since previous successful run')
200          return
201        else: 
202          NBlogMessages.writeMessage('  Rerunning. No changes, but no record of successful run')
203      elif configuration['Run']=='afterChange' :
204        NBlogMessages.writeMessage('  No changes since previous run')
205        return
206      else:
207        NBlogMessages.writeMessage('  No changes but run always selected')
208  else :
209    NBlogMessages.writeMessage('  Targets have not yet been built')
210
211
212  #---------------------------------------------------------------------
213  # svn checkout or update the project
214  #---------------------------------------------------------------------
215  # Don't get source from subversion if previously done
216  if not SVN_HISTORY.has_key(projectCheckOutDir) :
217    if not os.path.isdir(projectBaseDir) :
218      os.makedirs(projectBaseDir)
219    if not os.path.isdir(projectCheckOutDir) :
220      svnCmd='svn ' +\
221           'checkout ' +\
222           svnCheckOutUrl +\
223           ' '+svnVersionFlattened
224      commandHistory+=[ svnCmd ]
225      svnResult=NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])
226      if svnResult['returnCode'] != 0 :
227        return
228      runConfigure = True
229    else :
230      svnCmd='svn update'
231      commandHistory+=[ svnCmd ]
232      svnResult=NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])
233      if svnResult['returnCode'] != 0 :
234        return
235      #check whether a *.in or configure file was updated
236      r=r'(\S+\.in\s)|(configure\s)'
237      findResult=re.findall(r,svnResult['stdout'])
238      if len(findResult)!=0:
239        runConfigure = True
240      else :
241        runConfigure = False
242
243    SVN_HISTORY[projectCheckOutDir]=runConfigure
244  else :
245    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
246    runConfigure=SVN_HISTORY[projectCheckOutDir]
247
248  #---------------------------------------------------------------------
249  # If there are third party apps, then get these apps
250  #---------------------------------------------------------------------
251  if 'noThirdParty' in configuration :
252    if not configuration['noThirdParty'] :
253      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
254      if os.path.isdir(thirdPartyBaseDir) :
255        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
256          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
257          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
258          #clean up: take care of .svn, .OLD, and non-directory entries
259          for d in thirdPartyDirs[:] :
260            if d=='.svn' : 
261              thirdPartyDirs.remove(d)
262              continue
263            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
264            if not os.path.isdir(thirdPartyDir) :
265              thirdPartyDirs.remove(d)
266              continue
267            if d.endswith('.OLD') :
268              NBlogMessages.writeMessage('  removing '+d)
269              shutil.rmtree(thirdPartyDir)
270              fileToBeRemoved=os.path.join(thirdPartyDir[0:-4], 'NBinstalldone')
271              if os.path.isfile(fileToBeRemoved) :
272                os.remove(fileToBeRemoved)
273              thirdPartyDirs.remove(d)
274              continue
275          for d in thirdPartyDirs :
276            if d=='.svn' : continue
277            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
278            if not os.path.isdir(thirdPartyDir) : continue
279            install3rdPartyCmd=os.path.join(".","get."+d)
280            os.chdir(thirdPartyDir)
281            # If the install command has been updated since the last
282            # install, then do a new install
283            if os.path.isfile('NBinstalldone') :
284              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
285                os.remove('NBinstalldone')
286            if not os.path.isfile('NBinstalldone') :
287              if os.path.isfile(install3rdPartyCmd) :
288                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
289                commandHistory+=[ install3rdPartyCmd ]
290                installReturn = NBosCommand.run(install3rdPartyCmd)
291                if installReturn['returnCode'] :
292                  NBlogMessages.writeMessage('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
293                else :
294                  f=open('NBinstalldone','w')
295                  f.close()
296                writeResults(installReturn,install3rdPartyCmd)
297            else :
298              NBlogMessages.writeMessage('  skipped a new download of '+d)
299        else :
300          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
301
302   
303  #---------------------------------------------------------------------
304  # Create the build directory if it doesn't exist
305  #---------------------------------------------------------------------
306  if not os.path.isdir(fullBuildDir) : 
307    os.makedirs(fullBuildDir)
308
309  #---------------------------------------------------------------------
310  # Source is now available, so now it is time to run config
311  #---------------------------------------------------------------------
312  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw':
313    skipOptions=''
314
315    if 'SkipProjects' in configuration :
316      skipOptions+=configuration['SkipProjects']
317
318    # If needed create option for skipping 3rd party code
319    needSkip3PartySkipOptions=False
320    if 'noThirdParty' not in configuration : 
321      needSkip3PartySkipOptions=True
322    elif configuration['noThirdParty'] :
323      needSkip3PartySkipOptions=True
324    if needSkip3PartySkipOptions :
325      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
326      if os.path.isdir(thirdPartyBaseDir) :
327        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
328        for d in thirdPartyDirs :
329          if d=='.svn' : continue
330          skipOptions+=' ThirdParty/'+d
331
332    if skipOptions!='' :
333      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
334
335    os.chdir(fullBuildDir)
336    NBlogMessages.writeMessage('  cd '+fullBuildDir)
337
338    # Assemble all config options together and create config command
339    configOptions ="-C "+configuration['configOptions']['unique']
340    configOptions+=configuration['configOptions']['invariant']
341    configOptions+=skipOptions
342   
343
344    #start kipp change
345    #
346    if configuration['buildMethod']=='mingw' :
347      configCmd = os.path.join(projectCheckOutDir,"configure ")
348      #what a pain replace("\\", "/") does not work
349      # we must split and then join, ugh
350      pathParts = configCmd.split("\\")
351      sep = '/'
352      configCmd = sep.join(pathParts)
353      configCmd = "sh -c " + "'" + configCmd + configOptions +  "'"
354    else:
355      configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions) 
356    #
357    #end kipp change
358
359    # If config was previously run, then no need to run again.
360    if (not runConfigure) and NBcheckResult.didConfigRunOK() :
361      NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
362    else :
363      NBlogMessages.writeMessage("  "+configCmd)
364      commandHistory+=[ configCmd ]
365
366      # Finally run config
367      result=NBosCommand.run(configCmd)
368      writeResults(result,'config') 
369
370      # Check if configure worked
371      if result['returnCode'] != 0 :
372        error_msg = result
373        error_msg['configure flags']=configOptions
374        error_msg['svn version']=configuration['svnVersion']
375        error_msg['command history']=commandHistory
376        # Add contents of log file to message
377        logFileName = 'config.log'
378        if os.path.isfile(logFileName) :
379          logFilePtr = open(logFileName,'r')
380          error_msg['config.log'] = logFilePtr.read()
381          logFilePtr.close()
382        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
383        return
384
385    #---------------------------------------------------------------------
386    # Run make part of build
387    #---------------------------------------------------------------------
388    NBlogMessages.writeMessage( '  make' )
389    commandHistory+=[ 'make' ]
390   
391    #
392    # start kipp
393    if configuration['buildMethod']=='mingw' :
394      result=NBosCommand.run('sh -c make') 
395    else:
396      result=NBosCommand.run('make')
397     
398    # end kipp
399    #
400    writeResults(result,'make') 
401
402    # Check if make worked
403    if result['returnCode'] != 0 :
404      result['configure flags']=configOptions
405      result['svn version']=configuration['svnVersion']
406      result['command history']=commandHistory
407      NBemail.sendCmdMsgs(configuration['project'],result,'make')
408      return
409
410  if configuration['buildMethod']=='msSln' :
411    #---------------------------------------------------------------------
412    # Source is now available, so now it is time to run vcbuild
413    #---------------------------------------------------------------------
414
415    if configuration.has_key('slnDir') :
416      slnFileDir = os.path.join(projectCheckOutDir,configuration['slnDir'])
417    else :
418      slnFileDir = os.path.join(projectCheckOutDir,\
419                          configuration['project'],\
420                          'MSVisualStudio',\
421                          'v8')
422    if not os.path.isdir(slnFileDir) :
423      NBlogMessages.writeMessage("  Solution file directory does not exist: "+slnFileDir)
424      return
425
426    os.chdir(slnFileDir)
427    NBlogMessages.writeMessage('  cd '+slnFileDir)
428
429    if configuration.has_key('slnFile') :
430      slnFileName = configuration['slnFile']
431    else :
432      slnFileName = configuration['project']+'.sln'
433    if not os.path.isfile(slnFileName) :
434      NBlogMessages.writeMessage("  Solution file does not exist '" \
435                                 +slnFileName \
436                                 +"' in directory " \
437                                 +slnFileDir )
438
439      return
440
441    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
442             
443    NBlogMessages.writeMessage("  "+vcbuild)
444    commandHistory+=[ vcbuild ]
445
446    # Finally run vcbuild
447    result=NBosCommand.run(vcbuild)
448    writeResults(result,'vcbuild') 
449
450    # Check if vcbuild worked
451    if result['returnCode'] != 0 :
452        error_msg = result
453        error_msg['svn version']=configuration['svnVersion']
454        error_msg['command history']=commandHistory
455        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
456        return
457
458  #---------------------------------------------------------------------
459  # Run all test executables
460  #---------------------------------------------------------------------
461  if "test" in configuration :
462    for t in range( len(configuration['test']) ) :
463      testRelDir=configuration['test'][t]['dir']
464      testDir = os.path.join(fullBuildDir,testRelDir)
465      testCmd=configuration['test'][t]['cmd']
466      if not os.path.isdir(testDir) :
467        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
468        NBlogMessages.writeMessage('    Intended directory: '+testDir)
469        NBlogMessages.writeMessage('    Intended command: '+testCmd)
470        continue
471      os.chdir(testDir)
472      NBlogMessages.writeMessage('  cd '+testDir)
473
474      NBlogMessages.writeMessage( '  '+testCmd )
475      commandHistory+=[ testCmd ]
476      result=NBosCommand.run(testCmd)
477      writeResults(result,testCmd)
478
479       
480      for testFunc in configuration['test'][t]['check'] :
481        testResultFail=testFunc(result,configuration['project'])
482        if testResultFail :
483          result['svn version']=configuration['svnVersion']
484          result['test']=testResultFail
485          result['command history']=commandHistory
486          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
487          return
488
489  #---------------------------------------------------------------------
490  # Run all install executables
491  #---------------------------------------------------------------------
492  if "install" in configuration :
493    for t in range( len(configuration['install']) ) :
494      installRelDir=configuration['install'][t]['dir']
495      installDir = os.path.join(fullBuildDir, installRelDir)
496      installCmd=configuration['install'][t]['cmd']
497      if not os.path.isdir( installDir) :
498        NBlogMessages.writeMessage('  Directory to run install from does not exist:')
499        NBlogMessages.writeMessage('    Intended directory: '+installDir)
500        NBlogMessages.writeMessage('    Intended command: '+installCmd)
501        continue
502      os.chdir(installDir)
503      NBlogMessages.writeMessage('  cd '+installDir)
504      NBlogMessages.writeMessage( '  '+installCmd )
505      commandHistory+=[ installCmd ]
506      result=NBosCommand.run(installCmd)
507      writeResults(result,installCmd)
508      if result['returnCode'] != 0 :
509          result['svn version']=configuration['svnVersion']
510          result['install']=installResultFail
511          result['command history']=commandHistory
512          NBemail.sendCmdMsgs(configuration['project'],result,installCmd)
513          return
514
515     
516
517  #---------------------------------------------------------------------
518  # Everything build and all tests passed.
519  #---------------------------------------------------------------------
520  os.chdir(fullBuildDir)
521  f=open('NBallTestsPassed','w')
522  f.close()
Note: See TracBrowser for help on using the repository browser.