source: branches/testScripts/NBbuildConfig.py @ 820

Last change on this file since 820 was 820, checked in by jpfasano, 12 years ago

modified the way the build directory name is generated.
The way it had been done was causing the code which removes the NBallTestsPassed file to work incorrectly.
Added the writing of the config parms to the config log

  • Property svn:eol-style set to native
File size: 24.1 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10import re
11import shutil
12
13import NBlogMessages
14import NBemail
15import NBosCommand
16import NBsvnCommand
17import NBcheckResult
18
19import NBuserParameters
20
21#---------------------------------------------------------------------
22# Keep history so same project is not repeatedly getting code from
23# subversion repository.
24#---------------------------------------------------------------------
25SVN_HISTORY = {}
26THIRD_PARTY_HISTORY = []
27
28
29#---------------------------------------------------------------------
30# cleanUpName:
31# File and directory names are generated and may contain
32# undesireable characters.
33# Remove these characters from the name
34#---------------------------------------------------------------------
35def cleanUpName(messedUpName) :
36  cleanedUpName=messedUpName
37 
38  # Do not remove "-". This will cause problems when removing NBallTestsPassed.
39  #cleanedUpName=cleanedUpName.replace('-','')
40
41  cleanedUpName=cleanedUpName.replace('/','-')
42  cleanedUpName=cleanedUpName.replace('\\','-')
43  cleanedUpName=cleanedUpName.replace(' ','')
44  cleanedUpName=cleanedUpName.replace('"','')
45  cleanedUpName=cleanedUpName.replace("'",'')
46  cleanedUpName=cleanedUpName.replace("=",'-')
47  cleanedUpName=cleanedUpName.replace(":",'')
48  cleanedUpName=cleanedUpName.replace('--enable','')
49  cleanedUpName=cleanedUpName.replace('--','-')
50  return cleanedUpName
51
52
53#---------------------------------------------------------------------
54# writeResults:
55# After running a command write stdout and stderr to a file
56#---------------------------------------------------------------------
57def writeResults(result,filenameSuffix) :
58  cleanedUpSuffix=cleanUpName(filenameSuffix)
59  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
60  stdoutfile.write(result['stdout'])
61  stdoutfile.close()
62  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
63  stderrfile.write(result['stderr'])
64  stderrfile.close()
65 
66#------------------------------------------------------------------------
67#  Given a configuration, build and test it.
68#
69#  configuration['project']= name of project.
70#   examples: "Clp", "Ipopt"
71#
72#  configuration['rootDir']= root directory of nightlyBuild.
73#   This is where the project will be checked out from svn, and
74#   where the code will be compiled.  This directory must already
75#   exist.  If the testing requires, it needs to contain Netlib & miplib3
76#
77#  configuration['svnVersion']= svn version to be built.
78#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
79#
80#  configuration['buildMethod']= Defines method for building code.
81#  Choices are:
82#    msSoln: use microsoft compiler with a solution (sln) file.
83#    unixConfig: use sequence "./configure", "make", "make test"
84#
85#  configuration['noThirdParty']=True or False (optional). If False
86#   then 3rd party code will be used. If not specified then 3rd part
87#   code will be skipped.
88#   Only used if configuration['buildMethod']=='unixConfig'
89#
90#  configuration['configOptions']: Parameters to be passed to configure.
91#   The -C option and the options for skipping 3rd party code do not
92#   need to be specified.  These will be generated by this function.
93#   There are two types of configOptions to be specified.
94#   Only used if configuration['buildMethod']=='unixConfig'
95#  configuration['configOptions']['unique']= These are options that
96#   distinguish different build configurations.  These options are used
97#   to generate the vpath directory name where the code will be built.
98#   Examples are: "", "--enable-debug" "--enable-parrallel"
99#  configuration['configOptions']['invariant']= These are options that
100#   that are the same for every build configuration so they don't need
101#   to be part of the vpath directory name.
102#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
103
104#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
105#    from the build.
106#    examples: "Ipopt", "Ipopt DyLP"
107#
108#  configuration['slnFile']= path and name of solution file if it is not
109#    in the standard location.
110#    Only used if configuration['buildMethod']=='unixConfig'
111#
112#  configuration['test']=vector of triples indicating tests that
113#    are to be run after building the project. Each triple consists
114#    of:
115#    'dir': directory where test command is to be issued.
116#    'cmd': command to be run with any parameters.
117#    'check': vector of functions to be called which will check the
118#           results from running 'cmd' to determine if an error occurred
119#------------------------------------------------------------------------
120def run(configuration) :
121  NBlogMessages.clearMessages()
122  NBlogMessages.writeMessage( configuration['project'] )
123
124  # Create svn checkout target directory name
125  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
126
127  #---------------------------------------------------------------------
128  # Create names of directory where source is located and
129  # and were object, libs and executables are located (build directory)
130  # To compute build directory, the ./configure options need to be
131  # generated.
132  #---------------------------------------------------------------------
133  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
134  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
135
136  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
137                 configuration['project']+'/'+\
138                 configuration['svnVersion']
139
140  buildDir=svnVersionFlattened
141
142  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw' :
143    buildDir+=configuration['configOptions']['unique']
144    if 'SkipProjects' in configuration :
145      buildDir+="No"+configuration['SkipProjects']
146    if 'noThirdParty' in configuration : 
147      if configuration['noThirdParty'] :
148        buildDir+='-NoThirdParty'
149    buildDir=cleanUpName(buildDir)
150    if buildDir==svnVersionFlattened : buildDir+='-default'
151
152  fullBuildDir = os.path.join(projectBaseDir,buildDir)
153
154  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
155  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
156  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
157 
158  #for a list of commands that have been executed
159  commandHistory = []
160
161  #---------------------------------------------------------------------
162  # Completely remove a previous build if the user indicates this
163  #---------------------------------------------------------------------
164  if configuration['clear previous build'] and os.path.isdir(fullBuildDir) :
165    NBlogMessages.writeMessage('  Remove previous build in directory '+fullBuildDir)
166    try:
167      shutil.rmtree(fullBuildDir)
168    except shutil.Error :
169      NBlogMessages.writeMessage('  Warning: removal of directory '+fullBuildDir+' failed.')
170
171  #---------------------------------------------------------------------
172  # If nothing has changed and the prior run tested OK or there is
173  # a known problem being worked on, then there
174  # is no need to do anything.
175  #---------------------------------------------------------------------
176  if os.path.isdir(fullBuildDir) :
177    os.chdir(fullBuildDir)
178    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
179
180    if os.path.isfile('NBallTestsPassed') :
181      prevRunSuccess=True
182      NBlogMessages.writeMessage('  In prior build all test passed')
183    else:
184      prevRunSuccess=False
185      NBlogMessages.writeMessage('  No record of all tests having passed')
186
187    # If new source in svn, then must remove "allTestsPassed" files
188    if msg:
189      # SVN has changed since last run
190      NBlogMessages.writeMessage('  '+msg)
191      # Must remove file NBallTestsPassed from all build directories that
192      # use projectCheckoutDir for their source code. This is to ensure
193      # that make will be run in all the build dirs after "svn update"
194      dirs = os.listdir("..")
195      for d in dirs :
196        if d.startswith(svnVersionFlattened) :
197          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
198          if os.path.isfile(fileToBeRemoved) :
199            os.remove(fileToBeRemoved)
200            NBlogMessages.writeMessage('  Removing all test passed record from directory: '+d)
201    else :
202      # SVN has not changed since last run
203      if configuration['Run']=='noSuccessOrAfterChange':
204        if prevRunSuccess :
205          NBlogMessages.writeMessage('  No changes since previous successful run')
206          return
207        else: 
208          NBlogMessages.writeMessage('  Rerunning. No changes, but no record of successful run')
209      elif configuration['Run']=='afterChange' :
210        NBlogMessages.writeMessage('  No changes since previous run')
211        return
212      else:
213        NBlogMessages.writeMessage('  No changes but run always selected')
214  else :
215    NBlogMessages.writeMessage('  Targets have not yet been built')
216
217
218  #---------------------------------------------------------------------
219  # svn checkout or update the project
220  #---------------------------------------------------------------------
221  # Don't get source from subversion if previously done
222  if not SVN_HISTORY.has_key(projectCheckOutDir) :
223    if not os.path.isdir(projectBaseDir) :
224      os.makedirs(projectBaseDir)
225    if not os.path.isdir(projectCheckOutDir) :
226      svnCmd='svn ' +\
227           'checkout ' +\
228           svnCheckOutUrl +\
229           ' '+svnVersionFlattened
230      commandHistory+=[ svnCmd ]
231      svnResult=NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])
232      if svnResult['returnCode'] != 0 :
233        return
234      runConfigure = True
235    else :
236      svnCmd='svn update'
237      commandHistory+=[ svnCmd ]
238      svnResult=NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])
239      if svnResult['returnCode'] != 0 :
240        return
241      #check whether a *.in or configure file was updated
242      r=r'(\S+\.in\s)|(configure\s)'
243      findResult=re.findall(r,svnResult['stdout'])
244      if len(findResult)!=0:
245        runConfigure = True
246      else :
247        runConfigure = False
248
249    SVN_HISTORY[projectCheckOutDir]=runConfigure
250  else :
251    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
252    runConfigure=SVN_HISTORY[projectCheckOutDir]
253
254  #---------------------------------------------------------------------
255  # If there are third party apps, then get these apps
256  #---------------------------------------------------------------------
257  if 'noThirdParty' in configuration :
258    if not configuration['noThirdParty'] :
259      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
260      if os.path.isdir(thirdPartyBaseDir) :
261        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
262          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
263          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
264          #clean up: take care of .svn, .OLD, and non-directory entries
265          for d in thirdPartyDirs[:] :
266            if d=='.svn' : 
267              thirdPartyDirs.remove(d)
268              continue
269            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
270            if not os.path.isdir(thirdPartyDir) :
271              thirdPartyDirs.remove(d)
272              continue
273            if d.endswith('.OLD') :
274              NBlogMessages.writeMessage('  removing '+d)
275              shutil.rmtree(thirdPartyDir)
276              fileToBeRemoved=os.path.join(thirdPartyDir[0:-4], 'NBinstalldone')
277              if os.path.isfile(fileToBeRemoved) :
278                os.remove(fileToBeRemoved)
279              thirdPartyDirs.remove(d)
280              continue
281          for d in thirdPartyDirs :
282            if d=='.svn' : continue
283            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
284            if not os.path.isdir(thirdPartyDir) : continue
285            install3rdPartyCmd=os.path.join(".","get."+d)
286            os.chdir(thirdPartyDir)
287            # If the install command has been updated since the last
288            # install, then do a new install
289            if os.path.isfile('NBinstalldone') :
290              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
291                os.remove('NBinstalldone')
292            if not os.path.isfile('NBinstalldone') :
293              if os.path.isfile(install3rdPartyCmd) :
294                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
295                commandHistory+=[ install3rdPartyCmd ]
296                installReturn = NBosCommand.run(install3rdPartyCmd)
297                if installReturn['returnCode'] :
298                  NBlogMessages.writeMessage('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
299                else :
300                  f=open('NBinstalldone','w')
301                  f.close()
302                writeResults(installReturn,install3rdPartyCmd)
303            else :
304              NBlogMessages.writeMessage('  skipped a new download of '+d)
305        else :
306          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
307
308   
309  #---------------------------------------------------------------------
310  # Create the build directory if it doesn't exist
311  #---------------------------------------------------------------------
312  if not os.path.isdir(fullBuildDir) : 
313    os.makedirs(fullBuildDir)
314
315  #---------------------------------------------------------------------
316  # Source is now available, so now it is time to run config
317  #---------------------------------------------------------------------
318  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw':
319    skipOptions=''
320
321    if 'SkipProjects' in configuration :
322      skipOptions+=configuration['SkipProjects']
323
324    # If needed create option for skipping 3rd party code
325    needSkip3PartySkipOptions=False
326    if 'noThirdParty' not in configuration : 
327      needSkip3PartySkipOptions=True
328    elif configuration['noThirdParty'] :
329      needSkip3PartySkipOptions=True
330    if needSkip3PartySkipOptions :
331      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
332      if os.path.isdir(thirdPartyBaseDir) :
333        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
334        for d in thirdPartyDirs :
335          if d=='.svn' : continue
336          skipOptions+=' ThirdParty/'+d
337
338    if skipOptions!='' :
339      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
340
341    os.chdir(fullBuildDir)
342    NBlogMessages.writeMessage('  cd '+fullBuildDir)
343
344    # Assemble all config options together and create config command
345    configOptions ="-C "+configuration['configOptions']['unique']
346    configOptions+=configuration['configOptions']['invariant']
347    configOptions+=skipOptions
348    configOptions=configOptions.replace("  "," ")
349    configOptions=configOptions.replace("  "," ")
350    configOptions=configOptions.replace('=" ','="')
351
352    #start kipp change
353    #
354    if configuration['buildMethod']=='mingw' :
355      configCmd = os.path.join(projectCheckOutDir,"configure ")
356      #what a pain replace("\\", "/") does not work
357      # we must split and then join, ugh
358      pathParts = configCmd.split("\\")
359      sep = '/'
360      configCmd = sep.join(pathParts)
361      configCmd = "sh -c " + "'" + configCmd + configOptions +  "'"
362    else:
363      configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions) 
364    #
365    #end kipp change
366
367    # If config was previously run, then no need to run again.
368    if (not runConfigure) and NBcheckResult.didConfigRunOK() :
369      NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
370    else :
371      NBlogMessages.writeMessage("  "+configCmd)
372      commandHistory+=[ configCmd ]
373
374      # Finally run config
375      result=NBosCommand.run(configCmd)
376      result['stdout']=configOptions+"\n"+result['stdout']
377      writeResults(result,'config') 
378
379      # Check if configure worked
380      if result['returnCode'] != 0 :
381        error_msg = result
382        error_msg['configure flags']=configOptions
383        error_msg['svn version']=configuration['svnVersion']
384        error_msg['command history']=commandHistory
385        # Add contents of log file to message
386        logFileName = 'config.log'
387        if os.path.isfile(logFileName) :
388          logFilePtr = open(logFileName,'r')
389          error_msg['config.log'] = logFilePtr.read()
390          logFilePtr.close()
391        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
392        return
393
394    #---------------------------------------------------------------------
395    # Run make part of build
396    #---------------------------------------------------------------------
397    NBlogMessages.writeMessage( '  make' )
398    commandHistory+=[ 'make' ]
399   
400    #
401    # start kipp
402    if configuration['buildMethod']=='mingw' :
403      result=NBosCommand.run('sh -c make') 
404    else:
405      result=NBosCommand.run('make')
406     
407    # end kipp
408    #
409    writeResults(result,'make') 
410
411    # Check if make worked
412    if result['returnCode'] != 0 :
413      result['configure flags']=configOptions
414      result['svn version']=configuration['svnVersion']
415      result['command history']=commandHistory
416      NBemail.sendCmdMsgs(configuration['project'],result,'make')
417      return
418
419  if configuration['buildMethod']=='msSln' :
420    #---------------------------------------------------------------------
421    # Source is now available, so now it is time to run vcbuild
422    #---------------------------------------------------------------------
423
424    if configuration.has_key('slnDir') :
425      slnFileDir = os.path.join(projectCheckOutDir,configuration['slnDir'])
426    else :
427      slnFileDir = os.path.join(projectCheckOutDir,\
428                          configuration['project'],\
429                          'MSVisualStudio',\
430                          'v8')
431    if not os.path.isdir(slnFileDir) :
432      NBlogMessages.writeMessage("  Solution file directory does not exist: "+slnFileDir)
433      return
434
435    os.chdir(slnFileDir)
436    NBlogMessages.writeMessage('  cd '+slnFileDir)
437
438    if configuration.has_key('slnFile') :
439      slnFileName = configuration['slnFile']
440    else :
441      slnFileName = configuration['project']+'.sln'
442    if not os.path.isfile(slnFileName) :
443      NBlogMessages.writeMessage("  Solution file does not exist '" \
444                                 +slnFileName \
445                                 +"' in directory " \
446                                 +slnFileDir )
447
448      return
449
450    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
451             
452    NBlogMessages.writeMessage("  "+vcbuild)
453    commandHistory+=[ vcbuild ]
454
455    # Finally run vcbuild
456    result=NBosCommand.run(vcbuild)
457    writeResults(result,'vcbuild') 
458
459    # Check if vcbuild worked
460    if result['returnCode'] != 0 :
461        error_msg = result
462        error_msg['svn version']=configuration['svnVersion']
463        error_msg['command history']=commandHistory
464        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
465        return
466
467  #---------------------------------------------------------------------
468  # Run all test executables
469  #---------------------------------------------------------------------
470  if "test" in configuration :
471    for t in range( len(configuration['test']) ) :
472      testRelDir=configuration['test'][t]['dir']
473      testDir = os.path.join(fullBuildDir,testRelDir)
474      testCmd=configuration['test'][t]['cmd']
475      if not os.path.isdir(testDir) :
476        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
477        NBlogMessages.writeMessage('    Intended directory: '+testDir)
478        NBlogMessages.writeMessage('    Intended command: '+testCmd)
479        continue
480      os.chdir(testDir)
481      NBlogMessages.writeMessage('  cd '+testDir)
482
483      NBlogMessages.writeMessage( '  '+testCmd )
484      commandHistory+=[ testCmd ]
485      result=NBosCommand.run(testCmd)
486      writeResults(result,testCmd)
487
488       
489      for testFunc in configuration['test'][t]['check'] :
490        testResultFail=testFunc(result,configuration['project'])
491        if testResultFail :
492          result['svn version']=configuration['svnVersion']
493          result['test']=testResultFail
494          result['command history']=commandHistory
495          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
496          return
497
498  #---------------------------------------------------------------------
499  # Run all install executables
500  # We assume a Unix Installation
501  #---------------------------------------------------------------------
502
503
504  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw':
505    if "install" in configuration :
506      for t in range( len(configuration['install']) ) :
507        installRelDir=configuration['install'][t]['dir']
508        installDir = os.path.join(fullBuildDir, installRelDir)
509        installCmd=configuration['install'][t]['cmd']
510        if not os.path.isdir( installDir) :
511          NBlogMessages.writeMessage('  Directory to run install from does not exist:')
512          NBlogMessages.writeMessage('    Intended directory: '+installDir)
513          NBlogMessages.writeMessage('    Intended command: '+installCmd)
514          continue
515        os.chdir(installDir)
516        NBlogMessages.writeMessage('  cd '+installDir)
517        NBlogMessages.writeMessage( '  '+installCmd )
518        commandHistory+=[ installCmd ]
519        result=NBosCommand.run(installCmd)
520        writeResults(result,installCmd)
521        if result['returnCode'] != 0 :
522            result['svn version']=configuration['svnVersion']
523            # figure out what installResultFail should be
524            #result['install']=installResultFail
525            result['command history']=commandHistory
526            NBemail.sendCmdMsgs(configuration['project'],result,installCmd)
527            return
528
529   
530    #---------------------------------------------------------------------
531    # Build the binary distribution
532    # We assume a Unix distribution
533    #---------------------------------------------------------------------
534      directories = ""   
535      # when doing this right make sure the example, include, and lib directories
536      # are there.
537     
538      # if the lib  directory is there, add it
539      # delete these if already there
540      if os.path.isdir( "lib") == True :
541        directories  += " lib "
542
543      # if the include directory is there, add it
544      if os.path.isdir( "include") == True :
545        directories +=  " include "
546
547      # if the examples directory is there, add it
548      if os.path.isdir( "examples") == True :
549        directories +=  " examples "
550
551      # if the bin directory is there, add it
552      if os.path.isdir( "bin") == True :
553        directories +=  " bin "
554
555      # don't forget to add LICENSE and AUTHORS file
556      # Andreas is going to add a directory where these reside from doing
557      # make install
558
559      #if the binary directory is not there create it
560      binariesDir=os.path.join(projectBaseDir,"binaries")
561
562      if not os.path.isdir( binariesDir ) :
563        os.makedirs( binariesDir )
564
565      #configuration['project']
566      outputDirectory = os.path.join(binariesDir, configuration['project'])
567      print outputDirectory
568
569
570      if not os.path.isdir( outputDirectory) :
571        os.makedirs( outputDirectory)
572
573      # create the output directory
574     
575      # tar it up
576      # buidDir should be name of tar file -- make unique
577      tarCmd = "tar -czvf "
578      #do something better with tar file name
579      svnVersionFlattened = cleanUpName(configuration['svnVersion'])
580      tarFileName =  configuration['project'] + "-" + svnVersionFlattened + "-" + sys.platform + ".tgz"
581      tarCmd += os.path.join(outputDirectory, tarFileName)
582      tarCmd += directories
583
584
585
586 
587      NBlogMessages.writeMessage( '  '+ tarCmd )
588      commandHistory+=[ tarCmd ]
589      result=NBosCommand.run( tarCmd)
590      writeResults(result, tarCmd)
591      if result['returnCode'] != 0 :
592          result['svn version']=configuration['svnVersion']
593          # figure out what tarResultFail should be
594          #result['tar']=tarResultFail
595          result['command history']=commandHistory
596          NBemail.sendCmdMsgs(configuration['project'],result,  tarCmd)
597          return
598
599 
600  #---------------------------------------------------------------------
601  # Everything build and all tests passed.
602  #---------------------------------------------------------------------
603  os.chdir(fullBuildDir)
604  f=open('NBallTestsPassed','w')
605  f.close()
Note: See TracBrowser for help on using the repository browser.