source: branches/testScripts/NBbuildConfig.py @ 798

Last change on this file since 798 was 798, checked in by kmartin, 12 years ago

start to put in code to generate binaries

  • Property svn:eol-style set to native
File size: 22.3 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10import re
11import shutil
12
13import NBlogMessages
14import NBemail
15import NBosCommand
16import NBsvnCommand
17import NBcheckResult
18
19import NBuserParameters
20
21#---------------------------------------------------------------------
22# Keep history so same project is not repeatedly getting code from
23# subversion repository.
24#---------------------------------------------------------------------
25SVN_HISTORY = {}
26THIRD_PARTY_HISTORY = []
27
28
29#---------------------------------------------------------------------
30# cleanUpName:
31# File and directory names are generated and may contain
32# undesireable characters.
33# Remove these characters from the name
34#---------------------------------------------------------------------
35def cleanUpName(messedUpName) :
36  cleanedUpName=messedUpName
37  cleanedUpName=cleanedUpName.replace('-','')
38  cleanedUpName=cleanedUpName.replace('/','-')
39  cleanedUpName=cleanedUpName.replace('\\','-')
40  cleanedUpName=cleanedUpName.replace(' ','')
41  cleanedUpName=cleanedUpName.replace('"','')
42  cleanedUpName=cleanedUpName.replace("'",'')
43  cleanedUpName=cleanedUpName.replace("=",'-')
44  cleanedUpName=cleanedUpName.replace(":",'')
45  cleanedUpName=cleanedUpName.replace('--enable','')
46  return cleanedUpName
47
48
49#---------------------------------------------------------------------
50# writeResults:
51# After running a command write stdout and stderr to a file
52#---------------------------------------------------------------------
53def writeResults(result,filenameSuffix) :
54  cleanedUpSuffix=cleanUpName(filenameSuffix)
55  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
56  stdoutfile.write(result['stdout'])
57  stdoutfile.close()
58  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
59  stderrfile.write(result['stderr'])
60  stderrfile.close()
61 
62#------------------------------------------------------------------------
63#  Given a configuration, build and test it.
64#
65#  configuration['project']= name of project.
66#   examples: "Clp", "Ipopt"
67#
68#  configuration['rootDir']= root directory of nightlyBuild.
69#   This is where the project will be checked out from svn, and
70#   where the code will be compiled.  This directory must already
71#   exist.  If the testing requires, it needs to contain Netlib & miplib3
72#
73#  configuration['svnVersion']= svn version to be built.
74#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
75#
76#  configuration['buildMethod']= Defines method for building code.
77#  Choices are:
78#    msSoln: use microsoft compiler with a solution (sln) file.
79#    unixConfig: use sequence "./configure", "make", "make test"
80#
81#  configuration['noThirdParty']=True or False (optional). If False
82#   then 3rd party code will be used. If not specified then 3rd part
83#   code will be skipped.
84#   Only used if configuration['buildMethod']=='unixConfig'
85#
86#  configuration['configOptions']: Parameters to be passed to configure.
87#   The -C option and the options for skipping 3rd party code do not
88#   need to be specified.  These will be generated by this function.
89#   There are two types of configOptions to be specified.
90#   Only used if configuration['buildMethod']=='unixConfig'
91#  configuration['configOptions']['unique']= These are options that
92#   distinguish different build configurations.  These options are used
93#   to generate the vpath directory name where the code will be built.
94#   Examples are: "", "--enable-debug" "--enable-parrallel"
95#  configuration['configOptions']['invariant']= These are options that
96#   that are the same for every build configuration so they don't need
97#   to be part of the vpath directory name.
98#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
99
100#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
101#    from the build.
102#    examples: "Ipopt", "Ipopt DyLP"
103#
104#  configuration['slnFile']= path and name of solution file if it is not
105#    in the standard location.
106#    Only used if configuration['buildMethod']=='unixConfig'
107#
108#  configuration['test']=vector of triples indicating tests that
109#    are to be run after building the project. Each triple consists
110#    of:
111#    'dir': directory where test command is to be issued.
112#    'cmd': command to be run with any parameters.
113#    'check': vector of functions to be called which will check the
114#           results from running 'cmd' to determine if an error occurred
115#------------------------------------------------------------------------
116def run(configuration) :
117  NBlogMessages.clearMessages()
118  NBlogMessages.writeMessage( configuration['project'] )
119
120  # Create svn checkout target directory name
121  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
122
123  #---------------------------------------------------------------------
124  # Create names of directory where source is located and
125  # and were object, libs and executables are located (build directory)
126  # To compute build directory, the ./configure options need to be
127  # generated.
128  #---------------------------------------------------------------------
129  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
130  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
131
132  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
133                 configuration['project']+'/'+\
134                 configuration['svnVersion']
135
136  buildDir=svnVersionFlattened
137
138  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw' :
139    buildDir+=configuration['configOptions']['unique']
140    if 'SkipProjects' in configuration :
141      buildDir+="No"+configuration['SkipProjects']
142    if 'noThirdParty' in configuration : 
143      if configuration['noThirdParty'] :
144        buildDir+='-NoThirdParty'
145    buildDir=cleanUpName(buildDir)
146    if buildDir==svnVersionFlattened : buildDir+='-default'
147
148  fullBuildDir = os.path.join(projectBaseDir,buildDir)
149
150  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
151  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
152  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
153 
154  #for a list of commands that have been executed
155  commandHistory = []
156
157  #---------------------------------------------------------------------
158  # Completely remove a previous build if the user indicates this
159  #---------------------------------------------------------------------
160  if configuration['clear previous build'] and os.path.isdir(fullBuildDir) :
161    NBlogMessages.writeMessage('  Remove previous build in directory '+fullBuildDir)
162    try:
163      shutil.rmtree(fullBuildDir)
164    except shutil.Error :
165      NBlogMessages.writeMessage('  Warning: removal of directory '+fullBuildDir+' failed.')
166
167  #---------------------------------------------------------------------
168  # If nothing has changed and the prior run tested OK or there is
169  # a known problem being worked on, then there
170  # is no need to do anything.
171  #---------------------------------------------------------------------
172  if os.path.isdir(fullBuildDir) :
173    os.chdir(fullBuildDir)
174    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
175
176    if os.path.isfile('NBallTestsPassed') :
177      prevRunSuccess=True
178      NBlogMessages.writeMessage('  In prior build all test passed')
179    else:
180      prevRunSuccess=False
181      NBlogMessages.writeMessage('  No record of all tests having passed')
182
183    # If new source in svn, then must remove "allTestsPassed" files
184    if msg:
185      # SVN has changed since last run
186      NBlogMessages.writeMessage('  '+msg)
187      # Must remove file NBallTestsPassed from all build directories that
188      # use projectCheckoutDir for their source code. This is to ensure
189      # that make will be run in all the build dirs after "svn update"
190      dirs = os.listdir("..")
191      for d in dirs :
192        if d.startswith(svnVersionFlattened) :
193          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
194          if os.path.isfile(fileToBeRemoved) :
195            os.remove(fileToBeRemoved)
196            NBlogMessages.writeMessage('  Removing all test passed record from directory: '+d)
197    else :
198      # SVN has not changed since last run
199      if configuration['Run']=='noSuccessOrAfterChange':
200        if prevRunSuccess :
201          NBlogMessages.writeMessage('  No changes since previous successful run')
202          return
203        else: 
204          NBlogMessages.writeMessage('  Rerunning. No changes, but no record of successful run')
205      elif configuration['Run']=='afterChange' :
206        NBlogMessages.writeMessage('  No changes since previous run')
207        return
208      else:
209        NBlogMessages.writeMessage('  No changes but run always selected')
210  else :
211    NBlogMessages.writeMessage('  Targets have not yet been built')
212
213
214  #---------------------------------------------------------------------
215  # svn checkout or update the project
216  #---------------------------------------------------------------------
217  # Don't get source from subversion if previously done
218  if not SVN_HISTORY.has_key(projectCheckOutDir) :
219    if not os.path.isdir(projectBaseDir) :
220      os.makedirs(projectBaseDir)
221    if not os.path.isdir(projectCheckOutDir) :
222      svnCmd='svn ' +\
223           'checkout ' +\
224           svnCheckOutUrl +\
225           ' '+svnVersionFlattened
226      commandHistory+=[ svnCmd ]
227      svnResult=NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])
228      if svnResult['returnCode'] != 0 :
229        return
230      runConfigure = True
231    else :
232      svnCmd='svn update'
233      commandHistory+=[ svnCmd ]
234      svnResult=NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])
235      if svnResult['returnCode'] != 0 :
236        return
237      #check whether a *.in or configure file was updated
238      r=r'(\S+\.in\s)|(configure\s)'
239      findResult=re.findall(r,svnResult['stdout'])
240      if len(findResult)!=0:
241        runConfigure = True
242      else :
243        runConfigure = False
244
245    SVN_HISTORY[projectCheckOutDir]=runConfigure
246  else :
247    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
248    runConfigure=SVN_HISTORY[projectCheckOutDir]
249
250  #---------------------------------------------------------------------
251  # If there are third party apps, then get these apps
252  #---------------------------------------------------------------------
253  if 'noThirdParty' in configuration :
254    if not configuration['noThirdParty'] :
255      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
256      if os.path.isdir(thirdPartyBaseDir) :
257        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
258          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
259          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
260          #clean up: take care of .svn, .OLD, and non-directory entries
261          for d in thirdPartyDirs[:] :
262            if d=='.svn' : 
263              thirdPartyDirs.remove(d)
264              continue
265            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
266            if not os.path.isdir(thirdPartyDir) :
267              thirdPartyDirs.remove(d)
268              continue
269            if d.endswith('.OLD') :
270              NBlogMessages.writeMessage('  removing '+d)
271              shutil.rmtree(thirdPartyDir)
272              fileToBeRemoved=os.path.join(thirdPartyDir[0:-4], 'NBinstalldone')
273              if os.path.isfile(fileToBeRemoved) :
274                os.remove(fileToBeRemoved)
275              thirdPartyDirs.remove(d)
276              continue
277          for d in thirdPartyDirs :
278            if d=='.svn' : continue
279            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
280            if not os.path.isdir(thirdPartyDir) : continue
281            install3rdPartyCmd=os.path.join(".","get."+d)
282            os.chdir(thirdPartyDir)
283            # If the install command has been updated since the last
284            # install, then do a new install
285            if os.path.isfile('NBinstalldone') :
286              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
287                os.remove('NBinstalldone')
288            if not os.path.isfile('NBinstalldone') :
289              if os.path.isfile(install3rdPartyCmd) :
290                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
291                commandHistory+=[ install3rdPartyCmd ]
292                installReturn = NBosCommand.run(install3rdPartyCmd)
293                if installReturn['returnCode'] :
294                  NBlogMessages.writeMessage('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
295                else :
296                  f=open('NBinstalldone','w')
297                  f.close()
298                writeResults(installReturn,install3rdPartyCmd)
299            else :
300              NBlogMessages.writeMessage('  skipped a new download of '+d)
301        else :
302          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
303
304   
305  #---------------------------------------------------------------------
306  # Create the build directory if it doesn't exist
307  #---------------------------------------------------------------------
308  if not os.path.isdir(fullBuildDir) : 
309    os.makedirs(fullBuildDir)
310
311  #---------------------------------------------------------------------
312  # Source is now available, so now it is time to run config
313  #---------------------------------------------------------------------
314  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw':
315    skipOptions=''
316
317    if 'SkipProjects' in configuration :
318      skipOptions+=configuration['SkipProjects']
319
320    # If needed create option for skipping 3rd party code
321    needSkip3PartySkipOptions=False
322    if 'noThirdParty' not in configuration : 
323      needSkip3PartySkipOptions=True
324    elif configuration['noThirdParty'] :
325      needSkip3PartySkipOptions=True
326    if needSkip3PartySkipOptions :
327      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
328      if os.path.isdir(thirdPartyBaseDir) :
329        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
330        for d in thirdPartyDirs :
331          if d=='.svn' : continue
332          skipOptions+=' ThirdParty/'+d
333
334    if skipOptions!='' :
335      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
336
337    os.chdir(fullBuildDir)
338    NBlogMessages.writeMessage('  cd '+fullBuildDir)
339
340    # Assemble all config options together and create config command
341    configOptions ="-C "+configuration['configOptions']['unique']
342    configOptions+=configuration['configOptions']['invariant']
343    configOptions+=skipOptions
344   
345
346    #start kipp change
347    #
348    if configuration['buildMethod']=='mingw' :
349      configCmd = os.path.join(projectCheckOutDir,"configure ")
350      #what a pain replace("\\", "/") does not work
351      # we must split and then join, ugh
352      pathParts = configCmd.split("\\")
353      sep = '/'
354      configCmd = sep.join(pathParts)
355      configCmd = "sh -c " + "'" + configCmd + configOptions +  "'"
356    else:
357      configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions) 
358    #
359    #end kipp change
360
361    # If config was previously run, then no need to run again.
362    if (not runConfigure) and NBcheckResult.didConfigRunOK() :
363      NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
364    else :
365      NBlogMessages.writeMessage("  "+configCmd)
366      commandHistory+=[ configCmd ]
367
368      # Finally run config
369      result=NBosCommand.run(configCmd)
370      writeResults(result,'config') 
371
372      # Check if configure worked
373      if result['returnCode'] != 0 :
374        error_msg = result
375        error_msg['configure flags']=configOptions
376        error_msg['svn version']=configuration['svnVersion']
377        error_msg['command history']=commandHistory
378        # Add contents of log file to message
379        logFileName = 'config.log'
380        if os.path.isfile(logFileName) :
381          logFilePtr = open(logFileName,'r')
382          error_msg['config.log'] = logFilePtr.read()
383          logFilePtr.close()
384        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
385        return
386
387    #---------------------------------------------------------------------
388    # Run make part of build
389    #---------------------------------------------------------------------
390    NBlogMessages.writeMessage( '  make' )
391    commandHistory+=[ 'make' ]
392   
393    #
394    # start kipp
395    if configuration['buildMethod']=='mingw' :
396      result=NBosCommand.run('sh -c make') 
397    else:
398      result=NBosCommand.run('make')
399     
400    # end kipp
401    #
402    writeResults(result,'make') 
403
404    # Check if make worked
405    if result['returnCode'] != 0 :
406      result['configure flags']=configOptions
407      result['svn version']=configuration['svnVersion']
408      result['command history']=commandHistory
409      NBemail.sendCmdMsgs(configuration['project'],result,'make')
410      return
411
412  if configuration['buildMethod']=='msSln' :
413    #---------------------------------------------------------------------
414    # Source is now available, so now it is time to run vcbuild
415    #---------------------------------------------------------------------
416
417    if configuration.has_key('slnDir') :
418      slnFileDir = os.path.join(projectCheckOutDir,configuration['slnDir'])
419    else :
420      slnFileDir = os.path.join(projectCheckOutDir,\
421                          configuration['project'],\
422                          'MSVisualStudio',\
423                          'v8')
424    if not os.path.isdir(slnFileDir) :
425      NBlogMessages.writeMessage("  Solution file directory does not exist: "+slnFileDir)
426      return
427
428    os.chdir(slnFileDir)
429    NBlogMessages.writeMessage('  cd '+slnFileDir)
430
431    if configuration.has_key('slnFile') :
432      slnFileName = configuration['slnFile']
433    else :
434      slnFileName = configuration['project']+'.sln'
435    if not os.path.isfile(slnFileName) :
436      NBlogMessages.writeMessage("  Solution file does not exist '" \
437                                 +slnFileName \
438                                 +"' in directory " \
439                                 +slnFileDir )
440
441      return
442
443    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
444             
445    NBlogMessages.writeMessage("  "+vcbuild)
446    commandHistory+=[ vcbuild ]
447
448    # Finally run vcbuild
449    result=NBosCommand.run(vcbuild)
450    writeResults(result,'vcbuild') 
451
452    # Check if vcbuild worked
453    if result['returnCode'] != 0 :
454        error_msg = result
455        error_msg['svn version']=configuration['svnVersion']
456        error_msg['command history']=commandHistory
457        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
458        return
459
460  #---------------------------------------------------------------------
461  # Run all test executables
462  #---------------------------------------------------------------------
463  if "test" in configuration :
464    for t in range( len(configuration['test']) ) :
465      testRelDir=configuration['test'][t]['dir']
466      testDir = os.path.join(fullBuildDir,testRelDir)
467      testCmd=configuration['test'][t]['cmd']
468      if not os.path.isdir(testDir) :
469        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
470        NBlogMessages.writeMessage('    Intended directory: '+testDir)
471        NBlogMessages.writeMessage('    Intended command: '+testCmd)
472        continue
473      os.chdir(testDir)
474      NBlogMessages.writeMessage('  cd '+testDir)
475
476      NBlogMessages.writeMessage( '  '+testCmd )
477      commandHistory+=[ testCmd ]
478      result=NBosCommand.run(testCmd)
479      writeResults(result,testCmd)
480
481       
482      for testFunc in configuration['test'][t]['check'] :
483        testResultFail=testFunc(result,configuration['project'])
484        if testResultFail :
485          result['svn version']=configuration['svnVersion']
486          result['test']=testResultFail
487          result['command history']=commandHistory
488          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
489          return
490
491  #---------------------------------------------------------------------
492  # Run all install executables
493  #---------------------------------------------------------------------
494
495
496  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw':
497    if "install" in configuration :
498      for t in range( len(configuration['install']) ) :
499        installRelDir=configuration['install'][t]['dir']
500        installDir = os.path.join(fullBuildDir, installRelDir)
501        installCmd=configuration['install'][t]['cmd']
502        if not os.path.isdir( installDir) :
503          NBlogMessages.writeMessage('  Directory to run install from does not exist:')
504          NBlogMessages.writeMessage('    Intended directory: '+installDir)
505          NBlogMessages.writeMessage('    Intended command: '+installCmd)
506          continue
507        os.chdir(installDir)
508        NBlogMessages.writeMessage('  cd '+installDir)
509        NBlogMessages.writeMessage( '  '+installCmd )
510        commandHistory+=[ installCmd ]
511        result=NBosCommand.run(installCmd)
512        writeResults(result,installCmd)
513        if result['returnCode'] != 0 :
514            result['svn version']=configuration['svnVersion']
515            result['install']=installResultFail
516            result['command history']=commandHistory
517            NBemail.sendCmdMsgs(configuration['project'],result,installCmd)
518            return
519
520   
521    #---------------------------------------------------------------------
522    # Build the binary distribution
523    #---------------------------------------------------------------------
524       
525    # when doing this right make sure the example, include, and lib directories
526    # are there.
527    # if the lib  directory is there, add it
528    directories  = " lib "
529
530    # if the include directory is there, add it
531    directories +=  " include "
532
533    # if the examples directory is there, add it
534    #directories +=  " examples "
535
536    # if the bin directory is there, add it
537    #directories +=  " bin "
538
539    # don't forget to add LICENSE and AUTHORS file
540
541    #configuration['project']
542    outputDirectory = os.path.join(NBuserParameters.NIGHTLY_BUILD_BIN_DIR, configuration['project'])
543    print outputDirectory
544    # create the output directory
545   
546    # tar it up
547
548    tarCmd = "tar -czvf "
549    tarCmd += os.path.join(outputDirectory, "test.tgz")
550    tarCmd += directories
551
552    print tarCmd
553
554    result=NBosCommand.run( tarCmd)
555
556
557
558 
559  #---------------------------------------------------------------------
560  # Everything build and all tests passed.
561  #---------------------------------------------------------------------
562  os.chdir(fullBuildDir)
563  f=open('NBallTestsPassed','w')
564  f.close()
Note: See TracBrowser for help on using the repository browser.