source: branches/testScripts/NBbuildConfig.py @ 787

Last change on this file since 787 was 787, checked in by kmartin, 12 years ago

icorporate a mingw build

  • Property svn:eol-style set to native
File size: 19.1 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10import re
11import shutil
12
13import NBlogMessages
14import NBemail
15import NBosCommand
16import NBsvnCommand
17import NBcheckResult
18
19#---------------------------------------------------------------------
20# Keep history so same project is not repeatedly getting code from
21# subversion repository.
22#---------------------------------------------------------------------
23SVN_HISTORY = {}
24THIRD_PARTY_HISTORY = []
25
26
27#---------------------------------------------------------------------
28# cleanUpName:
29# File and directory names are generated and may contain
30# undesireable characters.
31# Remove these characters from the name
32#---------------------------------------------------------------------
33def cleanUpName(messedUpName) :
34  cleanedUpName=messedUpName
35  cleanedUpName=cleanedUpName.replace('-','')
36  cleanedUpName=cleanedUpName.replace('/','-')
37  cleanedUpName=cleanedUpName.replace('\\','-')
38  cleanedUpName=cleanedUpName.replace(' ','')
39  cleanedUpName=cleanedUpName.replace('"','')
40  cleanedUpName=cleanedUpName.replace("'",'')
41  cleanedUpName=cleanedUpName.replace("=",'-')
42  cleanedUpName=cleanedUpName.replace(":",'')
43  cleanedUpName=cleanedUpName.replace('--enable','')
44  return cleanedUpName
45
46
47#---------------------------------------------------------------------
48# writeResults:
49# After running a command write stdout and stderr to a file
50#---------------------------------------------------------------------
51def writeResults(result,filenameSuffix) :
52  cleanedUpSuffix=cleanUpName(filenameSuffix)
53  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
54  stdoutfile.write(result['stdout'])
55  stdoutfile.close()
56  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
57  stderrfile.write(result['stderr'])
58  stderrfile.close()
59 
60#------------------------------------------------------------------------
61#  Given a configuration, build and test it.
62#
63#  configuration['project']= name of project.
64#   examples: "Clp", "Ipopt"
65#
66#  configuration['rootDir']= root directory of nightlyBuild.
67#   This is where the project will be checked out from svn, and
68#   where the code will be compiled.  This directory must already
69#   exist.  If the testing requires, it needs to contain Netlib & miplib3
70#
71#  configuration['svnVersion']= svn version to be built.
72#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
73#
74#  configuration['buildMethod']= Defines method for building code.
75#  Choices are:
76#    msSoln: use microsoft compiler with a solution (sln) file.
77#    unixConfig: use sequence "./configure", "make", "make test"
78#
79#  configuration['noThirdParty']=True or False (optional). If False
80#   then 3rd party code will be used. If not specified then 3rd part
81#   code will be skipped.
82#   Only used if configuration['buildMethod']=='unixConfig'
83#
84#  configuration['configOptions']: Parameters to be passed to configure.
85#   The -C option and the options for skipping 3rd party code do not
86#   need to be specified.  These will be generated by this function.
87#   There are two types of configOptions to be specified.
88#   Only used if configuration['buildMethod']=='unixConfig'
89#  configuration['configOptions']['unique']= These are options that
90#   distinguish different build configurations.  These options are used
91#   to generate the vpath directory name where the code will be built.
92#   Examples are: "", "--enable-debug" "--enable-parrallel"
93#  configuration['configOptions']['invariant']= These are options that
94#   that are the same for every build configuration so they don't need
95#   to be part of the vpath directory name.
96#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
97
98#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
99#    from the build.
100#    examples: "Ipopt", "Ipopt DyLP"
101#
102#  configuration['slnFile']= path and name of solution file if it is not
103#    in the standard location.
104#    Only used if configuration['buildMethod']=='unixConfig'
105#
106#  configuration['test']=vector of triples indicating tests that
107#    are to be run after building the project. Each triple consists
108#    of:
109#    'dir': directory where test command is to be issued.
110#    'cmd': command to be run with any parameters.
111#    'check': vector of functions to be called which will check the
112#           results from running 'cmd' to determine if an error occurred
113#------------------------------------------------------------------------
114def run(configuration) :
115  NBlogMessages.clearMessages()
116  NBlogMessages.writeMessage( configuration['project'] )
117
118  # Create svn checkout target directory name
119  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
120
121  #---------------------------------------------------------------------
122  # Create names of directory where source is located and
123  # and were object, libs and executables are located (build directory)
124  # To compute build directory, the ./configure options need to be
125  # generated.
126  #---------------------------------------------------------------------
127  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
128  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
129
130  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
131                 configuration['project']+'/'+\
132                 configuration['svnVersion']
133
134  buildDir=svnVersionFlattened
135
136  if configuration['buildMethod']=='unixConfig' :
137    buildDir+=configuration['configOptions']['unique']
138    if 'SkipProjects' in configuration :
139      buildDir+="No"+configuration['SkipProjects']
140    if 'noThirdParty' in configuration : 
141      if configuration['noThirdParty'] :
142        buildDir+='-NoThirdParty'
143    buildDir=cleanUpName(buildDir)
144    if buildDir==svnVersionFlattened : buildDir+='-default'
145
146  fullBuildDir = os.path.join(projectBaseDir,buildDir)
147
148  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
149  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
150  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
151 
152  #for a list of commands that have been executed
153  commandHistory = []
154
155  #---------------------------------------------------------------------
156  # Completely remove a previous build if the user indicates this
157  #---------------------------------------------------------------------
158  if configuration['clear previous build'] and os.path.isdir(fullBuildDir) :
159    NBlogMessages.writeMessage('  Remove previous build in directory '+fullBuildDir)
160    try:
161      shutil.rmtree(fullBuildDir)
162    except shutils.Error :
163      NBlogMessages.writeMessage('  Warning: removal of directory '+fullBuildDir+' failed.')
164
165  #---------------------------------------------------------------------
166  # If nothing has changed and the prior run tested OK or there is
167  # a known problem being worked on, then there
168  # is no need to do anything.
169  #---------------------------------------------------------------------
170  if os.path.isdir(fullBuildDir) :
171    os.chdir(fullBuildDir)
172    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
173
174    if os.path.isfile('NBallTestsPassed') :
175      prevRunSuccess=True
176      NBlogMessages.writeMessage('  In prior build all test passed')
177    else:
178      prevRunSuccess=False
179      NBlogMessages.writeMessage('  No record of all tests having passed')
180
181    # If new source in svn, then must remove "allTestsPassed" files
182    if msg:
183      # SVN has changed since last run
184      NBlogMessages.writeMessage('  '+msg)
185      # Must remove file NBallTestsPassed from all build directories that
186      # use projectCheckoutDir for their source code. This is to ensure
187      # that make will be run in all the build dirs after "svn update"
188      dirs = os.listdir("..")
189      for d in dirs :
190        if d.startswith(svnVersionFlattened) :
191          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
192          if os.path.isfile(fileToBeRemoved) :
193            os.remove(fileToBeRemoved)
194            NBlogMessages.writeMessage('  Removing all test passed record from directory: '+d)
195    else :
196      # SVN has not changed since last run
197      if configuration['Run']=='noSuccessOrAfterChange':
198        if prevRunSuccess :
199          NBlogMessages.writeMessage('  No changes since previous successful run')
200          return
201        else: 
202          NBlogMessages.writeMessage('  Rerunning. No changes, but no record of successful run')
203      elif configuration['Run']=='afterChange' :
204        NBlogMessages.writeMessage('  No changes since previous run')
205        return
206      else:
207        NBlogMessages.writeMessage('  No changes but run always selected')
208  else :
209    NBlogMessages.writeMessage('  Targets have not yet been built')
210
211
212  #---------------------------------------------------------------------
213  # svn checkout or update the project
214  #---------------------------------------------------------------------
215  # Don't get source from subversion if previously done
216  if not SVN_HISTORY.has_key(projectCheckOutDir) :
217    if not os.path.isdir(projectBaseDir) :
218      os.makedirs(projectBaseDir)
219    if not os.path.isdir(projectCheckOutDir) :
220      svnCmd='svn ' +\
221           'checkout ' +\
222           svnCheckOutUrl +\
223           ' '+svnVersionFlattened
224      commandHistory+=[ svnCmd ]
225      svnResult=NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])
226      if svnResult['returnCode'] != 0 :
227        return
228      runConfigure = True
229    else :
230      svnCmd='svn update'
231      commandHistory+=[ svnCmd ]
232      svnResult=NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])
233      if svnResult['returnCode'] != 0 :
234        return
235      #check whether a *.in or configure file was updated
236      r=r'(\S+\.in\s)|(configure\s)'
237      findResult=re.findall(r,svnResult['stdout'])
238      if len(findResult)!=0:
239        runConfigure = True
240      else :
241        runConfigure = False
242
243    SVN_HISTORY[projectCheckOutDir]=runConfigure
244  else :
245    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
246    runConfigure=SVN_HISTORY[projectCheckOutDir]
247
248  #---------------------------------------------------------------------
249  # If there are third party apps, then get these apps
250  #---------------------------------------------------------------------
251  if 'noThirdParty' in configuration :
252    if not configuration['noThirdParty'] :
253      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
254      if os.path.isdir(thirdPartyBaseDir) :
255        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
256          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
257          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
258          for d in thirdPartyDirs :
259            if d=='.svn' : continue
260            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
261            install3rdPartyCmd=os.path.join(".","get."+d)
262            os.chdir(thirdPartyDir)
263            # If the install command has been updated since the last
264            # install, then do a new install
265            if os.path.isfile('NBinstalldone') :
266              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
267                os.remove('NBinstalldone')
268            if not os.path.isfile('NBinstalldone') :
269              if os.path.isfile(install3rdPartyCmd) :
270                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
271                commandHistory+=[ install3rdPartyCmd ]
272                installReturn = NBosCommand.run(install3rdPartyCmd)
273                if installReturn['returnCode'] :
274                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
275                else :
276                  f=open('NBinstalldone','w')
277                  f.close()
278                writeResults(installReturn,install3rdPartyCmd)
279            else :
280              NBlogMessages.writeMessage('  skipped a new download of '+d)
281        else :
282          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
283
284   
285  #---------------------------------------------------------------------
286  # Create the build directory if it doesn't exist
287  #---------------------------------------------------------------------
288  if not os.path.isdir(fullBuildDir) : 
289    os.makedirs(fullBuildDir)
290
291  #---------------------------------------------------------------------
292  # Source is now available, so now it is time to run config
293  #---------------------------------------------------------------------
294  if configuration['buildMethod']=='unixConfig' or configuration['buildMethod']=='mingw':
295    skipOptions=''
296
297    if 'SkipProjects' in configuration :
298      skipOptions+=configuration['SkipProjects']
299
300    # If needed create option for skipping 3rd party code
301    needSkip3PartySkipOptions=False
302    if 'noThirdParty' not in configuration : 
303      needSkip3PartySkipOptions=True
304    elif configuration['noThirdParty'] :
305      needSkip3PartySkipOptions=True
306    if needSkip3PartySkipOptions :
307      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
308      if os.path.isdir(thirdPartyBaseDir) :
309        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
310        for d in thirdPartyDirs :
311          if d=='.svn' : continue
312          skipOptions+=' ThirdParty/'+d
313
314    if skipOptions!='' :
315      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
316
317    os.chdir(fullBuildDir)
318    NBlogMessages.writeMessage('  cd '+fullBuildDir)
319
320    # Assemble all config options together and create config command
321    configOptions ="-C "+configuration['configOptions']['unique']
322    configOptions+=configuration['configOptions']['invariant']
323    configOptions+=skipOptions
324   
325
326    #start kipp change
327    #
328    if configuration['buildMethod']=='mingw' :
329      configCmd = os.path.join(projectCheckOutDir,"configure ")
330      #what a pain replace("\\", "/") does not work
331      # we must split and then join, ugh
332      pathParts = configCmd.split("\\")
333      sep = '/'
334      configCmd = sep.join(pathParts)
335      configCmd = "sh -c " + "'" + configCmd + configOptions +  "'"
336    else:
337      configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions) 
338    #
339    #end kipp change
340
341    # If config was previously run, then no need to run again.
342    if (not runConfigure) and NBcheckResult.didConfigRunOK() :
343      NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
344    else :
345      NBlogMessages.writeMessage("  "+configCmd)
346      commandHistory+=[ configCmd ]
347
348      # Finally run config
349      result=NBosCommand.run(configCmd)
350      writeResults(result,'config') 
351
352      # Check if configure worked
353      if result['returnCode'] != 0 :
354        error_msg = result
355        error_msg['configure flags']=configOptions
356        error_msg['svn version']=configuration['svnVersion']
357        error_msg['command history']=commandHistory
358        # Add contents of log file to message
359        logFileName = 'config.log'
360        if os.path.isfile(logFileName) :
361          logFilePtr = open(logFileName,'r')
362          error_msg['config.log'] = logFilePtr.read()
363          logFilePtr.close()
364        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
365        return
366
367    #---------------------------------------------------------------------
368    # Run make part of build
369    #---------------------------------------------------------------------
370    NBlogMessages.writeMessage( '  make' )
371    commandHistory+=[ 'make' ]
372   
373    #
374    # start kipp
375    if configuration['buildMethod']=='mingw' :
376      result=NBosCommand.run('sh -c make') 
377    else:
378      result=NBosCommand.run('make')
379     
380    # end kipp
381    #
382    writeResults(result,'make') 
383
384    # Check if make worked
385    if result['returnCode'] != 0 :
386      result['configure flags']=configOptions
387      result['svn version']=configuration['svnVersion']
388      result['command history']=commandHistory
389      NBemail.sendCmdMsgs(configuration['project'],result,'make')
390      return
391
392  if configuration['buildMethod']=='msSln' :
393    #---------------------------------------------------------------------
394    # Source is now available, so now it is time to run vcbuild
395    #---------------------------------------------------------------------
396
397    if configuration.has_key('slnDir') :
398      slnFileDir = os.path.join(projectCheckOutDir,configuration['slnDir'])
399    else :
400      slnFileDir = os.path.join(projectCheckOutDir,\
401                          configuration['project'],\
402                          'MSVisualStudio',\
403                          'v8')
404    if not os.path.isdir(slnFileDir) :
405      NBlogMessages.writeMessage("  Solution file directory does not exist: "+slnFileDir)
406      return
407
408    os.chdir(slnFileDir)
409    NBlogMessages.writeMessage('  cd '+slnFileDir)
410
411    if configuration.has_key('slnFile') :
412      slnFileName = configuration['slnFile']
413    else :
414      slnFileName = configuration['project']+'.sln'
415    if not os.path.isfile(slnFileName) :
416      NBlogMessages.writeMessage("  Solution file does not exist '" \
417                                 +slnFileName \
418                                 +"' in directory " \
419                                 +slnFileDir )
420
421      return
422
423    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
424             
425    NBlogMessages.writeMessage("  "+vcbuild)
426    commandHistory+=[ vcbuild ]
427
428    # Finally run vcbuild
429    result=NBosCommand.run(vcbuild)
430    writeResults(result,'vcbuild') 
431
432    # Check if vcbuild worked
433    if result['returnCode'] != 0 :
434        error_msg = result
435        error_msg['svn version']=configuration['svnVersion']
436        error_msg['command history']=commandHistory
437        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
438        return
439
440  #---------------------------------------------------------------------
441  # Run all test executables
442  #---------------------------------------------------------------------
443  if "test" in configuration :
444    for t in range( len(configuration['test']) ) :
445      testRelDir=configuration['test'][t]['dir']
446      testDir = os.path.join(fullBuildDir,testRelDir)
447      testCmd=configuration['test'][t]['cmd']
448      if not os.path.isdir(testDir) :
449        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
450        NBlogMessages.writeMessage('    Intended directory: '+testDir)
451        NBlogMessages.writeMessage('    Intended command: '+testCmd)
452        continue
453      os.chdir(testDir)
454      NBlogMessages.writeMessage('  cd '+testDir)
455
456      NBlogMessages.writeMessage( '  '+testCmd )
457      commandHistory+=[ testCmd ]
458      result=NBosCommand.run(testCmd)
459      writeResults(result,testCmd) 
460       
461      for testFunc in configuration['test'][t]['check'] :
462        testResultFail=testFunc(result,configuration['project'])
463        if testResultFail :
464          result['svn version']=configuration['svnVersion']
465          result['test']=testResultFail
466          result['command history']=commandHistory
467          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
468          return
469
470
471  #---------------------------------------------------------------------
472  # Everything build and all tests passed.
473  #---------------------------------------------------------------------
474  os.chdir(fullBuildDir)
475  f=open('NBallTestsPassed','w')
476  f.close()
Note: See TracBrowser for help on using the repository browser.