source: branches/testScripts/NBbuildConfig.py @ 770

Last change on this file since 770 was 770, checked in by stefan, 13 years ago

create history of commands for a build type and put into email

  • Property svn:eol-style set to native
File size: 16.4 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10
11import NBlogMessages
12import NBemail
13import NBosCommand
14import NBsvnCommand
15import NBcheckResult
16
17#---------------------------------------------------------------------
18# Keep history so same project is not repeatedly getting code from
19# subversion repository.
20#---------------------------------------------------------------------
21SVN_HISTORY = []
22THIRD_PARTY_HISTORY = []
23
24
25#---------------------------------------------------------------------
26# cleanUpName:
27# File and directory names are generated and may contain
28# undesireable characters.
29# Remove these characters from the name
30#---------------------------------------------------------------------
31def cleanUpName(messedUpName) :
32  cleanedUpName=messedUpName
33  cleanedUpName=cleanedUpName.replace('-','')
34  cleanedUpName=cleanedUpName.replace('/','-')
35  cleanedUpName=cleanedUpName.replace('\\','-')
36  cleanedUpName=cleanedUpName.replace(' ','')
37  cleanedUpName=cleanedUpName.replace('"','')
38  cleanedUpName=cleanedUpName.replace("'",'')
39  cleanedUpName=cleanedUpName.replace("=",'-')
40  cleanedUpName=cleanedUpName.replace(":",'')
41  cleanedUpName=cleanedUpName.replace('--enable','')
42  return cleanedUpName
43
44
45#---------------------------------------------------------------------
46# writeResults:
47# After running a command write stdout and stderr to a file
48#---------------------------------------------------------------------
49def writeResults(result,filenameSuffix) :
50  cleanedUpSuffix=cleanUpName(filenameSuffix)
51  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
52  stdoutfile.write(result['stdout'])
53  stdoutfile.close()
54  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
55  stderrfile.write(result['stderr'])
56  stderrfile.close()
57 
58#------------------------------------------------------------------------
59#  Given a configuration, build and test it.
60#
61#  configuration['project']= name of project.
62#   examples: "Clp", "Ipopt"
63#
64#  configuration['rootDir']= root directory of nightlyBuild.
65#   This is where the project will be checked out from svn, and
66#   where the code will be compiled.  This directory must already
67#   exist.  If the testing requires, it needs to contain Netlib & miplib3
68#
69#  configuration['svnVersion']= svn version to be built.
70#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
71#
72#  configuration['buildMethod']= Defines method for building code.
73#  Choices are:
74#    msSoln: use microsoft compiler with a solution (sln) file.
75#    unixConfig: use sequence "./configure", "make", "make test"
76#
77#  configuration['noThirdParty']=True or False (optional). If False
78#   then 3rd party code will be used. If not specified then 3rd part
79#   code will be skipped.
80#   Only used if configuration['buildMethod']=='unixConfig'
81#
82#  configuration['configOptions']: Parameters to be passed to configure.
83#   The -C option and the options for skipping 3rd party code do not
84#   need to be specified.  These will be generated by this function.
85#   There are two types of configOptions to be specified.
86#   Only used if configuration['buildMethod']=='unixConfig'
87#  configuration['configOptions']['unique']= These are options that
88#   distinguish different build configurations.  These options are used
89#   to generate the vpath directory name where the code will be built.
90#   Examples are: "", "--enable-debug" "--enable-parrallel"
91#  configuration['configOptions']['invariant']= These are options that
92#   that are the same for every build configuration so they don't need
93#   to be part of the vpath directory name.
94#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
95
96#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
97#    from the build.
98#    examples: "Ipopt", "Ipopt DyLP"
99#
100#  configuration['slnFile']= path and name of solution file if it is not
101#    in the standard location.
102#    Only used if configuration['buildMethod']=='unixConfig'
103#
104#  configuration['test']=vector of triples indicating tests that
105#    are to be run after building the project. Each triple consists
106#    of:
107#    'dir': directory where test command is to be issued.
108#    'cmd': command to be run with any parameters.
109#    'check': vector of functions to be called which will check the
110#           results from running 'cmd' to determine if an error occurred
111#------------------------------------------------------------------------
112def run(configuration) :
113  NBlogMessages.clearMessages()
114  NBlogMessages.writeMessage( configuration['project'] )
115
116  # Create svn checkout target directory name
117  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
118
119  #---------------------------------------------------------------------
120  # Create names of directory where source is located and
121  # and were object, libs and executables are located (build directory)
122  # To compute build directory, the ./configure options need to be
123  # generated.
124  #---------------------------------------------------------------------
125  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
126  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
127
128  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
129                 configuration['project']+'/'+\
130                 configuration['svnVersion']
131
132  buildDir=svnVersionFlattened
133
134  if configuration['buildMethod']=='unixConfig' :
135    buildDir+=configuration['configOptions']['unique']
136    if 'SkipProjects' in configuration :
137      buildDir+="No"+configuration['SkipProjects']
138    if 'noThirdParty' in configuration : 
139      if configuration['noThirdParty'] :
140        buildDir+='-NoThirdParty'
141    buildDir=cleanUpName(buildDir)
142    if buildDir==svnVersionFlattened : buildDir+='-default'
143
144  fullBuildDir = os.path.join(projectBaseDir,buildDir)
145
146  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
147  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
148  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
149 
150  #for a list of commands that have been executed
151  commandHistory = []
152
153  #---------------------------------------------------------------------
154  # If nothing has changed and the prior run tested OK, then there
155  # is no need to do anything.
156  #---------------------------------------------------------------------
157  if os.path.isdir(fullBuildDir) :
158    os.chdir(fullBuildDir)
159    if os.path.isfile('NBallTestsPassed') : 
160      msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
161      if not msg:
162        # Previous run ran fine, and nothing has changed.
163        NBlogMessages.writeMessage('  No changes since previous successfull run')
164        return
165      NBlogMessages.writeMessage('  '+msg)
166     
167      # Must remove file NBallTestsPassed from all build directories that
168      # use projectCheckoutDir for their source code. This is to ensure
169      # that make will be run in all the build dirs after "svn update"
170      dirs = os.listdir("..")
171      for d in dirs :
172        if d.startswith(configuration['svnVersion']) :
173          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
174          if os.path.isfile(fileToBeRemoved) :
175            os.remove(fileToBeRemoved)
176    else :
177      NBlogMessages.writeMessage('  No record of all tests having passed')
178  else :
179    NBlogMessages.writeMessage('  Targets have not yet been built')
180
181
182  #---------------------------------------------------------------------
183  # svn checkout or update the project
184  #---------------------------------------------------------------------
185  # Don't get source from subversion if previously done
186  if projectCheckOutDir not in SVN_HISTORY :
187    if not os.path.isdir(projectBaseDir) :
188      os.makedirs(projectBaseDir)
189    if not os.path.isdir(projectCheckOutDir) :
190      svnCmd='svn ' +\
191           'checkout ' +\
192           svnCheckOutUrl +\
193           ' '+svnVersionFlattened
194      commandHistory+=[ svnCmd ]
195      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
196        return
197    else :
198      svnCmd='svn update'
199      commandHistory+=[ svnCmd ]
200      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
201        return
202    SVN_HISTORY.append(projectCheckOutDir)
203  else :
204    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
205
206  #---------------------------------------------------------------------
207  # If there are third party apps, then get these apps
208  #---------------------------------------------------------------------
209  if 'noThirdParty' in configuration :
210    if not configuration['noThirdParty'] :
211      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
212      if os.path.isdir(thirdPartyBaseDir) :
213        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
214          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
215          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
216          for d in thirdPartyDirs :
217            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
218            install3rdPartyCmd=os.path.join(".","get."+d)
219            os.chdir(thirdPartyDir)
220            # If the install command has been updated since the last
221            # install, then do a new install
222            if os.path.isfile('NBinstalldone') :
223              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
224                os.remove('NBinstalldone')
225            if not os.path.isfile('NBinstalldone') :
226              if os.path.isfile(install3rdPartyCmd) :
227                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
228                commandHistory+=[ install3rdPartyCmd ]
229                installReturn = NBosCommand.run(install3rdPartyCmd)
230                if installReturn['returnCode'] :
231                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
232                else :
233                  f=open('NBinstalldone','w')
234                  f.close()
235                writeResults(installReturn,install3rdPartyCmd)
236            else :
237              NBlogMessages.writeMessage('  skipped a new download of '+d)
238        else :
239          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
240
241  #---------------------------------------------------------------------
242  # Create the build directory if it doesn't exist
243  # and remove file that indicates prior run tested ok.
244  #---------------------------------------------------------------------
245  if not os.path.isdir(fullBuildDir) : 
246    os.makedirs(fullBuildDir)
247
248  #---------------------------------------------------------------------
249  # Source is now available, so now it is time to run config
250  #---------------------------------------------------------------------
251  if configuration['buildMethod']=='unixConfig' :
252    skipOptions=''
253
254    if 'SkipProjects' in configuration :
255      skipOptions+=configuration['SkipProjects']
256
257    # If needed create option for skipping 3rd party code
258    needSkip3PartySkipOptions=False
259    if 'noThirdParty' not in configuration : 
260      needSkip3PartySkipOptions=True
261    elif configuration['noThirdParty'] :
262      needSkip3PartySkipOptions=True
263    if needSkip3PartySkipOptions :
264      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
265      if os.path.isdir(thirdPartyBaseDir) :
266        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
267        for d in thirdPartyDirs :
268          skipOptions+=' ThirdParty/'+d
269
270    if skipOptions!='' :
271      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
272
273    os.chdir(fullBuildDir)
274    NBlogMessages.writeMessage('  cd '+fullBuildDir)
275
276    # Assemble all config options together and create config command
277    configOptions ="-C "+configuration['configOptions']['unique']
278    configOptions+=configuration['configOptions']['invariant']
279    configOptions+=skipOptions
280    configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
281
282    # If config was previously run, then no need to run again.
283  #  if NBcheckResult.didConfigRunOK() :
284  #    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
285  #  else :
286    NBlogMessages.writeMessage("  "+configCmd)
287    commandHistory+=[ configCmd ]
288
289    # Finally run config
290    result=NBosCommand.run(configCmd)
291    writeResults(result,'config') 
292
293    # Check if configure worked
294    if result['returnCode'] != 0 :
295        error_msg = result
296        error_msg['configure flags']=configOptions
297        error_msg['svn version']=configuration['svnVersion']
298        error_msg['command history']=commandHistory
299        # Add contents of log file to message
300        logFileName = 'config.log'
301        if os.path.isfile(logFileName) :
302          logFilePtr = open(logFileName,'r')
303          error_msg['config.log'] = logFilePtr.read()
304          logFilePtr.close()
305        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
306        return
307
308    #---------------------------------------------------------------------
309    # Run make part of build
310    #---------------------------------------------------------------------
311    NBlogMessages.writeMessage( '  make' )
312    commandHistory+=[ 'make' ]
313    result=NBosCommand.run('make')
314    writeResults(result,'make') 
315
316    # Check if make worked
317    if result['returnCode'] != 0 :
318      result['configure flags']=configOptions
319      result['svn version']=configuration['svnVersion']
320      result['command history']=commandHistory
321      NBemail.sendCmdMsgs(configuration['project'],result,'make')
322      return
323
324  if configuration['buildMethod']=='msSln' :
325    #---------------------------------------------------------------------
326    # Source is now available, so now it is time to run vcbuild
327    #---------------------------------------------------------------------
328   
329    os.chdir(projectCheckOutDir)
330    NBlogMessages.writeMessage('  cd '+projectCheckOutDir)
331
332    if configuration.has_key('slnFile') :
333      slnFileName = os.path.join(projectCheckOutDir,configuration['slnFile'])
334    else :
335      slnFileName = os.path.join(projectCheckOutDir,\
336                          configuration['project'],\
337                          'MSVisualStudio',\
338                          'v8',\
339                          configuration['project']) +\
340                    '.sln'
341    if not os.path.isfile(slnFileName) :
342      NBlogMessages.writeMessage("  Solution file does not exist: "+slnFileName)
343      return
344
345    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
346             
347
348    NBlogMessages.writeMessage("  "+vcbuild)
349    commandHistory+=[ vcbuild ]
350
351    # Finally run vcbuild
352    result=NBosCommand.run(vcbuild)
353    writeResults(result,'vcbuild') 
354
355    # Check if vcbuild worked
356    if result['returnCode'] != 0 :
357        error_msg = result
358        error_msg['svn version']=configuration['svnVersion']
359        error_msg['command history']=commandHistory
360        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
361        return
362
363  #---------------------------------------------------------------------
364  # Run all test executables
365  #---------------------------------------------------------------------
366  if "test" in configuration :
367    for t in range( len(configuration['test']) ) :
368      testRelDir=configuration['test'][t]['dir']
369      testDir = os.path.join(fullBuildDir,testRelDir)
370      testCmd=configuration['test'][t]['cmd']
371      if not os.path.isdir(testDir) :
372        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
373        NBlogMessages.writeMessage('    Intended directory: '+testDir)
374        NBlogMessages.writeMessage('    Intended command: '+testCmd)
375        continue
376      os.chdir(testDir)
377      NBlogMessages.writeMessage('  cd '+testDir)
378
379      NBlogMessages.writeMessage( '  '+testCmd )
380      commandHistory+=[ testCmd ]
381      result=NBosCommand.run(testCmd)
382      writeResults(result,testCmd) 
383       
384      for testFunc in configuration['test'][t]['check'] :
385        testResultFail=testFunc(result,configuration['project'])
386        if testResultFail :
387          result['svn version']=configuration['svnVersion']
388          result['test']=testResultFail
389          result['command history']=commandHistory
390          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
391          return
392
393
394  #---------------------------------------------------------------------
395  # Everything build and all tests passed.
396  #---------------------------------------------------------------------
397  os.chdir(fullBuildDir)
398  f=open('NBallTestsPassed','w')
399  f.close()
Note: See TracBrowser for help on using the repository browser.