source: branches/testScripts/NBbuildConfig.py @ 761

Last change on this file since 761 was 760, checked in by jpfasano, 12 years ago

Added a check to make sure the test directory exists to avoid a crash if NBprojectConfig is coded incorrectly.

  • Property svn:eol-style set to native
File size: 15.9 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10
11import NBlogMessages
12import NBemail
13import NBosCommand
14import NBsvnCommand
15import NBcheckResult
16
17#---------------------------------------------------------------------
18# Keep history so same project is not repeatedly getting code from
19# subversion repository.
20#---------------------------------------------------------------------
21SVN_HISTORY = []
22THIRD_PARTY_HISTORY = []
23
24
25#---------------------------------------------------------------------
26# cleanUpName:
27# File and directory names are generated and may contain
28# undesireable characters.
29# Remove these characters from the name
30#---------------------------------------------------------------------
31def cleanUpName(messedUpName) :
32  cleanedUpName=messedUpName
33  cleanedUpName=cleanedUpName.replace('-','')
34  cleanedUpName=cleanedUpName.replace('/','-')
35  cleanedUpName=cleanedUpName.replace('\\','-')
36  cleanedUpName=cleanedUpName.replace(' ','')
37  cleanedUpName=cleanedUpName.replace('"','')
38  cleanedUpName=cleanedUpName.replace("'",'')
39  cleanedUpName=cleanedUpName.replace("=",'-')
40  cleanedUpName=cleanedUpName.replace(":",'')
41  cleanedUpName=cleanedUpName.replace('--enable','')
42  return cleanedUpName
43
44
45#---------------------------------------------------------------------
46# writeResults:
47# After running a command write stdout and stderr to a file
48#---------------------------------------------------------------------
49def writeResults(result,filenameSuffix) :
50  cleanedUpSuffix=cleanUpName(filenameSuffix)
51  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
52  stdoutfile.write(result['stdout'])
53  stdoutfile.close()
54  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
55  stderrfile.write(result['stderr'])
56  stderrfile.close()
57 
58#------------------------------------------------------------------------
59#  Given a configuration, build and test it.
60#
61#  configuration['project']= name of project.
62#   examples: "Clp", "Ipopt"
63#
64#  configuration['rootDir']= root directory of nightlyBuild.
65#   This is where the project will be checked out from svn, and
66#   where the code will be compiled.  This directory must already
67#   exist.  If the testing requires, it needs to contain Netlib & miplib3
68#
69#  configuration['svnVersion']= svn version to be built.
70#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
71#
72#  configuration['buildMethod']= Defines method for building code.
73#  Choices are:
74#    msSoln: use microsoft compiler with a solution (sln) file.
75#    unixConfig: use sequence "./configure", "make", "make test"
76#
77#  configuration['noThirdParty']=True or False (optional). If False
78#   then 3rd party code will be used. If not specified then 3rd part
79#   code will be skipped.
80#   Only used if configuration['buildMethod']=='unixConfig'
81#
82#  configuration['configOptions']: Parameters to be passed to configure.
83#   The -C option and the options for skipping 3rd party code do not
84#   need to be specified.  These will be generated by this function.
85#   There are two types of configOptions to be specified.
86#   Only used if configuration['buildMethod']=='unixConfig'
87#  configuration['configOptions']['unique']= These are options that
88#   distinguish different build configurations.  These options are used
89#   to generate the vpath directory name where the code will be built.
90#   Examples are: "", "--enable-debug" "--enable-parrallel"
91#  configuration['configOptions']['invariant']= These are options that
92#   that are the same for every build configuration so they don't need
93#   to be part of the vpath directory name.
94#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
95
96#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
97#    from the build.
98#    examples: "Ipopt", "Ipopt DyLP"
99#
100#  configuration['slnFile']= path and name of solution file if it is not
101#    in the standard location.
102#    Only used if configuration['buildMethod']=='unixConfig'
103#
104#  configuration['test']=vector of triples indicating tests that
105#    are to be run after building the project. Each triple consists
106#    of:
107#    'dir': directory where test command is to be issued.
108#    'cmd': command to be run with any parameters.
109#    'check': vector of functions to be called which will check the
110#           results from running 'cmd' to determine if an error occurred
111#------------------------------------------------------------------------
112def run(configuration) :
113  NBlogMessages.clearMessages()
114  NBlogMessages.writeMessage( configuration['project'] )
115
116  # Create svn checkout target directory name
117  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
118
119  #---------------------------------------------------------------------
120  # Create names of directory where source is located and
121  # and were object, libs and executables are located (build directory)
122  # To compute build directory, the ./configure options need to be
123  # generated.
124  #---------------------------------------------------------------------
125  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
126  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
127
128  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
129                 configuration['project']+'/'+\
130                 configuration['svnVersion']
131
132  buildDir=svnVersionFlattened
133
134  if configuration['buildMethod']=='unixConfig' :
135    buildDir+=configuration['configOptions']['unique']
136    if 'SkipProjects' in configuration :
137      buildDir+="No"+configuration['SkipProjects']
138    if 'noThirdParty' in configuration : 
139      if configuration['noThirdParty'] :
140        buildDir+='-NoThirdParty'
141    buildDir=cleanUpName(buildDir)
142    if buildDir==svnVersionFlattened : buildDir+='-default'
143
144  fullBuildDir = os.path.join(projectBaseDir,buildDir)
145
146  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
147  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
148  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
149 
150
151  #---------------------------------------------------------------------
152  # If nothing has changed and the prior run tested OK, then there
153  # is no need to do anything.
154  #---------------------------------------------------------------------
155  if os.path.isdir(fullBuildDir) :
156    os.chdir(fullBuildDir)
157    if os.path.isfile('NBallTestsPassed') : 
158      msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
159      if not msg:
160        # Previous run ran fine, and nothing has changed.
161        NBlogMessages.writeMessage('  No changes since previous successfull run')
162        return
163      NBlogMessages.writeMessage('  '+msg)
164     
165      # Must remove file NBallTestsPassed from all build directories that
166      # use projectCheckoutDir for their source code. This is to ensure
167      # that make will be run in all the build dirs after "svn update"
168      dirs = os.listdir("..")
169      for d in dirs :
170        if d.startswith(configuration['svnVersion']) :
171          fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
172          if os.path.isfile(fileToBeRemoved) :
173            os.remove(fileToBeRemoved)
174    else :
175      NBlogMessages.writeMessage('  No record of all tests having passed')
176  else :
177    NBlogMessages.writeMessage('  Targets have not yet been built')
178
179
180  #---------------------------------------------------------------------
181  # svn checkout or update the project
182  #---------------------------------------------------------------------
183  # Don't get source from subversion if previously done
184  if projectCheckOutDir not in SVN_HISTORY :
185    if not os.path.isdir(projectBaseDir) :
186      os.makedirs(projectBaseDir)
187    if not os.path.isdir(projectCheckOutDir) :
188      svnCmd='svn ' +\
189           'checkout ' +\
190           svnCheckOutUrl +\
191           ' '+svnVersionFlattened
192      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
193        return
194    else :
195      svnCmd='svn update'
196      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
197        return
198    SVN_HISTORY.append(projectCheckOutDir)
199  else :
200    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
201
202  #---------------------------------------------------------------------
203  # If there are third party apps, then get these apps
204  #---------------------------------------------------------------------
205  if 'noThirdParty' in configuration :
206    if not configuration['noThirdParty'] :
207      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
208      if os.path.isdir(thirdPartyBaseDir) :
209        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
210          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
211          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
212          for d in thirdPartyDirs :
213            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
214            install3rdPartyCmd=os.path.join(".","get."+d)
215            os.chdir(thirdPartyDir)
216            # If the install command has been updated since the last
217            # install, then do a new install
218            if os.path.isfile('NBinstalldone') :
219              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
220                os.remove('NBinstalldone')
221            if not os.path.isfile('NBinstalldone') :
222              if os.path.isfile(install3rdPartyCmd) :
223                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
224                installReturn = NBosCommand.run(install3rdPartyCmd)
225                if installReturn['returnCode'] :
226                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
227                else :
228                  f=open('NBinstalldone','w')
229                  f.close()
230                writeResults(installReturn,install3rdPartyCmd) 
231            else :
232              NBlogMessages.writeMessage('  skipped a new download of '+d)
233        else :
234          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
235
236  #---------------------------------------------------------------------
237  # Create the build directory if it doesn't exist
238  # and remove file that indicates prior run tested ok.
239  #---------------------------------------------------------------------
240  if not os.path.isdir(fullBuildDir) : 
241    os.makedirs(fullBuildDir)
242
243  #---------------------------------------------------------------------
244  # Source is now available, so now it is time to run config
245  #---------------------------------------------------------------------
246  if configuration['buildMethod']=='unixConfig' :
247    skipOptions=''
248
249    if 'SkipProjects' in configuration :
250      skipOptions+=configuration['SkipProjects']
251
252    # If needed create option for skipping 3rd party code
253    needSkip3PartySkipOptions=False
254    if 'noThirdParty' not in configuration : 
255      needSkip3PartySkipOptions=True
256    elif configuration['noThirdParty'] :
257      needSkip3PartySkipOptions=True
258    if needSkip3PartySkipOptions :
259      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
260      if os.path.isdir(thirdPartyBaseDir) :
261        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
262        for d in thirdPartyDirs :
263          skipOptions+=' ThirdParty/'+d
264
265    if skipOptions!='' :
266      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
267
268    os.chdir(fullBuildDir)
269    NBlogMessages.writeMessage('  cd '+fullBuildDir)
270
271    # Assemble all config options together and create config command
272    configOptions ="-C "+configuration['configOptions']['unique']
273    configOptions+=configuration['configOptions']['invariant']
274    configOptions+=skipOptions
275    configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
276
277    # If config was previously run, then no need to run again.
278  #  if NBcheckResult.didConfigRunOK() :
279  #    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
280  #  else :
281    NBlogMessages.writeMessage("  "+configCmd)
282
283    # Finally run config
284    result=NBosCommand.run(configCmd)
285    writeResults(result,'config') 
286
287    # Check if configure worked
288    if result['returnCode'] != 0 :
289        error_msg = result
290        error_msg['configure flags']=configOptions
291        error_msg['svn version']=configuration['svnVersion']
292        # Add contents of log file to message
293        logFileName = 'config.log'
294        if os.path.isfile(logFileName) :
295          logFilePtr = open(logFileName,'r')
296          error_msg['config.log'] = logFilePtr.read()
297          logFilePtr.close()
298        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
299        return
300
301    #---------------------------------------------------------------------
302    # Run make part of build
303    #---------------------------------------------------------------------
304    NBlogMessages.writeMessage( '  make' )
305    result=NBosCommand.run('make')
306    writeResults(result,'make') 
307
308    # Check if make worked
309    if result['returnCode'] != 0 :
310      result['configure flags']=configOptions
311      result['svn version']=configuration['svnVersion']
312      NBemail.sendCmdMsgs(configuration['project'],result,'make')
313      return
314
315  if configuration['buildMethod']=='msSln' :
316    #---------------------------------------------------------------------
317    # Source is now available, so now it is time to run vcbuild
318    #---------------------------------------------------------------------
319   
320    os.chdir(projectCheckOutDir)
321    NBlogMessages.writeMessage('  cd '+projectCheckOutDir)
322
323    if configuration.has_key('slnFile') :
324      slnFileName = os.path.join(projectCheckOutDir,configuration['slnFile'])
325    else :
326      slnFileName = os.path.join(projectCheckOutDir,\
327                          configuration['project'],\
328                          'MSVisualStudio',\
329                          'v8',\
330                          configuration['project']) +\
331                    '.sln'
332    if not os.path.isfile(slnFileName) :
333      NBlogMessages.writeMessage("  Solution file does not exist: "+slnFileName)
334      return
335
336    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
337             
338
339    NBlogMessages.writeMessage("  "+vcbuild)
340
341    # Finally run vcbuild
342    result=NBosCommand.run(vcbuild)
343    writeResults(result,'vcbuild') 
344
345    # Check if vcbuild worked
346    if result['returnCode'] != 0 :
347        error_msg = result
348        error_msg['svn version']=configuration['svnVersion']
349        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
350        return
351
352  #---------------------------------------------------------------------
353  # Run all test executables
354  #---------------------------------------------------------------------
355  if "test" in configuration :
356    for t in range( len(configuration['test']) ) :
357      testRelDir=configuration['test'][t]['dir']
358      testDir = os.path.join(fullBuildDir,testRelDir)
359      testCmd=configuration['test'][t]['cmd']
360      if not os.path.isdir(testDir) :
361        NBlogMessages.writeMessage('  Directory to run test from does not exist:')
362        NBlogMessages.writeMessage('    Intended directory: '+testDir)
363        NBlogMessages.writeMessage('    Intended command: '+testCmd)
364        continue
365      os.chdir(testDir)
366      NBlogMessages.writeMessage('  cd '+testDir)
367
368      NBlogMessages.writeMessage( '  '+testCmd )
369      result=NBosCommand.run(testCmd)
370      writeResults(result,testCmd) 
371       
372      for testFunc in configuration['test'][t]['check'] :
373        testResultFail=testFunc(result,configuration['project'])
374        if testResultFail :
375          result['svn version']=configuration['svnVersion']
376          result['test']=testResultFail
377          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
378          return
379
380
381  #---------------------------------------------------------------------
382  # Everything build and all tests passed.
383  #---------------------------------------------------------------------
384  os.chdir(fullBuildDir)
385  f=open('NBallTestsPassed','w')
386  f.close()
Note: See TracBrowser for help on using the repository browser.