source: branches/testScripts/NBbuildConfig.py @ 744

Last change on this file since 744 was 743, checked in by jpfasano, 13 years ago

Now remove : and = from generated file names

  • Property svn:eol-style set to native
File size: 15.5 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10
11import NBlogMessages
12import NBemail
13import NBosCommand
14import NBsvnCommand
15import NBcheckResult
16
17#---------------------------------------------------------------------
18# Keep history so same project is not repeatedly getting code from
19# subversion repository.
20#---------------------------------------------------------------------
21SVN_HISTORY = []
22THIRD_PARTY_HISTORY = []
23
24
25#---------------------------------------------------------------------
26# cleanUpName:
27# File and directory names are generated and may contain
28# undesireable characters.
29# Remove these characters from the name
30#---------------------------------------------------------------------
31def cleanUpName(messedUpName) :
32  cleanedUpName=messedUpName
33  cleanedUpName=cleanedUpName.replace('-','')
34  cleanedUpName=cleanedUpName.replace('/','-')
35  cleanedUpName=cleanedUpName.replace('\\','-')
36  cleanedUpName=cleanedUpName.replace(' ','')
37  cleanedUpName=cleanedUpName.replace('"','')
38  cleanedUpName=cleanedUpName.replace("'",'')
39  cleanedUpName=cleanedUpName.replace("=",'-')
40  cleanedUpName=cleanedUpName.replace(":",'')
41  cleanedUpName=cleanedUpName.replace('--enable','')
42  return cleanedUpName
43
44
45#---------------------------------------------------------------------
46# writeResults:
47# After running a command write stdout and stderr to a file
48#---------------------------------------------------------------------
49def writeResults(result,filenameSuffix) :
50  cleanedUpSuffix=cleanUpName(filenameSuffix)
51  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
52  stdoutfile.write(result['stdout'])
53  stdoutfile.close()
54  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
55  stderrfile.write(result['stderr'])
56  stderrfile.close()
57 
58#------------------------------------------------------------------------
59#  Given a configuration, build and test it.
60#
61#  configuration['project']= name of project.
62#   examples: "Clp", "Ipopt"
63#
64#  configuration['rootDir']= root directory of nightlyBuild.
65#   This is where the project will be checked out from svn, and
66#   where the code will be compiled.  This directory must already
67#   exist.  If the testing requires, it needs to contain Netlib & miplib3
68#
69#  configuration['svnVersion']= svn version to be built.
70#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
71#
72#  configuration['buildMethod']= Defines method for building code.
73#  Choices are:
74#    msSoln: use microsoft compiler with a solution (sln) file.
75#    unixConfig: use sequence "./configure", "make", "make test"
76#
77#  configuration['noThirdParty']=True or False (optional). If False
78#   then 3rd party code will be used. If not specified then 3rd part
79#   code will be skipped.
80#   Only used if configuration['buildMethod']=='unixConfig'
81#
82#  configuration['configOptions']: Parameters to be passed to configure.
83#   The -C option and the options for skipping 3rd party code do not
84#   need to be specified.  These will be generated by this function.
85#   There are two types of configOptions to be specified.
86#   Only used if configuration['buildMethod']=='unixConfig'
87#  configuration['configOptions']['unique']= These are options that
88#   distinguish different build configurations.  These options are used
89#   to generate the vpath directory name where the code will be built.
90#   Examples are: "", "--enable-debug" "--enable-parrallel"
91#  configuration['configOptions']['invariant']= These are options that
92#   that are the same for every build configuration so they don't need
93#   to be part of the vpath directory name.
94#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
95# 
96#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
97#    from the build.
98#    examples: "Ipopt", "Ipopt DyLP"
99#
100#  configuration['slnFile']= path and name of solution file if it is not
101#    in the standard location.
102#    Only used if configuration['buildMethod']=='unixConfig'
103#
104#  configuration['test']=vector of triples indicating tests that
105#    are to be run after building the project. Each triple consists
106#    of:
107#    'dir': directory where test command is to be issued.
108#    'cmd': command to be run with any parameters.
109#    'check': vector of functions to be called which will check the
110#           results from running 'cmd' to determine if an error occurred
111#------------------------------------------------------------------------
112def run(configuration) :
113  NBlogMessages.clearMessages()
114  NBlogMessages.writeMessage( configuration['project'] )
115
116  # Create svn checkout target directory name
117  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
118
119  #---------------------------------------------------------------------
120  # Create names of directory where source is located and
121  # and were object, libs and executables are located (build directory)
122  # To compute build directory, the ./configure options need to be
123  # generated.
124  #---------------------------------------------------------------------
125  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
126  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
127
128  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
129                 configuration['project']+'/'+\
130                 configuration['svnVersion']
131
132  buildDir=svnVersionFlattened
133
134  if configuration['buildMethod']=='unixConfig' :
135    buildDir+=configuration['configOptions']['unique']
136    if 'SkipProjects' in configuration :
137      buildDir+="No"+configuration['SkipProjects']
138    if 'noThirdParty' in configuration : 
139      if configuration['noThirdParty'] :
140        buildDir+='-NoThirdParty'
141    buildDir=cleanUpName(buildDir)
142    if buildDir==svnVersionFlattened : buildDir+='-default'
143
144  fullBuildDir = os.path.join(projectBaseDir,buildDir)
145
146  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
147  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
148  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
149 
150
151  #---------------------------------------------------------------------
152  # Create the build directory if it doesn't exist
153  # and remove file that indicates prior run tested ok.
154  #---------------------------------------------------------------------
155  if not os.path.isdir(fullBuildDir) : 
156    os.makedirs(fullBuildDir)
157
158  #---------------------------------------------------------------------
159  # If nothing has changed and the prior run tested OK, then there
160  # is no need to do anything.
161  #---------------------------------------------------------------------
162
163  os.chdir(fullBuildDir)
164  if os.path.isfile('NBallTestsPassed') : 
165    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
166    if not msg:
167      # Previous run ran fine, and nothing has changed.
168      NBlogMessages.writeMessage('  No changes since previous successfull run')
169      return
170    NBlogMessages.writeMessage('  '+msg)
171   
172    # Must remove file NBallTestsPassed from all build directories that
173    # use projectCheckoutDir for their source code. This is to ensure
174    # that make will be run in all the build dirs after "svn update"
175    dirs = os.listdir("..")
176    for d in dirs :
177      if d.startswith(configuration['svnVersion']) :
178        fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
179        if os.path.isfile(fileToBeRemoved) :
180          os.remove(fileToBeRemoved)
181  else :
182    NBlogMessages.writeMessage('  No record of all tests having passed')
183
184  #---------------------------------------------------------------------
185  # svn checkout or update the project
186  #---------------------------------------------------------------------
187  # Don't get source from subversion if previously done
188  if projectCheckOutDir not in SVN_HISTORY :
189    if not os.path.isdir(projectBaseDir) :
190      os.makedirs(projectBaseDir)
191    if not os.path.isdir(projectCheckOutDir) :
192      svnCmd='svn ' +\
193           'checkout ' +\
194           svnCheckOutUrl +\
195           ' '+svnVersionFlattened
196      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
197        return
198    else :
199      svnCmd='svn update'
200      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
201        return
202    SVN_HISTORY.append(projectCheckOutDir)
203  else :
204    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
205
206  #---------------------------------------------------------------------
207  # If there are third party apps, then get these apps
208  #---------------------------------------------------------------------
209  if 'noThirdParty' in configuration :
210    if not configuration['noThirdParty'] :
211      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
212      if os.path.isdir(thirdPartyBaseDir) :
213        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
214          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
215          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
216          for d in thirdPartyDirs :
217            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
218            install3rdPartyCmd=os.path.join(".","get."+d)
219            os.chdir(thirdPartyDir)
220            # If the install command has been updated since the last
221            # install, then do a new install
222            if os.path.isfile('NBinstalldone') :
223              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
224                os.remove('NBinstalldone')
225            if not os.path.isfile('NBinstalldone') :
226              if os.path.isfile(install3rdPartyCmd) :
227                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
228                installReturn = NBosCommand.run(install3rdPartyCmd)
229                if installReturn['returnCode'] :
230                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
231                else :
232                  f=open('NBinstalldone','w')
233                  f.close()
234                writeResults(result,install3rdPartyCmd) 
235            else :
236              NBlogMessages.writeMessage('  skipped a new download of '+d)
237        else :
238          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
239
240  #---------------------------------------------------------------------
241  # Source is now available, so now it is time to run config
242  #---------------------------------------------------------------------
243  if configuration['buildMethod']=='unixConfig' :
244    skipOptions=''
245
246    if 'SkipProjects' in configuration :
247      skipOptions+=configuration['SkipProjects']
248
249    # If needed create option for skipping 3rd party code
250    needSkip3PartySkipOptions=False
251    if 'noThirdParty' not in configuration : 
252      needSkip3PartySkipOptions=True
253    elif configuration['noThirdParty'] :
254      needSkip3PartySkipOptions=True
255    if needSkip3PartySkipOptions :
256      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
257      if os.path.isdir(thirdPartyBaseDir) :
258        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
259        for d in thirdPartyDirs :
260          skipOptions+=' ThirdParty/'+d
261
262    if skipOptions!='' :
263      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
264
265    os.chdir(fullBuildDir)
266    NBlogMessages.writeMessage('  Current directory: '+fullBuildDir)
267
268    # Assemble all config options together and create config command
269    configOptions ="-C "+configuration['configOptions']['unique']
270    configOptions+=configuration['configOptions']['invariant']
271    configOptions+=skipOptions
272    configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
273
274    # If config was previously run, then no need to run again.
275  #  if NBcheckResult.didConfigRunOK() :
276  #    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
277  #  else :
278    NBlogMessages.writeMessage("  "+configCmd)
279
280    # Finally run config
281    result=NBosCommand.run(configCmd)
282    writeResults(result,'config') 
283
284    # Check if configure worked
285    if result['returnCode'] != 0 :
286        error_msg = result
287        error_msg['configure flags']=configOptions
288        error_msg['svn version']=configuration['svnVersion']
289        # Add contents of log file to message
290        logFileName = 'config.log'
291        if os.path.isfile(logFileName) :
292          logFilePtr = open(logFileName,'r')
293          error_msg['config.log'] = logFilePtr.read()
294          logFilePtr.close()
295        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
296        return
297
298    #---------------------------------------------------------------------
299    # Run make part of build
300    #---------------------------------------------------------------------
301    NBlogMessages.writeMessage( '  make' )
302    result=NBosCommand.run('make')
303    writeResults(result,'make') 
304
305    # Check if make worked
306    if result['returnCode'] != 0 :
307      result['configure flags']=configOptions
308      result['svn version']=configuration['svnVersion']
309      NBemail.sendCmdMsgs(configuration['project'],result,'make')
310      return
311
312  if configuration['buildMethod']=='msSln' :
313    #---------------------------------------------------------------------
314    # Source is now available, so now it is time to run vcbuild
315    #---------------------------------------------------------------------
316   
317    os.chdir(projectCheckOutDir)
318    NBlogMessages.writeMessage('  Current directory: '+projectCheckOutDir)
319
320    if configuration.has_key('slnFile') :
321      slnFileName = os.path.join(projectCheckOutDir,configuration['slnFile'])
322    else :
323      slnFileName = os.path.join(projectCheckOutDir,\
324                          configuration['project'],\
325                          'MSVisualStudio',\
326                          'v8',\
327                          configuration['project']) +\
328                    '.sln'
329    if not os.path.isfile(slnFileName) :
330      NBlogMessages.writeMessage("  Solution file does not exist: "+slnFileName)
331      return
332
333    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
334             
335
336    NBlogMessages.writeMessage("  "+vcbuild)
337
338    # Finally run vcbuild
339    result=NBosCommand.run(vcbuild)
340    writeResults(result,'vcbuild') 
341
342    # Check if vcbuild worked
343    if result['returnCode'] != 0 :
344        error_msg = result
345        error_msg['svn version']=configuration['svnVersion']
346        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
347        return
348
349  #---------------------------------------------------------------------
350  # Run all test executables
351  #---------------------------------------------------------------------
352  if "test" in configuration :
353    for t in range( len(configuration['test']) ) :
354      testRelDir=configuration['test'][t]['dir']
355      testDir = os.path.join(fullBuildDir,testRelDir)
356      os.chdir(testDir)
357      NBlogMessages.writeMessage('  Current directory: '+testDir)
358
359      testCmd=configuration['test'][t]['cmd']
360
361      NBlogMessages.writeMessage( '  '+testCmd )
362      result=NBosCommand.run(testCmd)
363      writeResults(result,testCmd) 
364       
365      for testFunc in configuration['test'][t]['check'] :
366        testResultFail=testFunc(result,configuration['project'])
367        if testResultFail :
368          result['svn version']=configuration['svnVersion']
369          result['test']=testResultFail
370          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
371          return
372
373
374  #---------------------------------------------------------------------
375  # Everything build and all tests passed.
376  #---------------------------------------------------------------------
377  os.chdir(fullBuildDir)
378  f=open('NBallTestsPassed','w')
379  f.close()
Note: See TracBrowser for help on using the repository browser.