source: branches/testScripts/NBbuildConfig.py @ 740

Last change on this file since 740 was 740, checked in by jpfasano, 12 years ago

Merging code for windows and unix

  • Property svn:eol-style set to native
File size: 15.3 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10
11import NBlogMessages
12import NBemail
13import NBosCommand
14import NBsvnCommand
15import NBcheckResult
16
17#---------------------------------------------------------------------
18# Keep history so same project is not repeatedly getting code from
19# subversion repository.
20#---------------------------------------------------------------------
21SVN_HISTORY = []
22THIRD_PARTY_HISTORY = []
23
24
25#---------------------------------------------------------------------
26# cleanUpName:
27# File and directory names are generated and may contain
28# undesireable characters.
29# Remove these characters from the name
30#---------------------------------------------------------------------
31def cleanUpName(messedUpName) :
32  cleanedUpName=messedUpName
33  cleanedUpName=cleanedUpName.replace('/','-')
34  cleanedUpName=cleanedUpName.replace('\\','-')
35  cleanedUpName=cleanedUpName.replace(' ','')
36  cleanedUpName=cleanedUpName.replace('"','')
37  cleanedUpName=cleanedUpName.replace("'",'')
38  cleanedUpName=cleanedUpName.replace('--enable','')
39  return cleanedUpName
40
41
42#---------------------------------------------------------------------
43# writeResults:
44# After running a command write stdout and stderr to a file
45#---------------------------------------------------------------------
46def writeResults(result,filenameSuffix) :
47  cleanedUpSuffix=cleanUpName(filenameSuffix)
48  stdoutfile=open('NBstdout-'+cleanedUpSuffix,'w')
49  stdoutfile.write(result['stdout'])
50  stdoutfile.close()
51  stderrfile=open('NBstderr-'+cleanedUpSuffix,'w')
52  stderrfile.write(result['stderr'])
53  stderrfile.close()
54 
55#------------------------------------------------------------------------
56#  Given a configuration, build and test it.
57#
58#  configuration['project']= name of project.
59#   examples: "Clp", "Ipopt"
60#
61#  configuration['rootDir']= root directory of nightlyBuild.
62#   This is where the project will be checked out from svn, and
63#   where the code will be compiled.  This directory must already
64#   exist.  If the testing requires, it needs to contain Netlib & miplib3
65#
66#  configuration['svnVersion']= svn version to be built.
67#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
68#
69#  configuration['buildMethod']= Defines method for building code.
70#  Choices are:
71#    msSoln: use microsoft compiler with a solution (sln) file.
72#    unixConfig: use sequence "./configure", "make", "make test"
73#
74#  configuration['noThirdParty']=True or False (optional). If False
75#   then 3rd party code will be used. If not specified then 3rd part
76#   code will be skipped.
77#   Only used if configuration['buildMethod']=='unixConfig'
78#
79#  configuration['configOptions']: Parameters to be passed to configure.
80#   The -C option and the options for skipping 3rd party code do not
81#   need to be specified.  These will be generated by this function.
82#   There are two types of configOptions to be specified.
83#   Only used if configuration['buildMethod']=='unixConfig'
84#  configuration['configOptions']['unique']= These are options that
85#   distinguish different build configurations.  These options are used
86#   to generate the vpath directory name where the code will be built.
87#   Examples are: "", "--enable-debug" "--enable-parrallel"
88#  configuration['configOptions']['invariant']= These are options that
89#   that are the same for every build configuration so they don't need
90#   to be part of the vpath directory name.
91#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
92
93#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
94#    from the build.
95#    examples: "Ipopt", "Ipopt DyLP"
96#
97#  configuration['slnFile']= path and name of solution file if it is not
98#    in the standard location.
99#    Only used if configuration['buildMethod']=='unixConfig'
100#
101#  configuration['test']=vector of triples indicating tests that
102#    are to be run after building the project. Each triple consists
103#    of:
104#    'dir': directory where test command is to be issued.
105#    'cmd': command to be run with any parameters.
106#    'check': vector of functions to be called which will check the
107#           results from running 'cmd' to determine if an error occurred
108#------------------------------------------------------------------------
109def run(configuration) :
110  NBlogMessages.writeMessage( configuration['project'] )
111
112  # Create svn checkout target directory name
113  svnVersionFlattened=cleanUpName(configuration['svnVersion'])
114
115  #---------------------------------------------------------------------
116  # Create names of directory where source is located and
117  # and were object, libs and executables are located (build directory)
118  # To compute build directory, the ./configure options need to be
119  # generated.
120  #---------------------------------------------------------------------
121  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
122  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
123
124  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
125                 configuration['project']+'/'+\
126                 configuration['svnVersion']
127
128  buildDir=''
129
130  if configuration['buildMethod']=='unixConfig' :
131    if 'SkipProjects' in configuration :
132      buildDir+="No"+configuration['SkipProjects']
133    if 'noThirdParty' in configuration : 
134      if configuration['noThirdParty'] :
135        buildDir+='-NoThirdParty'
136    buildDir=svnVersionFlattened+\
137            configuration['configOptions']['unique']+\
138            buildDir
139    buildDir=cleanUpName(buildDir)
140    if buildDir==svnVersionFlattened : buildDir+='-default'
141
142  fullBuildDir = os.path.join(projectBaseDir,buildDir)
143
144  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
145  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
146  NBlogMessages.writeMessage('  Build directory: '+fullBuildDir)
147 
148
149  #---------------------------------------------------------------------
150  # Create the build directory if it doesn't exist
151  # and remove file that indicates prior run tested ok.
152  #---------------------------------------------------------------------
153  if not os.path.isdir(fullBuildDir) : 
154    os.makedirs(fullBuildDir)
155
156  #---------------------------------------------------------------------
157  # If nothing has changed and the prior run tested OK, then there
158  # is no need to do anything.
159  #---------------------------------------------------------------------
160
161  os.chdir(fullBuildDir)
162  if os.path.isfile('NBallTestsPassed') : 
163    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
164    if not msg:
165      # Previous run ran fine, and nothing has changed.
166      NBlogMessages.writeMessage('  No changes since previous successfull run')
167      return
168    NBlogMessages.writeMessage('  '+msg)
169   
170    # Must remove file NBallTestsPassed from all build directories that
171    # use projectCheckoutDir for their source code. This is to ensure
172    # that make will be run in all the build dirs after "svn update"
173    dirs = os.listdir("..")
174    for d in dirs :
175      if d.startswith(configuration['svnVersion']) :
176        fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
177        if os.path.isfile(fileToBeRemoved) :
178          os.remove(fileToBeRemoved)
179  else :
180    NBlogMessages.writeMessage('  No record of all tests having passed')
181
182  #---------------------------------------------------------------------
183  # svn checkout or update the project
184  #---------------------------------------------------------------------
185  # Don't get source from subversion if previously done
186  if projectCheckOutDir not in SVN_HISTORY :
187    if not os.path.isdir(projectBaseDir) :
188      os.makedirs(projectBaseDir)
189    if not os.path.isdir(projectCheckOutDir) :
190      svnCmd='svn ' +\
191           'checkout ' +\
192           svnCheckOutUrl +\
193           ' '+svnVersionFlattened
194      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
195        return
196    else :
197      svnCmd='svn update'
198      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
199        return
200    SVN_HISTORY.append(projectCheckOutDir)
201  else :
202    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
203
204  #---------------------------------------------------------------------
205  # If there are third party apps, then get these apps
206  #---------------------------------------------------------------------
207  if 'noThirdParty' in configuration :
208    if not configuration['noThirdParty'] :
209      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
210      if os.path.isdir(thirdPartyBaseDir) :
211        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
212          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
213          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
214          for d in thirdPartyDirs :
215            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
216            install3rdPartyCmd=os.path.join(".","get."+d)
217            os.chdir(thirdPartyDir)
218            # If the install command has been updated since the last
219            # install, then do a new install
220            if os.path.isfile('NBinstalldone') :
221              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
222                os.remove('NBinstalldone')
223            if not os.path.isfile('NBinstalldone') :
224              if os.path.isfile(install3rdPartyCmd) :
225                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
226                installReturn = NBosCommand.run(install3rdPartyCmd)
227                if installReturn['returnCode'] :
228                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
229                else :
230                  f=open('NBinstalldone','w')
231                  f.close()
232                writeResults(result,install3rdPartyCmd) 
233            else :
234              NBlogMessages.writeMessage('  skipped a new download of '+d)
235        else :
236          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
237
238  #---------------------------------------------------------------------
239  # Source is now available, so now it is time to run config
240  #---------------------------------------------------------------------
241  if configuration['buildMethod']=='unixConfig' :
242    skipOptions=''
243
244    if 'SkipProjects' in configuration :
245      skipOptions+=configuration['SkipProjects']
246
247    # If needed create option for skipping 3rd party code
248    needSkip3PartySkipOptions=False
249    if 'noThirdParty' not in configuration : 
250      needSkip3PartySkipOptions=True
251    elif configuration['noThirdParty'] :
252      needSkip3PartySkipOptions=True
253    if needSkip3PartySkipOptions :
254      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
255      if os.path.isdir(thirdPartyBaseDir) :
256        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
257        for d in thirdPartyDirs :
258          skipOptions+=' ThirdParty/'+d
259
260    if skipOptions!='' :
261      skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
262
263    os.chdir(fullBuildDir)
264    NBlogMessages.writeMessage('  Current directory: '+fullBuildDir)
265
266    # Assemble all config options together and create config command
267    configOptions ="-C "+configuration['configOptions']['unique']
268    configOptions+=configuration['configOptions']['invariant']
269    configOptions+=skipOptions
270    configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
271
272    # If config was previously run, then no need to run again.
273  #  if NBcheckResult.didConfigRunOK() :
274  #    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
275  #  else :
276    NBlogMessages.writeMessage("  "+configCmd)
277
278    # Finally run config
279    result=NBosCommand.run(configCmd)
280    writeResults(result,'config') 
281
282    # Check if configure worked
283    if result['returnCode'] != 0 :
284        error_msg = result
285        error_msg['configure flags']=configOptions
286        error_msg['svn version']=configuration['svnVersion']
287        # Add contents of log file to message
288        logFileName = 'config.log'
289        if os.path.isfile(logFileName) :
290          logFilePtr = open(logFileName,'r')
291          error_msg['config.log'] = logFilePtr.read()
292          logFilePtr.close()
293        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
294        return
295
296    #---------------------------------------------------------------------
297    # Run make part of build
298    #---------------------------------------------------------------------
299    NBlogMessages.writeMessage( '  make' )
300    result=NBosCommand.run('make')
301    writeResults(result,'make') 
302
303    # Check if make worked
304    if result['returnCode'] != 0 :
305      result['configure flags']=configOptions
306      result['svn version']=configuration['svnVersion']
307      NBemail.sendCmdMsgs(configuration['project'],result,'make')
308      return
309
310  if configuration['buildMethod']=='msSln' :
311    #---------------------------------------------------------------------
312    # Source is now available, so now it is time to run vcbuild
313    #---------------------------------------------------------------------
314   
315    os.chdir(projectCheckOutDir)
316    NBlogMessages.writeMessage('  Current directory: '+projectCheckOutDir)
317
318    if configuration.has_key('slnFile') :
319      slnFileName = os.path.join(projectCheckOutDir,configuration['slnFile'])
320    else :
321      slnFileName = os.path.join(projectCheckOutDir,\
322                          configuration['project'],\
323                          'MSVisualStudio',\
324                          'v8',\
325                          configuration['project']) +\
326                    '.sln'
327    if not os.path.isfile(slnFileName) :
328      NBlogMessages.writeMessage("  Solution file does not exist: "+slnFileName)
329      return
330
331    vcbuild='vcbuild /u ' + slnFileName + ' $ALL'
332             
333
334    NBlogMessages.writeMessage("  "+vcbuild)
335
336    # Finally run vcbuild
337    result=NBosCommand.run(vcbuild)
338    writeResults(result,'vcbuild') 
339
340    # Check if vcbuild worked
341    if result['returnCode'] != 0 :
342        error_msg = result
343        error_msg['svn version']=configuration['svnVersion']
344        NBemail.sendCmdMsgs(configuration['project'],error_msg,vcbuild)
345        return
346
347  #---------------------------------------------------------------------
348  # Run all test executables
349  #---------------------------------------------------------------------
350  if "test" in configuration :
351    for t in range( len(configuration['test']) ) :
352      testRelDir=configuration['test'][t]['dir']
353      testDir = os.path.join(fullBuildDir,testRelDir)
354      os.chdir(testDir)
355      NBlogMessages.writeMessage('  Current directory: '+testDir)
356
357      testCmd=configuration['test'][t]['cmd']
358
359      NBlogMessages.writeMessage( '  '+testCmd )
360      result=NBosCommand.run(testCmd)
361      writeResults(result,testCmd) 
362       
363      for testFunc in configuration['test'][t]['check'] :
364        testResultFail=testFunc(result,configuration['project'])
365        if testResultFail :
366          result['svn version']=configuration['svnVersion']
367          result['unitTest']=testResultFail
368          NBemail.sendCmdMsgs(configuration['project'],result,testCmd)
369          return
370
371
372  #---------------------------------------------------------------------
373  # Everything build and all tests passed.
374  #---------------------------------------------------------------------
375  os.chdir(fullBuildDir)
376  f=open('NBallTestsPassed','w')
377  f.close()
Note: See TracBrowser for help on using the repository browser.