source: branches/testScripts/NBbuildConfig.py @ 730

Last change on this file since 730 was 730, checked in by jpfasano, 12 years ago

Mofified NBcheckResult so that it is a bunch of small functions that check for one specific thing.
NBprojectConfig now defines a dictionary where the key is the project.
The value is a list of functions that need to be run to check the success of a particular test run.
The code still needs to be cleaned up, but it seems to be running and was time to do a check-in so I can move it to other platforms for testing.

  • Property svn:eol-style set to native
File size: 14.6 KB
Line 
1#! /usr/bin/env python
2
3#------------------------------------------------------------------------
4# This file is distributed under the Common Public License.
5# It is part of the BuildTools project in COIN-OR (www.coin-or.org)
6#------------------------------------------------------------------------
7
8import os
9import sys
10
11import NBlogMessages
12import NBemail
13import NBosCommand
14import NBsvnCommand
15import NBcheckResult
16
17#---------------------------------------------------------------------
18# Keep history so same project is not repeatedly getting code from
19# subversion repository.
20#---------------------------------------------------------------------
21SVN_HISTORY = []
22THIRD_PARTY_HISTORY = []
23
24#------------------------------------------------------------------------
25#  Given a configuration, build and test it.
26#
27#  configuration['project']= name of project.
28#   examples: "Clp", "Ipopt"
29#
30#  configuration['rootDir']= root directory of nightlyBuild.
31#   This is where the project will be checked out from svn, and
32#   where the code will be compiled.  This directory must already
33#   exist.  If the testing requires, it needs to contain Netlib & miplib3
34#
35#  configuration['svnVersion']= svn version to be built.
36#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
37#
38#  configuration['noThirdParty']=True or False (optional). If False
39#   then 3rd party code will be used. If not specified then 3rd part
40#   code will be skipped.
41#
42#  configuration['configOptions']: Parameters to be passed to configure.
43#   The -C option and the options for skipping 3rd party code do not
44#   need to be specified.  These will be generated by this function.
45#   There are two types of configOptions to be specified.
46#  configuration['configOptions']['unique']= These are options that
47#   distinguish different build configurations.  These options are used
48#   to generate the vpath directory name where the code will be built.
49#   Examples are: "", "--enable-debug" "--enable-parrallel"
50#  configuration['configOptions']['invariant']= These are options that
51#   that are the same for every build configuration so they don't need
52#   to be part of the vpath directory name.
53#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
54#
55#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
56#    from the build.
57#    examples: "Ipopt", "Ipopt DyLP"
58#
59#  configuration['checkMakeTest']= function to be called to determine
60#    if "make test" ran correctly
61#
62#  configuration['unitTest']= undefined or dictionary D where
63#    D['path']= relative path were unitTest is to be run from.
64#    D['command']= command to be issued to run unitTest
65#    D['checkUnitTest']= function to be called to determine if unitTest
66#       ran correctly.
67#------------------------------------------------------------------------
68def run(configuration) :
69  NBlogMessages.writeMessage( configuration['project'] )
70
71  # Create svn checkout target directory name
72  svnVersionFlattened=configuration['svnVersion'].replace('/','-')
73
74  #---------------------------------------------------------------------
75  # Create names of directory where source is located and
76  # and were object, libs and executables are located (vpath).
77  # To compute vpath directory, the ./configure options need to be
78  # generated.
79  #---------------------------------------------------------------------
80  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
81  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
82
83  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
84                 configuration['project']+'/'+\
85                 configuration['svnVersion']
86
87  vpathDir=''
88
89  if 'SkipProjects' in configuration :
90    vpathDir+="No"+configuration['SkipProjects']
91  if 'noThirdParty' in configuration : 
92    if configuration['noThirdParty'] :
93      vpathDir+='-NoThirdParty'
94  vpathDir=svnVersionFlattened+\
95          configuration['configOptions']['unique']+\
96          vpathDir
97  vpathDir=vpathDir.replace(' ','')
98  vpathDir=vpathDir.replace('"','')
99  vpathDir=vpathDir.replace("'",'')
100  vpathDir=vpathDir.replace('--enable','')
101  if vpathDir==svnVersionFlattened : vpathDir+='-default'
102
103  fullVpathDir = os.path.join(projectBaseDir,vpathDir)
104
105  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
106  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
107  NBlogMessages.writeMessage('  Vpath directory: '+fullVpathDir)
108 
109
110  #---------------------------------------------------------------------
111  # Create the vpath directory if it doesn't exist
112  # and remove file that indicates prior run tested ok.
113  #---------------------------------------------------------------------
114  if not os.path.isdir(fullVpathDir) : 
115    os.makedirs(fullVpathDir)
116
117  #---------------------------------------------------------------------
118  # If nothing has changed and the prior run tested OK, then there
119  # is no need to do anything.
120  #---------------------------------------------------------------------
121
122  os.chdir(fullVpathDir)
123  if os.path.isfile('NBallTestsPassed') : 
124    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
125    if not msg:
126      # Previous run ran fine, and nothing has changed.
127      NBlogMessages.writeMessage('  No changes since previous successfull run')
128      return
129    NBlogMessages.writeMessage('  '+msg)
130   
131    # Must remove file NBallTestsPassed from all vpath directories that
132    # use projectCheckoutDir for their source code. This is to ensure
133    # that make will be run in all the vpath dirs after "svn update"
134    dirs = os.listdir("..")
135    for d in dirs :
136      if d.startswith(configuration['svnVersion']) and \
137         d!=configuration['svnVersion']:
138        fileToBeRemoved=os.path.join("..",d,'NBallTestsPassed')
139        if os.path.isfile(fileToBeRemoved) :
140          os.remove(fileToBeRemoved)
141  else :
142    NBlogMessages.writeMessage('  No record of all tests having passed')
143
144  #---------------------------------------------------------------------
145  # svn checkout or update the project
146  #---------------------------------------------------------------------
147  # Don't get source from subversion if previously done
148  if projectCheckOutDir not in SVN_HISTORY :
149    if not os.path.isdir(projectBaseDir) :
150      os.makedirs(projectBaseDir)
151    if not os.path.isdir(projectCheckOutDir) :
152      svnCmd='svn ' +\
153           'checkout ' +\
154           svnCheckOutUrl +\
155           ' '+svnVersionFlattened
156      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
157        return
158    else :
159      svnCmd='svn update'
160      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
161        return
162    SVN_HISTORY.append(projectCheckOutDir)
163  else :
164    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
165
166  #---------------------------------------------------------------------
167  # If there are third party apps, then get these apps
168  #---------------------------------------------------------------------
169  if 'noThirdParty' in configuration :
170    if not configuration['noThirdParty'] :
171      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
172      if os.path.isdir(thirdPartyBaseDir) :
173        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
174          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
175          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
176          for d in thirdPartyDirs :
177            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
178            install3rdPartyCmd=os.path.join(".","get."+d)
179            os.chdir(thirdPartyDir)
180            # If the install command has been updated since the last
181            # install, then do a new install
182            if os.path.isfile('NBinstalldone') :
183              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
184                os.remove('NBinstalldone')
185            if not os.path.isfile('NBinstalldone') :
186              if os.path.isfile(install3rdPartyCmd) :
187                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
188                installReturn = NBosCommand.run(install3rdPartyCmd)
189                if installReturn['returnCode'] :
190                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
191                else :
192                  f=open('NBinstalldone','w')
193                  f.close()
194                stdoutfile=open(thirdPartyDir+'/NBstdout','w')
195                stdoutfile.write(installReturn['stdout'])
196                stdoutfile.close()
197                stderrfile=open(thirdPartyDir+'/NBstderr','w')
198                stderrfile.write(installReturn['stderr'])
199                stderrfile.close()
200            else :
201              NBlogMessages.writeMessage('  skipped a new download of '+d)
202        else :
203          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
204
205  #---------------------------------------------------------------------
206  # Source is now available, so now it is time to run config
207  #---------------------------------------------------------------------
208  skipOptions=''
209
210  if 'SkipProjects' in configuration :
211    skipOptions+=configuration['SkipProjects']
212
213  # If needed create option for skipping 3rd party code
214  needSkip3PartySkipOptions=False
215  if 'noThirdParty' not in configuration : 
216    needSkip3PartySkipOptions=True
217  elif configuration['noThirdParty'] :
218    needSkip3PartySkipOptions=True
219  if needSkip3PartySkipOptions :
220    thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
221    if os.path.isdir(thirdPartyBaseDir) :
222      thirdPartyDirs = os.listdir(thirdPartyBaseDir)
223      for d in thirdPartyDirs :
224        skipOptions+=' ThirdParty/'+d
225
226  if skipOptions!='' :
227    skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
228
229  os.chdir(fullVpathDir)
230  NBlogMessages.writeMessage('  Current directory: '+fullVpathDir)
231
232  # Assemble all config options together and create config command
233  configOptions ="-C "+configuration['configOptions']['unique']
234  configOptions+=configuration['configOptions']['invariant']
235  configOptions+=skipOptions
236  configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
237
238
239
240  # If config was previously run, then no need to run again.
241  if NBcheckResult.didConfigRunOK() :
242    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
243  else :
244    NBlogMessages.writeMessage("  "+configCmd)
245
246    # Finally run config
247    result=NBosCommand.run(configCmd)
248    stdoutfile=open('NBconfig.stdout','w')
249    stdoutfile.write(result['stdout'])
250    stdoutfile.close()
251    stderrfile=open('NBconfig.stderr','w')
252    stderrfile.write(result['stderr'])
253    stderrfile.close()
254
255    # Check if configure worked
256    if result['returnCode'] != 0 :
257        error_msg = result
258        error_msg['configure flags']=configOptions
259        error_msg['svn version']=configuration['svnVersion']
260        # Add contents of log file to message
261        logFileName = 'config.log'
262        if os.path.isfile(logFileName) :
263          logFilePtr = open(logFileName,'r')
264          error_msg['config.log'] = logFilePtr.read()
265          logFilePtr.close()
266        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
267        return
268
269  #---------------------------------------------------------------------
270  # Run make part of build
271  #---------------------------------------------------------------------
272  NBlogMessages.writeMessage( '  make' )
273  result=NBosCommand.run('make')
274  stdoutfile=open('NBmake.stdout','w')
275  stdoutfile.write(result['stdout'])
276  stdoutfile.close()
277  stderrfile=open('NBmake.stderr','w')
278  stderrfile.write(result['stderr'])
279  stderrfile.close()
280
281  # Check if make worked
282  if result['returnCode'] != 0 :
283    result['configure flags']=configOptions
284    result['svn version']=configuration['svnVersion']
285    NBemail.sendCmdMsgs(configuration['project'],result,'make')
286    return
287
288  #---------------------------------------------------------------------
289  # Run 'make test' part of build
290  #---------------------------------------------------------------------
291  NBlogMessages.writeMessage( '  make test' )
292  result=NBosCommand.run('make test')
293  stdoutfile=open('NBmaketest.stdout','w')
294  stdoutfile.write(result['stdout'])
295  stdoutfile.close()
296  stderrfile=open('NBmaketest.stderr','w')
297  stderrfile.write(result['stderr'])
298  stderrfile.close()
299
300  # Check if 'make test' worked
301  for testFunc in configuration['checkMakeTest'] :
302    testResultFail=testFunc(result,configuration['project'])
303    if testResultFail :
304      result['configure flags']=configOptions
305      result['svn version']=configuration['svnVersion']
306      result['make test']=testResultFail
307      NBemail.sendCmdMsgs(configuration['project'],result,"make test")
308      return
309
310
311  #---------------------------------------------------------------------
312  # Run unitTest if available and different from 'make test'
313  #---------------------------------------------------------------------
314  if "unitTest" in configuration :
315    unitTestRelPath=configuration['unitTest']['path']
316    unitTestPath = os.path.join(fullVpathDir,unitTestRelPath)
317    os.chdir(unitTestPath)
318    NBlogMessages.writeMessage('  Current directory: '+unitTestPath)
319
320    unitTestCmdTemplate=configuration['unitTest']['command']
321
322    dataBaseDir=os.path.join(configuration['rootDir'],'Data')
323    netlibDir=os.path.join(dataBaseDir,'Netlib')
324    miplib3Dir=os.path.join(dataBaseDir,'miplib3')
325
326    unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
327    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
328
329    NBlogMessages.writeMessage( '  '+unitTestCmd )
330    result=NBosCommand.run(unitTestCmd)
331    stdoutfile=open('NBunittest.stdout','w')
332    stdoutfile.write(result['stdout'])
333    stdoutfile.close()
334    stderrfile=open('NBunittest.stderr','w')
335    stderrfile.write(result['stderr'])
336    stderrfile.close()
337
338    #didUnitTestFail=configuration['unitTest']['checkUnitTest'](result,configuration['project'],unitTestCmdTemplate)
339    #if didUnitTestFail :
340    #  result['configure flags']=configOptions
341    #  result['svn version']=configuration['svnVersion']
342    #  result['unitTest']=didUnitTestFail
343    #  NBemail.sendCmdMsgs(p,result,unitTestCmd)
344    #  return
345   
346    for testFunc in configuration['unitTest']['checkUnitTest'] :
347      testResultFail=testFunc(result,configuration['project'])
348      if testResultFail :
349        result['configure flags']=configOptions
350        result['svn version']=configuration['svnVersion']
351        result['unitTest']=testResultFail
352        NBemail.sendCmdMsgs(configuration['project'],result,unitTestCmd)
353        return
354
355
356  #---------------------------------------------------------------------
357  # Everything build and all tests passed.
358  #---------------------------------------------------------------------
359  os.chdir(fullVpathDir)
360  f=open('NBallTestsPassed','w')
361  f.close()
Note: See TracBrowser for help on using the repository browser.