source: branches/testScripts/NBbuildConfig.py @ 696

Last change on this file since 696 was 696, checked in by jpfasano, 12 years ago

updated output message

  • Property svn:eol-style set to native
File size: 13.2 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5
6import NBlogMessages
7import NBemail
8import NBosCommand
9import NBsvnCommand
10import NBcheckResult
11
12#---------------------------------------------------------------------
13# Keep history so same project is not repeatedly getting code from
14# subversion repository.
15#---------------------------------------------------------------------
16SVN_HISTORY = []
17THIRD_PARTY_HISTORY = []
18
19#------------------------------------------------------------------------
20#  Given a configuration, build and test it.
21#
22#  configuration['project']= name of project.
23#   examples: "Clp", "Ipopt"
24#
25#  configuration['rootDir']= root directory of nightlyBuild.
26#   This is where the project will be checked out from svn, and
27#   where the code will be compiled.  This directory must already
28#   exist.  If the testing requires, it needs to contain Netlib & miplib3
29#
30#  configuration['svnVersion']= svn version to be built.
31#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
32#
33#  configuration['noThirdParty']=True or False (optional). If False
34#   then 3rd party code will be used. If not specified then 3rd part
35#   code will be skipped.
36#
37#  configuration['configOptions']: Parameters to be passed to configure.
38#   The -C option and the options for skipping 3rd party code do not
39#   need to be specified.  These will be generated by this function.
40#   There are two types of configOptions to be specified.
41#  configuration['configOptions']['unique']= These are options that
42#   distinguish different build configurations.  These options are used
43#   to generate the vpath directory name where the code will be built.
44#   Examples are: "", "--enable-debug" "--enable-parrallel"
45#  configuration['configOptions']['invariant']= These are options that
46#   that are the same for every build configuration so they don't need
47#   to be part of the vpath directory name.
48#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
49#
50#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
51#    from the build.
52#    examples: "Ipopt", "Ipopt DyLP"
53#
54#  configuration['checkMakeTest']= function to be called to determine
55#    if "make test" ran correctly
56#
57#  configuration['unitTest']= undefined or dictionary D where
58#    D['path']= relative path were unitTest is to be run from.
59#    D['command']= command to be issued to run unitTest
60#    D['checkUnitTest']= function to be called to determine if unitTest
61#       ran correctly.
62#------------------------------------------------------------------------
63def run(configuration) :
64  NBlogMessages.writeMessage( configuration['project'] )
65
66  # Create svn checkout target directory name
67  svnVersionFlattened=configuration['svnVersion'].replace('/','-')
68
69  #---------------------------------------------------------------------
70  # Create names of directory where source is located and
71  # and were object, libs and executables are located (vpath).
72  # To compute vpath directory, the ./configure options need to be
73  # generated.
74  #---------------------------------------------------------------------
75  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
76  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
77
78  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
79                 configuration['project']+'/'+\
80                 configuration['svnVersion']
81
82  vpathDir=''
83
84  if 'SkipProjects' in configuration :
85    vpathDir+="No"+configuration['SkipProjects']
86  if 'noThirdParty' in configuration : 
87    if configuration['noThirdParty'] :
88      vpathDir+='-NoThirdParty'
89  vpathDir=svnVersionFlattened+\
90          configuration['configOptions']['unique']+\
91          vpathDir
92  vpathDir=vpathDir.replace(' ','')
93  vpathDir=vpathDir.replace('"','')
94  vpathDir=vpathDir.replace("'",'')
95  vpathDir=vpathDir.replace('--enable','')
96  if vpathDir==svnVersionFlattened : vpathDir+='-default'
97
98  fullVpathDir = os.path.join(projectBaseDir,vpathDir)
99
100  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
101  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
102  NBlogMessages.writeMessage('  Vpath directory: '+fullVpathDir)
103 
104
105  #---------------------------------------------------------------------
106  # Create the vpath directory if it doesn't exist
107  # and remove file that indicates prior run tested ok.
108  #---------------------------------------------------------------------
109  if not os.path.isdir(fullVpathDir) : 
110    os.makedirs(fullVpathDir)
111
112  #---------------------------------------------------------------------
113  # If nothing has changed and the prior run tested OK, then there
114  # is no need to do anything.
115  #---------------------------------------------------------------------
116  os.chdir(fullVpathDir)
117
118  if os.path.isfile('NBallTestsPassed') : 
119    msg=NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir)
120    if not msg:
121      # Previous run ran fine, and nothing has changed.
122      NBlogMessages.writeMessage('  No changes since previous successfull run')
123      return
124    NBlogMessages.writeMessage('  '+msg)
125    os.remove('NBallTestsPassed')
126  else :
127    NBlogMessages.writeMessage('  No record of all tests having passed')
128
129  #---------------------------------------------------------------------
130  # svn checkout or update the project
131  #---------------------------------------------------------------------
132  # Don't get source from subversion if previously done
133  if projectCheckOutDir not in SVN_HISTORY :
134    if not os.path.isdir(projectBaseDir) :
135      os.makedirs(projectBaseDir)
136    if not os.path.isdir(projectCheckOutDir) :
137      svnCmd='svn ' +\
138           'checkout ' +\
139           svnCheckOutUrl +\
140           ' '+svnVersionFlattened
141      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
142        return
143    else :
144      svnCmd='svn update'
145      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
146        return
147    SVN_HISTORY.append(projectCheckOutDir)
148  else :
149    NBlogMessages.writeMessage('  "svn update" skipped. nightlyBuild has already updated for prior build configuration')
150
151  #---------------------------------------------------------------------
152  # If there are third party apps, then get these apps
153  #---------------------------------------------------------------------
154  if 'noThirdParty' in configuration :
155    if not configuration['noThirdParty'] :
156      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
157      if os.path.isdir(thirdPartyBaseDir) :
158        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
159          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
160          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
161          for d in thirdPartyDirs :
162            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
163            install3rdPartyCmd=os.path.join(".","get."+d)
164            os.chdir(thirdPartyDir)
165            # If the install command has been updated since the last
166            # install, then do a new install
167            if os.path.isfile('NBinstalldone') :
168              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
169                os.remove('NBinstalldone')
170            if not os.path.isfile('NBinstalldone') :
171              if os.path.isfile(install3rdPartyCmd) :
172                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
173                installReturn = NBosCommand.run(install3rdPartyCmd)
174                if installReturn['returnCode'] :
175                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
176                else :
177                  f=open('NBinstalldone','w')
178                  f.close()
179                stdoutfile=open(thirdPartyDir+'/NBstdout','w')
180                stdoutfile.write(installReturn['stdout'])
181                stdoutfile.close()
182                stderrfile=open(thirdPartyDir+'/NBstderr','w')
183                stderrfile.write(installReturn['stderr'])
184                stderrfile.close()
185            else :
186              NBlogMessages.writeMessage('  skipped a new download of '+d)
187        else :
188          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
189
190  #---------------------------------------------------------------------
191  # Source is now available, so now it is time to run config
192  #---------------------------------------------------------------------
193  skipOptions=''
194
195  if 'SkipProjects' in configuration :
196    skipOptions+=configuration['SkipProjects']
197
198  # If needed create option for skipping 3rd party code
199  needSkip3PartySkipOptions=False
200  if 'noThirdParty' not in configuration : 
201    needSkip3PartySkipOptions=True
202  elif configuration['noThirdParty'] :
203    needSkip3PartySkipOptions=True
204  if needSkip3PartySkipOptions :
205    thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
206    if os.path.isdir(thirdPartyBaseDir) :
207      thirdPartyDirs = os.listdir(thirdPartyBaseDir)
208      for d in thirdPartyDirs :
209        skipOptions+=' ThirdParty/'+d
210
211  if skipOptions!='' :
212    skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
213
214  os.chdir(fullVpathDir)
215  NBlogMessages.writeMessage('  Current directory: '+fullVpathDir)
216 
217  # Assemble all config options together and create config command
218  configOptions ="-C "+configuration['configOptions']['unique']
219  configOptions+=configuration['configOptions']['unique']
220  configOptions+=configuration['configOptions']['invariant']
221  configOptions+=skipOptions
222  configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
223
224
225
226  # If config was previously run, then no need to run again.
227  if NBcheckResult.didConfigRunOK() :
228    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
229  else :
230    NBlogMessages.writeMessage("  "+configCmd)
231
232    # Finally run config
233    result=NBosCommand.run(configCmd)
234    stdoutfile=open('NBconfig.stdout','w')
235    stdoutfile.write(result['stdout'])
236    stdoutfile.close()
237    stderrfile=open('NBconfig.stderr','w')
238    stderrfile.write(result['stderr'])
239    stderrfile.close()
240
241    # Check if configure worked
242    if result['returnCode'] != 0 :
243        error_msg = result
244        # Add contents of log file to message
245        logFileName = 'config.log'
246        if os.path.isfile(logFileName) :
247          logFilePtr = open(logFileName,'r')
248          error_msg['config.log']  = "config.log contains: \n" 
249          error_msg['config.log'] += logFilePtr.read()
250          logFilePtr.close()
251        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
252        return
253
254  #---------------------------------------------------------------------
255  # Run make part of build
256  #---------------------------------------------------------------------
257  NBlogMessages.writeMessage( '  make' )
258  result=NBosCommand.run('make')
259  stdoutfile=open('NBmake.stdout','w')
260  stdoutfile.write(result['stdout'])
261  stdoutfile.close()
262  stderrfile=open('NBmake.stderr','w')
263  stderrfile.write(result['stderr'])
264  stderrfile.close()
265
266  # Check if make worked
267  if result['returnCode'] != 0 :
268    NBemail.sendCmdMsgs(configuration['project'],result,'make')
269    return
270
271  #---------------------------------------------------------------------
272  # Run 'make test' part of build
273  #---------------------------------------------------------------------
274  NBlogMessages.writeMessage( '  make test' )
275  result=NBosCommand.run('make test')
276  stdoutfile=open('NBmaketest.stdout','w')
277  stdoutfile.write(result['stdout'])
278  stdoutfile.close()
279  stderrfile=open('NBmaketest.stderr','w')
280  stderrfile.write(result['stderr'])
281  stderrfile.close()
282
283  # Check if 'make test' worked
284  didMakeTestFail=configuration['checkMakeTest'](result,configuration['project'],"make test")
285  if didMakeTestFail :
286    result['make test']=didMakeTestFail
287    NBemail.sendCmdMsgs(configuration['project'],result,"make test")
288    return
289
290  #---------------------------------------------------------------------
291  # Run unitTest if available and different from 'make test'
292  #---------------------------------------------------------------------
293  if "unitTest" in configuration :
294    unitTestRelPath=configuration['unitTest']['path']
295    unitTestPath = os.path.join(fullVpathDir,unitTestRelPath)
296    os.chdir(unitTestPath)
297    NBlogMessages.writeMessage('  Current directory: '+unitTestPath)
298
299    unitTestCmdTemplate=configuration['unitTest']['command']
300
301    dataBaseDir=os.path.join(configuration['rootDir'],'Data')
302    netlibDir=os.path.join(dataBaseDir,'Netlib')
303    miplib3Dir=os.path.join(dataBaseDir,'miplib3')
304
305    unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
306    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
307
308    NBlogMessages.writeMessage( '  '+unitTestCmd )
309    result=NBosCommand.run(unitTestCmd)
310    stdoutfile=open('NBunittest.stdout','w')
311    stdoutfile.write(result['stdout'])
312    stdoutfile.close()
313    stderrfile=open('NBunittest.stderr','w')
314    stderrfile.write(result['stderr'])
315    stderrfile.close()
316
317    didUnitTestFail=configuration['unitTest']['checkUnitTest'](result,configuration['project'],unitTestCmdTemplate)
318    if didUnitTestFail :
319      result['unitTest']=didUnitTestFail
320      NBemail.sendCmdMsgs(p,result,unitTestCmd)
321      return
322
323
324  #---------------------------------------------------------------------
325  # Everything build and all tests passed.
326  #---------------------------------------------------------------------
327  os.chdir(fullVpathDir)
328  f=open('NBallTestsPassed','w')
329  f.close()
Note: See TracBrowser for help on using the repository browser.