source: branches/testScripts/NBbuildConfig.py @ 687

Last change on this file since 687 was 687, checked in by jpfasano, 13 years ago

Modified to skip build if prior build finished OK, and nothing has changed (ie
no modifed source code)

  • Property svn:eol-style set to native
File size: 13.0 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5
6import NBlogMessages
7import NBemail
8import NBosCommand
9import NBsvnCommand
10import NBcheckResult
11
12#---------------------------------------------------------------------
13# Keep history so same project is not repeatedly getting code from
14# subversion repository.
15#---------------------------------------------------------------------
16SVN_HISTORY = []
17THIRD_PARTY_HISTORY = []
18
19#------------------------------------------------------------------------
20#  Given a configuration, build and test it.
21#
22#  configuration['project']= name of project.
23#   examples: "Clp", "Ipopt"
24#
25#  configuration['rootDir']= root directory of nightlyBuild.
26#   This is where the project will be checked out from svn, and
27#   where the code will be compiled.  This directory must already
28#   exist.  If the testing requires, it needs to contain Netlib & miplib3
29#
30#  configuration['svnVersion']= svn version to be built.
31#   Examples are: "trunk", "stable/3.2", "releases/3.3.3"
32#
33#  configuration['noThirdParty']=True or False (optional). If False
34#   then 3rd party code will be used. If not specified then 3rd part
35#   code will be skipped.
36#
37#  configuration['configOptions']: Parameters to be passed to configure.
38#   The -C option and the options for skipping 3rd party code do not
39#   need to be specified.  These will be generated by this function.
40#   There are two types of configOptions to be specified.
41#  configuration['configOptions']['unique']= These are options that
42#   distinguish different build configurations.  These options are used
43#   to generate the vpath directory name where the code will be built.
44#   Examples are: "", "--enable-debug" "--enable-parrallel"
45#  configuration['configOptions']['invariant']= These are options that
46#   that are the same for every build configuration so they don't need
47#   to be part of the vpath directory name.
48#   Example: 'CXX="g++ -m64" LDFLAGS=-lstdc++'
49#
50#  configuration['SkipProjects']= List of COIN projects to skip (exclude)
51#    from the build.
52#    examples: "Ipopt", "Ipopt DyLP"
53#
54#  configuration['checkMakeTest']= function to be called to determine
55#    if "make test" ran correctly
56#
57#  configuration['unitTest']= undefined or dictionary D where
58#    D['path']= relative path were unitTest is to be run from.
59#    D['command']= command to be issued to run unitTest
60#    D['checkUnitTest']= function to be called to determine if unitTest
61#       ran correctly.
62#------------------------------------------------------------------------
63def run(configuration) :
64  NBlogMessages.writeMessage( configuration['project'] )
65
66  # Create svn checkout target directory name
67  svnVersionFlattened=configuration['svnVersion'].replace('/','-')
68
69  #---------------------------------------------------------------------
70  # Create names of directory where source is located and
71  # and were object, libs and executables are located (vpath).
72  # To compute vpath directory, the ./configure options need to be
73  # generated.
74  #---------------------------------------------------------------------
75  projectBaseDir=os.path.join(configuration['rootDir'],configuration['project'])
76  projectCheckOutDir=os.path.join(projectBaseDir,svnVersionFlattened)
77
78  svnCheckOutUrl='https://projects.coin-or.org/svn/'+\
79                 configuration['project']+'/'+\
80                 configuration['svnVersion']
81
82  vpathDir=''
83
84  if 'SkipProjects' in configuration :
85    vpathDir+="No"+configuration['SkipProjects']
86  if 'noThirdParty' in configuration : 
87    if configuration['noThirdParty'] :
88      vpathDir+='-NoThirdParty'
89  vpathDir=svnVersionFlattened+\
90          configuration['configOptions']['unique']+\
91          vpathDir
92  vpathDir=vpathDir.replace(' ','')
93  vpathDir=vpathDir.replace('"','')
94  vpathDir=vpathDir.replace("'",'')
95  vpathDir=vpathDir.replace('--enable','')
96  if vpathDir==svnVersionFlattened : vpathDir+='-default'
97
98  fullVpathDir = os.path.join(projectBaseDir,vpathDir)
99
100  NBlogMessages.writeMessage('  SVN source URL: '+svnCheckOutUrl)
101  NBlogMessages.writeMessage('  Checkout directory: '+projectCheckOutDir)
102  NBlogMessages.writeMessage('  Vpath directory: '+fullVpathDir)
103 
104
105  #---------------------------------------------------------------------
106  # Create the vpath directory if it doesn't exist
107  # and remove file that indicates prior run tested ok.
108  #---------------------------------------------------------------------
109  if not os.path.isdir(fullVpathDir) : 
110    os.makedirs(fullVpathDir)
111
112  #---------------------------------------------------------------------
113  # If nothing has changed and the prior run tested OK, then there
114  # is no need to do anything.
115  #---------------------------------------------------------------------
116  os.chdir(fullVpathDir)
117
118  if os.path.isfile('NBallTestsPassed') : 
119    if not NBsvnCommand.newer(svnCheckOutUrl,projectCheckOutDir):
120      # Previous run ran fine, and nothing has changed.
121      NBlogMessages.writeMessage('  No changes since previous successfull run')
122      return
123    os.remove('NBallTestsPassed')
124
125  #---------------------------------------------------------------------
126  # svn checkout or update the project
127  #---------------------------------------------------------------------
128  # Don't get source from subversion if previously done
129  if projectCheckOutDir not in SVN_HISTORY :
130    if not os.path.isdir(projectBaseDir) :
131      os.makedirs(projectBaseDir)
132    if not os.path.isdir(projectCheckOutDir) :
133      svnCmd='svn ' +\
134           'checkout ' +\
135           svnCheckOutUrl +\
136           ' '+svnVersionFlattened
137      if NBsvnCommand.run(svnCmd,projectBaseDir,configuration['project'])!='OK' :
138        return
139    else :
140      svnCmd='svn update'
141      if NBsvnCommand.run(svnCmd,projectCheckOutDir,configuration['project'])!='OK' :
142        return
143    SVN_HISTORY.append(projectCheckOutDir)
144  else :
145    NBlogMessages.writeMessage('  Skipping an "svn update"')
146
147  #---------------------------------------------------------------------
148  # If there are third party apps, then get these apps
149  #---------------------------------------------------------------------
150  if 'noThirdParty' in configuration :
151    if not configuration['noThirdParty'] :
152      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
153      if os.path.isdir(thirdPartyBaseDir) :
154        if thirdPartyBaseDir not in THIRD_PARTY_HISTORY :
155          THIRD_PARTY_HISTORY.append(thirdPartyBaseDir)
156          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
157          for d in thirdPartyDirs :
158            thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
159            install3rdPartyCmd=os.path.join(".","get."+d)
160            os.chdir(thirdPartyDir)
161            # If the install command has been updated since the last
162            # install, then do a new install
163            if os.path.isfile('NBinstalldone') :
164              if NBosCommand.newer(install3rdPartyCmd,'NBinstalldone') :
165                os.remove('NBinstalldone')
166            if not os.path.isfile('NBinstalldone') :
167              if os.path.isfile(install3rdPartyCmd) :
168                NBlogMessages.writeMessage('  '+install3rdPartyCmd)
169                installReturn = NBosCommand.run(install3rdPartyCmd)
170                if installReturn['returnCode'] :
171                  NBlogMessages.writeMsg('  warning: Install of 3rd party code in '+thirdPartyDir+' returned '+installReturn['returnCode'])
172                else :
173                  f=open('NBinstalldone','w')
174                  f.close()
175                stdoutfile=open(thirdPartyDir+'/NBstdout','w')
176                stdoutfile.write(installReturn['stdout'])
177                stdoutfile.close()
178                stderrfile=open(thirdPartyDir+'/NBstderr','w')
179                stderrfile.write(installReturn['stderr'])
180                stderrfile.close()
181            else :
182              NBlogMessages.writeMessage('  skipped a new download of '+d)
183        else :
184          NBlogMessages.writeMessage('  Skipped a new download into '+thirdPartyBaseDir)
185
186  #---------------------------------------------------------------------
187  # Source is now available, so now it is time to run config
188  #---------------------------------------------------------------------
189  skipOptions=''
190
191  if 'SkipProjects' in configuration :
192    skipOptions+=configuration['SkipProjects']
193
194  # If needed create option for skipping 3rd party code
195  needSkip3PartySkipOptions=False
196  if 'noThirdParty' not in configuration : 
197    needSkip3PartySkipOptions=True
198  elif configuration['noThirdParty'] :
199    needSkip3PartySkipOptions=True
200  if needSkip3PartySkipOptions :
201    thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
202    if os.path.isdir(thirdPartyBaseDir) :
203      thirdPartyDirs = os.listdir(thirdPartyBaseDir)
204      for d in thirdPartyDirs :
205        skipOptions+=' ThirdParty/'+d
206
207  if skipOptions!='' :
208    skipOptions=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
209
210  os.chdir(fullVpathDir)
211  NBlogMessages.writeMessage('  Current directory: '+fullVpathDir)
212 
213  # Assemble all config options together and create config command
214  configOptions ="-C "+configuration['configOptions']['unique']
215  configOptions+=configuration['configOptions']['unique']
216  configOptions+=configuration['configOptions']['invariant']
217  configOptions+=skipOptions
218  configCmd = os.path.join(projectCheckOutDir,"configure "+configOptions)
219
220
221
222  # If config was previously run, then no need to run again.
223  if NBcheckResult.didConfigRunOK() :
224    NBlogMessages.writeMessage("  configure previously ran. Not rerunning.")
225  else :
226    NBlogMessages.writeMessage("  "+configCmd)
227
228    # Finally run config
229    result=NBosCommand.run(configCmd)
230    stdoutfile=open('NBconfig.stdout','w')
231    stdoutfile.write(result['stdout'])
232    stdoutfile.close()
233    stderrfile=open('NBconfig.stderr','w')
234    stderrfile.write(result['stderr'])
235    stderrfile.close()
236
237    # Check if configure worked
238    if result['returnCode'] != 0 :
239        error_msg = result
240        # Add contents of log file to message
241        logFileName = 'config.log'
242        if os.path.isfile(logFileName) :
243          logFilePtr = open(logFileName,'r')
244          error_msg['config.log']  = "config.log contains: \n" 
245          error_msg['config.log'] += logFilePtr.read()
246          logFilePtr.close()
247        NBemail.sendCmdMsgs(configuration['project'],error_msg,configCmd)
248        return
249
250  #---------------------------------------------------------------------
251  # Run make part of build
252  #---------------------------------------------------------------------
253  NBlogMessages.writeMessage( '  make' )
254  result=NBosCommand.run('make')
255  stdoutfile=open('NBmake.stdout','w')
256  stdoutfile.write(result['stdout'])
257  stdoutfile.close()
258  stderrfile=open('NBmake.stderr','w')
259  stderrfile.write(result['stderr'])
260  stderrfile.close()
261
262  # Check if make worked
263  if result['returnCode'] != 0 :
264    NBemail.sendCmdMsgs(configuration['project'],result,'make')
265    return
266
267  #---------------------------------------------------------------------
268  # Run 'make test' part of build
269  #---------------------------------------------------------------------
270  NBlogMessages.writeMessage( '  make test' )
271  result=NBosCommand.run('make test')
272  stdoutfile=open('NBmaketest.stdout','w')
273  stdoutfile.write(result['stdout'])
274  stdoutfile.close()
275  stderrfile=open('NBmaketest.stderr','w')
276  stderrfile.write(result['stderr'])
277  stderrfile.close()
278
279  # Check if 'make test' worked
280  didMakeTestFail=configuration['checkMakeTest'](result,configuration['project'],"make test")
281  if didMakeTestFail :
282    result['make test']=didMakeTestFail
283    NBemail.sendCmdMsgs(configuration['project'],result,"make test")
284    return
285
286  #---------------------------------------------------------------------
287  # Run unitTest if available and different from 'make test'
288  #---------------------------------------------------------------------
289  if "unitTest" in configuration :
290    unitTestRelPath=configuration['unitTest']['path']
291    unitTestPath = os.path.join(fullVpathDir,unitTestRelPath)
292    os.chdir(unitTestPath)
293    NBlogMessages.writeMessage('  Current directory: '+unitTestPath)
294
295    unitTestCmdTemplate=configuration['unitTest']['command']
296
297    dataBaseDir=os.path.join(configuration['rootDir'],'Data')
298    netlibDir=os.path.join(dataBaseDir,'Netlib')
299    miplib3Dir=os.path.join(dataBaseDir,'miplib3')
300
301    unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
302    unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
303
304    NBlogMessages.writeMessage( '  '+unitTestCmd )
305    result=NBosCommand.run(unitTestCmd)
306    stdoutfile=open('NBunittest.stdout','w')
307    stdoutfile.write(result['stdout'])
308    stdoutfile.close()
309    stderrfile=open('NBunittest.stderr','w')
310    stderrfile.write(result['stderr'])
311    stderrfile.close()
312
313    didUnitTestFail=configuration['unitTest']['checkUnitTest'](result,configuration['project'],unitTestCmdTemplate)
314    if didUnitTestFail :
315      result['unitTest']=didUnitTestFail
316      NBemail.sendCmdMsgs(p,result,unitTestCmd)
317      return
318
319
320  #---------------------------------------------------------------------
321  # Everything build and all tests passed.
322  #---------------------------------------------------------------------
323  os.chdir(fullVpathDir)
324  f=open('NBallTestsPassed','w')
325  f.close()
Note: See TracBrowser for help on using the repository browser.