source: branches/testScripts/nightlyBuild.py @ 652

Last change on this file since 652 was 651, checked in by stefan, 12 years ago

set marker when third party code had been downloaded, so we can skip anew download

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 11.7 KB
Line 
1#! /usr/bin/env python
2
3import os
4import sys
5#import distutils.dir_util
6
7import NBuserConfig
8import NBprojectConfig
9import NBlogMessages
10import NBemail
11import NBosCommand
12import NBsvnCommand
13import NBcheckResult
14
15# TODO:
16#   -Get some information about the platform and put this in email failure message.
17#   -In userConfig allow one to optionally do a clean checkout and/or config
18#   -Don't do build if 'svn update' doesn't change anything and prior test was OK.
19#     (no need to re-run if nothing has changed since prior run)
20#   -Figure out how to run configure with no third party code (done, or?)
21#   -Remove markerfile in ThirdParty dir if svn update updated the get... script of the ThirdParty project
22
23
24
25
26#------------------------------------------------------------------------
27#  Main Program Starts Here 
28#------------------------------------------------------------------------
29
30#------------------------------------------------------------------------
31#  If needed create the top level directory
32#------------------------------------------------------------------------
33if not os.path.isdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR) :
34  os.makedirs(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
35os.chdir(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR)
36
37#------------------------------------------------------------------------
38#  Get the data directories if they don't already exist
39#------------------------------------------------------------------------
40dataBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,'Data')
41if not os.path.isdir(dataBaseDir) :
42  os.makedirs(dataBaseDir)
43dataDirs=['Netlib','miplib3']
44for d in dataDirs :
45  dataDir=os.path.join(dataBaseDir,d)
46  if not os.path.isdir(dataDir) :
47    svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' checkout https://projects.coin-or.org/svn/Data/releases/1.0.0/'+d+' '+d
48    if NBsvnCommand.run(svnCmd,dataBaseDir,'Data')!='OK' :
49      sys.exit(1)
50    result=NBosCommand.run('find '+d+' -name \*.gz -print | xargs gzip -d')
51netlibDir=os.path.join(dataBaseDir,'Netlib')
52miplib3Dir=os.path.join(dataBaseDir,'miplib3')
53
54#------------------------------------------------------------------------
55# Loop once for each project
56#------------------------------------------------------------------------
57for p in NBuserConfig.PROJECTS:
58  NBlogMessages.writeMessage( p )
59
60  #---------------------------------------------------------------------
61  # Loop once for each version of the project to be checked out.
62  # The supported types are trunk & latestStable
63  #---------------------------------------------------------------------
64  projectVersions=[]
65  if 'latestStable' in NBuserConfig.PROJECT_VERSIONS :
66    lsv = NBsvnCommand.latestStableVersion(p)
67    projectVersions.append(['stable'+lsv,'stable/'+lsv])
68  if 'trunk' in NBuserConfig.PROJECT_VERSIONS :
69    projectVersions.append(['trunk','trunk'])
70  for projectVersion in projectVersions :
71    #---------------------------------------------------------------------
72    # svn checkout or update the project
73    #---------------------------------------------------------------------
74    projectBaseDir=os.path.join(NBuserConfig.NIGHTLY_BUILD_ROOT_DIR,p)
75    projectCheckOutDir=os.path.join(projectBaseDir,projectVersion[0])
76    if not os.path.isdir(projectBaseDir) :
77      os.makedirs(projectBaseDir)
78    if not os.path.isdir(projectCheckOutDir) :
79      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') +\
80             ' checkout https://projects.coin-or.org/svn/'+p+'/'+projectVersion[1]+' '+projectVersion[0]
81      if NBsvnCommand.run(svnCmd,projectBaseDir,p)!='OK' :
82        continue
83    else :
84      svnCmd=os.path.join(NBuserConfig.SVNPATH_PREFIX,'svn') + ' update'
85      if NBsvnCommand.run(svnCmd,projectCheckOutDir,p)!='OK' :
86        continue
87
88    #---------------------------------------------------------------------
89    # If there are third party apps, then get these apps
90    #---------------------------------------------------------------------
91    if NBuserConfig.DOWNLOAD_3RD_PARTY :
92      thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
93      if os.path.isdir(thirdPartyBaseDir) :
94        thirdPartyDirs = os.listdir(thirdPartyBaseDir)
95        for d in thirdPartyDirs :
96          thirdPartyDir=os.path.join(thirdPartyBaseDir,d)
97          install3rdPartyCmd=os.path.join(".","get."+d)
98          os.chdir(thirdPartyDir)
99          if not os.path.isfile('NBinstalldone') :
100            if os.path.isfile(install3rdPartyCmd) :
101              NBlogMessages.writeMessage('  '+install3rdPartyCmd)
102              NBosCommand.run(install3rdPartyCmd)
103              f=open('NBinstalldone','w')
104              f.close()
105          else :
106            NBlogMessages.writeMessage('  skipped anew download of '+d)
107   
108    #---------------------------------------------------------------------
109    # Loop once for each type of build to be done.
110    # Debug, use third party code, ...
111    # vpath and options to configure must be set for the buildType
112    #---------------------------------------------------------------------
113    buildtypes=NBprojectConfig.PROJECT_CONFIG_LINES[p]
114    for buildType, configLine in buildtypes.iteritems() :
115   
116      #---------------------------------------------------------------------
117      # Setup the directory where the build will be done and the configure
118      # command options
119      #---------------------------------------------------------------------
120      vpathDir=projectVersion[0]+'-'+buildType
121      configOptions='-C '+configLine
122
123#      if "ThirdParty" in buildType :
124#        vpathDir += "ThirdParty"
125#      else :
126#        vpathDir += "NoThirdParty"
127#        thirdPartyBaseDir=os.path.join(projectCheckOutDir,'ThirdParty')
128#        if os.path.isdir(thirdPartyBaseDir) :
129#          thirdPartyDirs = os.listdir(thirdPartyBaseDir)
130#          skipOptions=''
131#          for d in thirdPartyDirs :
132#            skipOptions+=' ThirdParty/'+d
133#          configOptions+=' COIN_SKIP_PROJECTS="'+skipOptions+'"'
134     
135      fullVpathDir = os.path.join(projectBaseDir,vpathDir)
136      #TODO: if (MAKE_CLEAN) : distutils.dir_util.remove_tree(fullVpathDir)
137      if not os.path.isdir(fullVpathDir) : os.mkdir(fullVpathDir)
138      NBlogMessages.writeMessage("  build "+buildType+" in "+fullVpathDir)
139
140      #---------------------------------------------------------------------
141      # Run configure part of build (only if config has not previously
142      # ran successfully).
143      #---------------------------------------------------------------------
144      os.chdir(fullVpathDir)
145      configCmd = os.path.join('.',projectCheckOutDir,"configure "+configOptions)
146      if NBcheckResult.didConfigRunOK() :
147        NBlogMessages.writeMessage("  '"+configCmd+"' previously ran. Not rerunning")
148      else :
149        NBlogMessages.writeMessage('  '+configCmd)
150        result=NBosCommand.run(configCmd)
151     
152        # Check if configure worked
153        if result['returnCode'] != 0 :
154          error_msg = result
155          # Add contents of log file to message
156          logFileName = 'config.log'
157          if os.path.isfile(logFileName) :
158            logFilePtr = open(logFileName,'r')
159            error_msg['config.log']  = "config.log contains: \n" 
160            error_msg['config.log'] += logFilePtr.read()
161            logFilePtr.close()
162          NBemail.sendCmdMsgs(p,error_msg,configCmd)
163          continue
164
165      #---------------------------------------------------------------------
166      # Run make part of build
167      #---------------------------------------------------------------------
168      NBlogMessages.writeMessage( '  make' )
169      result=NBosCommand.run('make')
170     
171      # Check if make worked
172      if result['returnCode'] != 0 :
173        NBemail.sendCmdMsgs(p,result,'make')
174        continue
175
176      #---------------------------------------------------------------------
177      # Run 'make test' part of build
178      #---------------------------------------------------------------------
179      NBlogMessages.writeMessage( '  make test' )
180      result=NBosCommand.run('make test')
181     
182      # Check if 'make test' worked
183      didMakeTestFail=NBcheckResult.didTestFail(result,p,"make test")
184      if didMakeTestFail :
185        result['make test']=didMakeTestFail
186        NBemail.sendCmdMsgs(p,result,"make test")
187        continue
188
189      #---------------------------------------------------------------------
190      # Run unitTest if available and different from 'make test'
191      #---------------------------------------------------------------------
192      if NBprojectConfig.UNITTEST_CMD.has_key(p) :
193        unitTestPath = os.path.join(fullVpathDir,NBprojectConfig.UNITTEST_DIR[p])
194        os.chdir(unitTestPath)
195
196        unitTestCmdTemplate=NBprojectConfig.UNITTEST_CMD[p]
197        unitTestCmd=unitTestCmdTemplate.replace('_NETLIBDIR_',netlibDir)
198        unitTestCmd=unitTestCmd.replace('_MIPLIB3DIR_',miplib3Dir)
199
200        NBlogMessages.writeMessage( '  '+unitTestCmd )
201        result=NBosCommand.run(unitTestCmd)
202     
203        didUnitTestFail=NBcheckResult.didTestFail(result,p,unitTestCmdTemplate)
204        if didUnitTestFail :
205          result['unitTest']=didUnitTestFail
206          NBemail.sendCmdMsgs(p,result,unitTestCmd)
207          continue
208
209NBlogMessages.writeMessage( "nightlyBuild.py Finished" )
210
211sys.exit(0)
212
213
214# START KIPP
215#----------------------------------------------------------------------
216# CONFIG FILE PATH:
217#   path to the config file for the build
218#   done. If the directory does not exist, it will be created.
219#   this should have all of the user specific data
220#   it should have values for
221#   NIGHTLY_BUILD_ROOT
222#   NBuserConfig.SMTP_SERVER_NAME
223#   NBuserConfig.SMTP_SERVER_PORT
224#   NBuserConfig.SMTP_SSL_SERVER
225#   NBuserConfig.SMTP_USER_NAME
226#   NBuserConfig.SMTP_PASSWORD_FILENAME = '/home/jp/bin/smtpPwFile'
227#   NBuserConfig.SENDER_EMAIL_ADDR='jpfasano _AT_ verizon _DOT_ net'
228#   NBuserConfig.MY_EMAIL_ADDR='jpfasano _AT_ us _DOT_ ibm _DOT_ com'
229#   
230#----------------------------------------------------------------------
231
232CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
233CONFIG_FILENAME = 'config.txt'
234
235
236# Get configFile data
237
238configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
239                                 os.path.basename(CONFIG_FILENAME ))
240if os.path.isfile(  configFile) :
241  pwFilePtr = open(configFile ,'r')
242  d = pwFilePtr.readlines()
243  # do pwFilePtr.read() to get a string object
244  # we have a list object
245  print d[0]
246  print d[1]
247  # make a dictionary
248  config_dic = {}
249
250  #smtppass  = pwFilePtr.read().strip()
251  pwFilePtr.close()
252else :
253  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
254  #print cmdMsgs
255  sys.exit( 1)
256sys.exit( 0)
257
258
259
260# START KIPP
261#----------------------------------------------------------------------
262#   path to the config file for the build
263#   get the user dependent variables
264# CONFIG FILE PATH:
265#   it should have values for
266#   NIGHTLY_BUILD_ROOT
267#   NBuserConfig.SMTP_SERVER_NAME
268#   NBuserConfig.SMTP_SERVER_PORT
269#   NBuserConfig.SMTP_SSL_SERVER
270#   NBuserConfig.SMTP_USER_NAME
271#   NBuserConfig.SMTP_PASSWORD_FILENAME
272#   NBuserConfig.SENDER_EMAIL_ADDR
273#   NBuserConfig.MY_EMAIL_ADDR
274#   
275#----------------------------------------------------------------------
276
277CONFIG_FILE_PATH = '/Users/kmartin/Documents/files/configDir/'
278CONFIG_FILENAME = 'config.txt'
279
280
281# Get configFile data
282
283configFile = os.path.join(os.path.dirname( CONFIG_FILE_PATH),
284                                 os.path.basename(CONFIG_FILENAME ))
285if os.path.isfile(  configFile) :
286  pwFilePtr = open(configFile ,'r')
287  d = pwFilePtr.readlines()
288  # do pwFilePtr.read() to get a string object
289  # we have a list object
290  print d[0]
291  print d[1]
292  # make a dictionary
293  config_dic = {}
294
295  #smtppass  = pwFilePtr.read().strip()
296  pwFilePtr.close()
297else :
298  #NBlogMessages.writeMessage( "Failure reading pwFileName=" + CONFIG_FILENAME )
299  #print cmdMsgs
300  sys.exit( 1)
301sys.exit( 0)
302
303# END KIPP
Note: See TracBrowser for help on using the repository browser.