source: trunk/coopr/pysp/ph_script.py @ 1758

Last change on this file since 1758 was 1758, checked in by jwatson, 11 years ago

Some preliminary work on pickel-based restoration of PH state from a checkpoint file.

  • Property svn:executable set to *
File size: 20.9 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib
17import textwrap
18import traceback
19
20# garbage collection control.
21import gc
22
23# for profiling
24import cProfile
25import pstats
26
27# for serializing
28import pickle
29
30from coopr.pysp.convergence import *
31from coopr.pysp.scenariotree import *
32from coopr.pysp.ph import *
33from coopr.pysp.ef import *
34from coopr.opt.base import SolverFactory
35from coopr.opt.parallel import SolverManagerFactory
36
37#
38# Setup command-line options
39#
40
41parser = OptionParser()
42parser.add_option("--verbose",
43                  help="Generate verbose output for both initialization and execution.",
44                  action="store_true",
45                  dest="verbose",
46                  default=False)
47parser.add_option("--model-directory",
48                  help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
49                  action="store",
50                  dest="model_directory",
51                  type="string",
52                  default=".")
53parser.add_option("--instance-directory",
54                  help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
55                  action="store",
56                  dest="instance_directory",
57                  type="string",
58                  default=".")
59parser.add_option("--solver",
60                  help="The type of solver used to solve scenario sub-problems. Default is cplex.",
61                  action="store",
62                  dest="solver_type",
63                  type="string",
64                  default="cplex")
65parser.add_option("--solver-manager",
66                  help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
67                  action="store",
68                  dest="solver_manager_type",
69                  type="string",
70                  default="serial")
71parser.add_option("--max-iterations",
72                  help="The maximal number of PH iterations. Default is 100.",
73                  action="store",
74                  dest="max_iterations",
75                  type="int",
76                  default=100)
77parser.add_option("--default-rho",
78                  help="The default (global) rho for all blended variables. Default is 1.",
79                  action="store",
80                  dest="default_rho",
81                  type="float",
82                  default=1.0)
83parser.add_option("--rho-cfgfile",
84                  help="The name of a configuration script to compute PH rho values. Default is None.",
85                  action="store",
86                  dest="rho_cfgfile",
87                  type="string",
88                  default=None)
89parser.add_option("--enable-termdiff-convergence",
90                  help="Terminate PH based on the termdiff convergence metric. Default is True.",
91                  action="store_true",
92                  dest="enable_termdiff_convergence",
93                  default=True)
94parser.add_option("--enable-normalized-termdiff-convergence",
95                  help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
96                  action="store_true",
97                  dest="enable_normalized_termdiff_convergence",
98                  default=False)
99parser.add_option("--termdiff-threshold",
100                  help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
101                  action="store",
102                  dest="termdiff_threshold",
103                  type="float",
104                  default=0.01)
105parser.add_option("--enable-free-discrete-count-convergence",
106                  help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
107                  action="store_true",
108                  dest="enable_free_discrete_count_convergence",
109                  default=False)
110parser.add_option("--free-discrete-count-threshold",
111                  help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
112                  action="store",
113                  dest="free_discrete_count_threshold",
114                  type="float",
115                  default=20)
116parser.add_option("--enable-ww-extensions",
117                  help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
118                  action="store_true",
119                  dest="enable_ww_extensions",
120                  default=False)
121parser.add_option("--ww-extension-cfgfile",
122                  help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
123                  action="store",
124                  dest="ww_extension_cfgfile",
125                  type="string",
126                  default="wwph.cfg")
127parser.add_option("--ww-extension-suffixfile",
128                  help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
129                  action="store",
130                  dest="ww_extension_suffixfile",
131                  type="string",
132                  default="wwph.suffixes")
133parser.add_option("--write-ef",
134                  help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
135                  action="store_true",
136                  dest="write_ef",
137                  default=False)
138parser.add_option("--solve-ef",
139                  help="Following write of the extensive form model, solve it.",
140                  action="store_true",
141                  dest="solve_ef",
142                  default=False)
143parser.add_option("--ef-output-file",
144                  help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
145                  action="store",
146                  dest="ef_output_file",
147                  type="string",
148                  default="efout.lp")
149parser.add_option("--suppress-continuous-variable-output",
150                  help="Eliminate PH-related output involving continuous variables.",
151                  action="store_true",
152                  dest="suppress_continuous_variable_output",
153                  default=False)
154parser.add_option("--keep-solver-files",
155                  help="Retain temporary input and output files for scenario sub-problem solves",
156                  action="store_true",
157                  dest="keep_solver_files",
158                  default=False)
159parser.add_option("--output-solver-logs",
160                  help="Output solver logs during scenario sub-problem solves",
161                  action="store_true",
162                  dest="output_solver_logs",
163                  default=False)
164parser.add_option("--output-ef-solver-log",
165                  help="Output solver log during the extensive form solve",
166                  action="store_true",
167                  dest="output_ef_solver_log",
168                  default=False)
169parser.add_option("--output-solver-results",
170                  help="Output solutions obtained after each scenario sub-problem solve",
171                  action="store_true",
172                  dest="output_solver_results",
173                  default=False)
174parser.add_option("--output-times",
175                  help="Output timing statistics for various PH components",
176                  action="store_true",
177                  dest="output_times",
178                  default=False)
179parser.add_option("--disable-warmstarts",
180                  help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1",
181                  action="store_true",
182                  dest="disable_warmstarts",
183                  default=False)
184parser.add_option("--retain-quadratic-binary-terms",
185                  help="Do not linearize PH objective terms involving binary decision variables",
186                  action="store_true",
187                  dest="retain_quadratic_binary_terms",
188                  default=False)
189parser.add_option("--checkpoint-interval",
190                  help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
191                  action="store",
192                  dest="checkpoint_interval",
193                  type="int",
194                  default=0)
195parser.add_option("--restore-from-checkpoint",
196                  help="The name of the checkpoint file from which PH should be initialized. Default is "", indicating no checkpoint restoration",
197                  action="store",
198                  dest="restore_from_checkpoint",
199                  type="string",
200                  default="")
201parser.add_option("--profile",
202                  help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
203                  action="store",
204                  dest="profile",
205                  type="int",
206                  default=0)
207parser.add_option("--enable-gc",
208                  help="Enable the python garbage collecter.",
209                  action="store_true",
210                  dest="enable_gc",
211                  default=False)
212
213
214parser.usage="runph [options]"
215
216#
217# The main PH initialization / runner routine.
218#
219
220def run_ph(options, args):
221
222   start_time = time.time()
223
224   ph = None
225
226   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
227   if len(options.restore_from_checkpoint) > 0:
228
229      # we need to load the reference model, as pickle doesn't save contents of .py files!
230      try:
231         reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
232         if options.verbose is True:
233            print "Scenario reference model filename="+reference_model_filename
234         model_import = pyutilib.import_file(reference_model_filename)
235         if "model" not in dir(model_import):
236            print ""
237            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
238            return
239
240         if model_import.model is None:
241            print ""
242            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
243            return
244 
245         reference_model = model_import.model
246      except IOError:
247         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
248         return     
249
250      # import the saved state
251     
252      try:
253         checkpoint_file = open(options.restore_from_checkpoint,"r")
254         ph = pickle.load(checkpoint_file)
255         checkpoint_file.close()
256         
257#         print "PH=",ph
258#         ph._scenario_tree.pprint()         
259      except IOError, msg:
260         raise RuntimeError, msg
261
262      # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
263
264      raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
265     
266   else:
267      #
268      # create and populate the reference model/instance pair.
269      #
270
271      reference_model = None
272      reference_instance = None
273
274      try:
275         reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
276         if options.verbose is True:
277            print "Scenario reference model filename="+reference_model_filename
278         model_import = pyutilib.import_file(reference_model_filename)
279         if "model" not in dir(model_import):
280            print ""
281            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
282            return
283
284         if model_import.model is None:
285            print ""
286            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
287            return
288 
289         reference_model = model_import.model
290      except IOError:
291         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
292         return
293
294      try:
295         reference_instance_filename = options.instance_directory+os.sep+"ReferenceModel.dat"
296         if options.verbose is True:
297            print "Scenario reference instance filename="+reference_instance_filename
298         reference_instance = reference_model.create(reference_instance_filename)
299      except IOError:
300         print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
301         return     
302
303      #
304      # create and populate the scenario tree model
305      #
306
307      scenario_tree_model = None
308      scenario_tree_instance = None
309
310      try:
311         scenario_tree_model_filename = options.model_directory+os.sep+"ScenarioStructure.py"
312         if options.verbose is True:
313            print "Scenario tree model filename="+scenario_tree_model_filename
314         scenario_tree_import = pyutilib.import_file(scenario_tree_model_filename)
315         scenario_tree_model = scenario_tree_import.model
316      except IOError:
317         print "***ERROR: Failed to load scenario tree reference model from file="+scenario_tree_model_filename
318         return   
319
320      try:
321         scenario_tree_instance_filename = options.instance_directory+os.sep+"ScenarioStructure.dat"
322         if options.verbose is True:
323            print "Scenario tree instance filename="+scenario_tree_instance_filename
324         scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
325      except IOError:
326         print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
327         return
328   
329      #
330      # construct the scenario tree
331      #
332      scenario_tree = ScenarioTree(model=reference_instance,
333                                   nodes=scenario_tree_instance.Nodes,
334                                   nodechildren=scenario_tree_instance.Children,
335                                   nodestages=scenario_tree_instance.NodeStage,
336                                   nodeprobabilities=scenario_tree_instance.ConditionalProbability,
337                                   stages=scenario_tree_instance.Stages,
338                                   stagevariables=scenario_tree_instance.StageVariables,
339                                   stagecostvariables=scenario_tree_instance.StageCostVariable,
340                                   scenarios=scenario_tree_instance.Scenarios,
341                                   scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
342                                   scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
343
344      #
345      # print the input tree for validation/information purposes.
346      #
347      if options.verbose is True:
348         scenario_tree.pprint()
349
350      #
351      # validate the tree prior to doing anything serious
352      #
353      print ""
354      if scenario_tree.validate() is False:
355         print "***ERROR: Scenario tree is invalid****"
356         return
357      else:
358         if options.verbose is True:
359            print "Scenario tree is valid!"
360      print ""
361
362      #
363      # deal with any plugins (we currently only deal with one!)
364      #
365      if options.enable_ww_extensions is True:
366
367         from coopr.pysp import wwphextension
368
369         plugin = ExtensionPoint(IPHExtension)
370         plugin.service()._configuration_filename = options.ww_extension_cfgfile
371         plugin.service()._suffix_filename = options.ww_extension_suffixfile
372
373      #
374      # construct the convergence "computer" class.
375      #
376      converger = None
377      # go with the non-defaults first, and then with the default.
378      if options.enable_free_discrete_count_convergence is True:
379         converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
380      elif options.enable_normalized_termdiff_convergence is True:
381         converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
382      else:
383         converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
384
385     
386      #
387      # construct and initialize PH
388      #
389      ph = ProgressiveHedging(max_iterations=options.max_iterations, \
390                              rho=options.default_rho, \
391                              rho_setter=options.rho_cfgfile, \
392                              solver=options.solver_type, \
393                              solver_manager=options.solver_manager_type, \
394                              keep_solver_files=options.keep_solver_files, \
395                              output_solver_log=options.output_solver_logs, \
396                              output_solver_results=options.output_solver_results, \
397                              verbose=options.verbose, \
398                              output_times=options.output_times, \
399                              disable_warmstarts=options.disable_warmstarts,
400                              retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
401                              checkpoint_interval=options.checkpoint_interval)
402   
403      ph.initialize(scenario_data_directory_name=options.instance_directory, \
404                    model=reference_model, \
405                    model_instance=reference_instance, \
406                    scenario_tree=scenario_tree, \
407                    converger=converger)
408
409      if options.suppress_continuous_variable_output is True:
410         ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
411
412   #
413   # at this point, we have a PH object by some means.
414   #
415
416   #
417   # kick off the solve
418   #
419   ph.solve()
420
421   print ""
422   print "DONE..."
423
424   end_time = time.time()
425
426   print ""
427   print "Total execution time=%8.2f seconds" %(end_time - start_time)
428   print ""
429   if options.output_times is True:
430      ph.print_time_stats()
431
432   #
433   # write the extensive form, accounting for any fixed variables.
434   #
435   if (options.write_ef is True) or (options.solve_ef is True):
436      print ""
437      print "Writing EF for remainder problem"
438      print ""
439      write_ef(ph._scenario_tree, ph._instances, options.ef_output_file)
440
441   #
442   # solve the extensive form.
443   #
444   if options.solve_ef is True:
445      print ""
446      print "Solving extensive form written to file="+options.ef_output_file
447      print ""
448
449      ef_solver = SolverFactory(options.solver_type)
450      if ef_solver is None:
451         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
452
453      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
454      if ef_solver is None:
455         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
456
457      print "Queuing extensive form solve"
458      ef_action_handle = ef_solver_manager.queue(options.ef_output_file, opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
459      print "Waiting for extensive form solve"
460      ef_results = ef_solver_manager.wait_for(ef_action_handle)
461      print "Extensive form solve results:"
462      print ef_results
463     
464
465def run(args=None):
466
467    #
468    # Top-level command that executes the extensive form writer.
469    # This is segregated from run_ef_writer to enable profiling.
470    #
471
472    #
473    # Parse command-line options.
474    #
475    try:
476       (options, args) = parser.parse_args(args=args)
477    except SystemExit:
478       # the parser throws a system exit if "-h" is specified - catch
479       # it to exit gracefully.
480       return
481
482    # for a one-pass execution, garbage collection doesn't make
483    # much sense - so it is disabled by default. Because: It drops
484    # the run-time by a factor of 3-4 on bigger instances.
485    if options.enable_gc is False:
486       gc.disable()       
487
488    if options.profile > 0:
489        #
490        # Call the main PH routine with profiling.
491        #
492        tfile = pyutilib.TempfileManager.create_tempfile(suffix=".profile")
493        tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
494        p = pstats.Stats(tfile).strip_dirs()
495        p.sort_stats('time', 'cum')
496        options.profile = eval(options.profile)
497        p = p.print_stats(options.profile)
498        p.print_callers(options.profile)
499        p.print_callees(options.profile)
500        p = p.sort_stats('cum','calls')
501        p.print_stats(options.profile)
502        p.print_callers(options.profile)
503        p.print_callees(options.profile)
504        p = p.sort_stats('calls')
505        p.print_stats(options.profile)
506        p.print_callers(options.profile)
507        p.print_callees(options.profile)
508        pyutilib.TempfileManager.clear_tempfiles()
509        ans = [tmp, None]
510    else:
511        #
512        # Call the main PH routine without profiling.
513        #
514        ans = run_ph(options, args)
515
516    gc.enable()
517   
518    return ans
519
Note: See TracBrowser for help on using the repository browser.