source: coopr.pysp/trunk/coopr/pysp/ph_script.py @ 2255

Last change on this file since 2255 was 2255, checked in by jwatson, 10 years ago

Re-factoring of PH options parser code to accomodate MRP work being done by DLW.

  • Property svn:executable set to *
File size: 27.6 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20
21# garbage collection control.
22import gc
23
24# for profiling
25import cProfile
26import pstats
27
28# for serializing
29import pickle
30
31from coopr.pysp.convergence import *
32from coopr.pysp.scenariotree import *
33from coopr.pysp.ph import *
34from coopr.pysp.ef import *
35from coopr.opt.base import SolverFactory
36from coopr.opt.parallel import SolverManagerFactory
37
38#
39# utility method to construct an option parser for ph arguments, to be
40# supplied as an argument to the runph method.
41#
42
43def construct_ph_options_parser():
44
45   parser = OptionParser()
46   parser.add_option("--verbose",
47                     help="Generate verbose output for both initialization and execution. Default is False.",
48                     action="store_true",
49                     dest="verbose",
50                     default=False)
51   parser.add_option("--report-solutions",
52                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
53                     action="store_true",
54                     dest="report_solutions",
55                     default=False)
56   parser.add_option("--report-weights",
57                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
58                     action="store_true",
59                     dest="report_weights",
60                     default=False)
61   parser.add_option("--model-directory",
62                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
63                     action="store",
64                     dest="model_directory",
65                     type="string",
66                     default=".")
67   parser.add_option("--instance-directory",
68                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
69                     action="store",
70                     dest="instance_directory",
71                     type="string",
72                     default=".")
73   parser.add_option("--solver",
74                     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
75                     action="store",
76                     dest="solver_type",
77                     type="string",
78                     default="cplex")
79   parser.add_option("--solver-manager",
80                     help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
81                     action="store",
82                     dest="solver_manager_type",
83                     type="string",
84                     default="serial")
85   parser.add_option("--scenario-solver-options",
86                     help="Solver options for all PH scenario sub-problems",
87                     action="append",
88                     dest="scenario_solver_options",
89                     type="string",
90                     default=[])
91   parser.add_option("--scenario-mipgap",
92                     help="Specifies the mipgap for all PH scenario sub-problems",
93                     action="store",
94                     dest="scenario_mipgap",
95                     type="float",
96                     default=None)
97   parser.add_option("--ef-solver-options",
98                     help="Solver options for the extension form problem",
99                     action="append",
100                     dest="ef_solver_options",
101                     type="string",
102                     default=[])
103   parser.add_option("--ef-mipgap",
104                     help="Specifies the mipgap for the EF solve",
105                     action="store",
106                     dest="ef_mipgap",
107                     type="float",
108                     default=None)
109   parser.add_option("--max-iterations",
110                     help="The maximal number of PH iterations. Default is 100.",
111                     action="store",
112                     dest="max_iterations",
113                     type="int",
114                     default=100)
115   parser.add_option("--default-rho",
116                     help="The default (global) rho for all blended variables. Default is 1.",
117                     action="store",
118                     dest="default_rho",
119                     type="float",
120                     default=1.0)
121   parser.add_option("--rho-cfgfile",
122                     help="The name of a configuration script to compute PH rho values. Default is None.",
123                     action="store",
124                     dest="rho_cfgfile",
125                     type="string",
126                     default=None)
127   parser.add_option("--bounds-cfgfile",
128                     help="The name of a configuration script to set variable bound values. Default is None.",
129                     action="store",
130                     dest="bounds_cfgfile",
131                     default=None)
132   parser.add_option("--enable-termdiff-convergence",
133                     help="Terminate PH based on the termdiff convergence metric. Default is True.",
134                     action="store_true",
135                     dest="enable_termdiff_convergence",
136                     default=True)
137   parser.add_option("--enable-normalized-termdiff-convergence",
138                     help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
139                     action="store_true",
140                     dest="enable_normalized_termdiff_convergence",
141                     default=False)
142   parser.add_option("--termdiff-threshold",
143                     help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
144                     action="store",
145                     dest="termdiff_threshold",
146                     type="float",
147                     default=0.01)
148   parser.add_option("--enable-free-discrete-count-convergence",
149                     help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
150                     action="store_true",
151                     dest="enable_free_discrete_count_convergence",
152                     default=False)
153   parser.add_option("--free-discrete-count-threshold",
154                     help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
155                     action="store",
156                     dest="free_discrete_count_threshold",
157                     type="float",
158                     default=20)
159   parser.add_option("--enable-ww-extensions",
160                     help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
161                     action="store_true",
162                     dest="enable_ww_extensions",
163                     default=False)
164   parser.add_option("--ww-extension-cfgfile",
165                     help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
166                     action="store",
167                     dest="ww_extension_cfgfile",
168                     type="string",
169                     default="")
170   parser.add_option("--ww-extension-suffixfile",
171                     help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
172                     action="store",
173                     dest="ww_extension_suffixfile",
174                     type="string",
175                     default="")
176   parser.add_option("--user-defined-extension",
177                     help="The name of a python module specifying a user-defined PH extension plugin.",
178                     action="store",
179                     dest="user_defined_extension",
180                     type="string",
181                     default=None)
182   parser.add_option("--write-ef",
183                     help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
184                     action="store_true",
185                     dest="write_ef",
186                     default=False)
187   parser.add_option("--solve-ef",
188                     help="Following write of the extensive form model, solve it.",
189                     action="store_true",
190                     dest="solve_ef",
191                     default=False)
192   parser.add_option("--ef-output-file",
193                     help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
194                     action="store",
195                     dest="ef_output_file",
196                     type="string",
197                     default="efout.lp")
198   parser.add_option("--suppress-continuous-variable-output",
199                     help="Eliminate PH-related output involving continuous variables.",
200                     action="store_true",
201                     dest="suppress_continuous_variable_output",
202                     default=False)
203   parser.add_option("--keep-solver-files",
204                     help="Retain temporary input and output files for scenario sub-problem solves",
205                     action="store_true",
206                     dest="keep_solver_files",
207                     default=False)
208   parser.add_option("--output-solver-logs",
209                     help="Output solver logs during scenario sub-problem solves",
210                     action="store_true",
211                     dest="output_solver_logs",
212                     default=False)
213   parser.add_option("--output-ef-solver-log",
214                     help="Output solver log during the extensive form solve",
215                     action="store_true",
216                     dest="output_ef_solver_log",
217                     default=False)
218   parser.add_option("--output-solver-results",
219                     help="Output solutions obtained after each scenario sub-problem solve",
220                     action="store_true",
221                     dest="output_solver_results",
222                     default=False)
223   parser.add_option("--output-times",
224                     help="Output timing statistics for various PH components",
225                     action="store_true",
226                     dest="output_times",
227                     default=False)
228   parser.add_option("--disable-warmstarts",
229                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
230                     action="store_true",
231                     dest="disable_warmstarts",
232                     default=False)
233   parser.add_option("--drop-proximal-terms",
234                     help="Eliminate proximal terms (i.e., the quadratic penalty terms) from the weighted PH objective. Default is False.",
235                     action="store_true",
236                     dest="drop_proximal_terms",
237                     default=False)
238   parser.add_option("--retain-quadratic-binary-terms",
239                     help="Do not linearize PH objective terms involving binary decision variables",
240                     action="store_true",
241                     dest="retain_quadratic_binary_terms",
242                     default=False)
243   parser.add_option("--linearize-nonbinary-penalty-terms",
244                     help="Approximate the PH quadratic term for non-binary variables with a piece-wise linear function, using the supplied number of equal-length pieces from each bound to the average",
245                     action="store",
246                     dest="linearize_nonbinary_penalty_terms",
247                     type="int",
248                     default=0)
249   parser.add_option("--breakpoint-strategy",
250                     help="Specify the strategy to distribute breakpoints on the [lb, ub] interval of each variable when linearizing. 0 indicates uniform distribution. 1 indicates breakpoints at the node min and max, uniformly in-between. 2 indicates more aggressive concentration of breakpoints near the observed node min/max.",
251                     action="store",
252                     dest="breakpoint_strategy",
253                     type="int",
254                     default=0)
255   parser.add_option("--checkpoint-interval",
256                     help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
257                     action="store",
258                     dest="checkpoint_interval",
259                     type="int",
260                     default=0)
261   parser.add_option("--restore-from-checkpoint",
262                     help="The name of the checkpoint file from which PH should be initialized. Default is \"\", indicating no checkpoint restoration",
263                     action="store",
264                     dest="restore_from_checkpoint",
265                     type="string",
266                     default="")
267   parser.add_option("--profile",
268                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
269                     action="store",
270                     dest="profile",
271                     type="int",
272                     default=0)
273   parser.add_option("--enable-gc",
274                     help="Enable the python garbage collecter. Default is True.",
275                     action="store_true",
276                     dest="enable_gc",
277                     default=True)
278
279   parser.usage="runph [options]"
280
281   return parser
282
283#
284# The main PH initialization / runner routine.
285#
286
287def run_ph(options, args):
288
289   start_time = time.time()
290
291   ph = None
292
293   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
294   if len(options.restore_from_checkpoint) > 0:
295
296      # we need to load the reference model, as pickle doesn't save contents of .py files!
297      try:
298         reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
299         if options.verbose is True:
300            print "Scenario reference model filename="+reference_model_filename
301         model_import = pyutilib.misc.import_file(reference_model_filename)
302         if "model" not in dir(model_import):
303            print ""
304            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
305            return
306
307         if model_import.model is None:
308            print ""
309            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
310            return
311 
312         reference_model = model_import.model
313      except IOError:
314         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
315         return     
316
317      # import the saved state
318     
319      try:
320         checkpoint_file = open(options.restore_from_checkpoint,"r")
321         ph = pickle.load(checkpoint_file)
322         checkpoint_file.close()
323         
324#         print "PH=",ph
325#         ph._scenario_tree.pprint()         
326      except IOError, msg:
327         raise RuntimeError, msg
328
329      # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
330
331      raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
332     
333   else:
334      #
335      # create and populate the reference model/instance pair.
336      #
337
338      reference_model = None
339      reference_instance = None
340
341      try:
342         reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
343         if options.verbose is True:
344            print "Scenario reference model filename="+reference_model_filename
345         model_import = pyutilib.misc.import_file(reference_model_filename)
346         if "model" not in dir(model_import):
347            print ""
348            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
349            return
350
351         if model_import.model is None:
352            print ""
353            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
354            return
355 
356         reference_model = model_import.model
357      except IOError:
358         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
359         return
360
361      try:
362         reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
363         if options.verbose is True:
364            print "Scenario reference instance filename="+reference_instance_filename
365         reference_instance = reference_model.create(reference_instance_filename)
366      except IOError:
367         print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
368         return     
369
370      #
371      # create and populate the scenario tree model
372      #
373
374      from coopr.pysp.util.scenariomodels import scenario_tree_model
375      scenario_tree_instance = None
376
377      try:
378         scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
379         if options.verbose is True:
380            print "Scenario tree instance filename="+scenario_tree_instance_filename
381         scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
382      except IOError:
383         print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
384         return
385   
386      #
387      # construct the scenario tree
388      #
389      scenario_tree = ScenarioTree(model=reference_instance,
390                                   nodes=scenario_tree_instance.Nodes,
391                                   nodechildren=scenario_tree_instance.Children,
392                                   nodestages=scenario_tree_instance.NodeStage,
393                                   nodeprobabilities=scenario_tree_instance.ConditionalProbability,
394                                   stages=scenario_tree_instance.Stages,
395                                   stagevariables=scenario_tree_instance.StageVariables,
396                                   stagecostvariables=scenario_tree_instance.StageCostVariable,
397                                   scenarios=scenario_tree_instance.Scenarios,
398                                   scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
399                                   scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
400
401      #
402      # print the input tree for validation/information purposes.
403      #
404      if options.verbose is True:
405         scenario_tree.pprint()
406
407      #
408      # validate the tree prior to doing anything serious
409      #
410      print ""
411      if scenario_tree.validate() is False:
412         print "***ERROR: Scenario tree is invalid****"
413         return
414      else:
415         if options.verbose is True:
416            print "Scenario tree is valid!"
417      print ""
418
419      #
420      # if any of the ww extension configuration options are specified without the
421      # ww extension itself being enabled, halt and warn the user - this has led
422      # to confusion in the past, and will save user support time.
423      #
424      if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
425         print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
426         return
427
428      if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
429         print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"         
430         return
431
432      #
433      # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
434      #
435      if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
436         print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
437         return         
438
439      #
440      # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
441      # order only matters if both are specified.
442      #
443      if options.enable_ww_extensions is True:
444
445         from coopr.pysp import wwphextension
446
447         plugin = ExtensionPoint(IPHExtension)
448         if len(options.ww_extension_cfgfile) > 0:
449            plugin.service()._configuration_filename = options.ww_extension_cfgfile
450         if len(options.ww_extension_suffixfile) > 0:
451            plugin.service()._suffix_filename = options.ww_extension_suffixfile
452
453      if options.user_defined_extension is not None:
454         print "Trying to import user-defined PH extension module="+options.user_defined_extension
455         # JPW removed the exception handling logic, as the module importer
456         # can raise a broad array of exceptions.
457         __import__(options.user_defined_extension)
458         print "Module successfully loaded"
459
460      #
461      # construct the convergence "computer" class.
462      #
463      converger = None
464      # go with the non-defaults first, and then with the default.
465      if options.enable_free_discrete_count_convergence is True:
466         converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
467      elif options.enable_normalized_termdiff_convergence is True:
468         converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
469      else:
470         converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
471
472     
473      #
474      # construct and initialize PH
475      #
476      ph = ProgressiveHedging(max_iterations=options.max_iterations, \
477                              rho=options.default_rho, \
478                              rho_setter=options.rho_cfgfile, \
479                              bounds_setter=options.bounds_cfgfile, \
480                              solver=options.solver_type, \
481                              solver_manager=options.solver_manager_type, \
482                              scenario_solver_options=options.scenario_solver_options, \
483                              scenario_mipgap=options.scenario_mipgap, \
484                              keep_solver_files=options.keep_solver_files, \
485                              output_solver_log=options.output_solver_logs, \
486                              output_solver_results=options.output_solver_results, \
487                              verbose=options.verbose, \
488                              report_solutions=options.report_solutions, \
489                              report_weights=options.report_weights, \
490                              output_times=options.output_times, \
491                              disable_warmstarts=options.disable_warmstarts,
492                              drop_proximal_terms=options.drop_proximal_terms,
493                              retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
494                              linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
495                              breakpoint_strategy=options.breakpoint_strategy, \
496                              checkpoint_interval=options.checkpoint_interval)
497   
498      ph.initialize(scenario_data_directory_name=os.path.expanduser(options.instance_directory), \
499                    model=reference_model, \
500                    model_instance=reference_instance, \
501                    scenario_tree=scenario_tree, \
502                    converger=converger)
503
504      if options.suppress_continuous_variable_output is True:
505         ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
506
507   #
508   # at this point, we have a PH object by some means.
509   #
510
511   #
512   # kick off the solve
513   #
514   ph.solve()
515
516   print ""
517   print "DONE..."
518
519   end_time = time.time()
520
521   print ""
522   print "Total execution time=%8.2f seconds" %(end_time - start_time)
523   print ""
524   if options.output_times is True:
525      ph.print_time_stats()
526
527   #
528   # write the extensive form, accounting for any fixed variables.
529   #
530   if (options.write_ef is True) or (options.solve_ef is True):
531      print ""
532      print "Writing EF for remainder problem"
533      print ""
534      write_ef(ph._scenario_tree, ph._instances, os.path.expanduser(options.ef_output_file))
535
536   #
537   # solve the extensive form.
538   #
539   if options.solve_ef is True:
540      print ""
541      print "Solving extensive form written to file="+os.path.expanduser(options.ef_output_file)
542      print ""
543
544      ef_solver = SolverFactory(options.solver_type)
545      if ef_solver is None:
546         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
547      if len(options.ef_solver_options) > 0:
548         print "Initializing ef solver with options="+str(options.ef_solver_options)         
549         ef_solver.set_options("".join(options.ef_solver_options))
550      if options.ef_mipgap is not None:
551         if (options.ef_mipgap < 0.0) or (options.ef_mipgap > 1.0):
552            raise ValueError, "Value of the mipgap parameter for the EF solve must be on the unit interval; value specified=" + `options.ef_mipgap`
553         else:
554            ef_solver.mipgap = options.ef_mipgap
555
556      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
557      if ef_solver is None:
558         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
559
560      print "Queuing extensive form solve"
561      ef_action_handle = ef_solver_manager.queue(os.path.expanduser(options.ef_output_file), opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
562      print "Waiting for extensive form solve"
563      ef_results = ef_solver_manager.wait_for(ef_action_handle)
564      print "Extensive form solve results:"
565      ef_results.write(num=1)
566
567def run(args=None):
568
569    #
570    # Top-level command that executes the extensive form writer.
571    # This is segregated from run_ef_writer to enable profiling.
572    #
573
574    #
575    # Parse command-line options.
576    #
577    try:
578       ph_options_parser = construct_ph_options_parser()
579       (options, args) = ph_options_parser.parse_args(args=args)
580    except SystemExit:
581       # the parser throws a system exit if "-h" is specified - catch
582       # it to exit gracefully.
583       return
584
585    # for a one-pass execution, garbage collection doesn't make
586    # much sense - so it is disabled by default. Because: It drops
587    # the run-time by a factor of 3-4 on bigger instances.
588    if options.enable_gc is False:
589       gc.disable()
590    else:
591       gc.enable()
592
593    if options.profile > 0:
594        #
595        # Call the main PH routine with profiling.
596        #
597        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
598        tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
599        p = pstats.Stats(tfile).strip_dirs()
600        p.sort_stats('time', 'cum')
601        p = p.print_stats(options.profile)
602        p.print_callers(options.profile)
603        p.print_callees(options.profile)
604        p = p.sort_stats('cum','calls')
605        p.print_stats(options.profile)
606        p.print_callers(options.profile)
607        p.print_callees(options.profile)
608        p = p.sort_stats('calls')
609        p.print_stats(options.profile)
610        p.print_callers(options.profile)
611        p.print_callees(options.profile)
612        pyutilib.services.TempfileManager.clear_tempfiles()
613        ans = [tmp, None]
614    else:
615        #
616        # Call the main PH routine without profiling.
617        #
618        ans = run_ph(options, args)
619
620    gc.enable()
621   
622    return ans
623
Note: See TracBrowser for help on using the repository browser.