source: coopr.pysp/trunk/coopr/pysp/phinit.py @ 2277

Last change on this file since 2277 was 2277, checked in by jwatson, 10 years ago

Fixed problem with runph --profile option, broken by my recent factoring of phinit.py.

  • Property svn:executable set to *
File size: 28.0 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20
21# garbage collection control.
22import gc
23
24# for profiling
25import cProfile
26import pstats
27
28# for serializing
29import pickle
30
31from coopr.pysp.convergence import *
32from coopr.pysp.scenariotree import *
33from coopr.pysp.ph import *
34from coopr.pysp.ef import *
35from coopr.opt.base import SolverFactory
36from coopr.opt.parallel import SolverManagerFactory
37
38#
39# utility method to construct an option parser for ph arguments, to be
40# supplied as an argument to the runph method.
41#
42
43def construct_ph_options_parser(usage_string):
44
45   parser = OptionParser()
46   parser.add_option("--verbose",
47                     help="Generate verbose output for both initialization and execution. Default is False.",
48                     action="store_true",
49                     dest="verbose",
50                     default=False)
51   parser.add_option("--report-solutions",
52                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
53                     action="store_true",
54                     dest="report_solutions",
55                     default=False)
56   parser.add_option("--report-weights",
57                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
58                     action="store_true",
59                     dest="report_weights",
60                     default=False)
61   parser.add_option("--model-directory",
62                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
63                     action="store",
64                     dest="model_directory",
65                     type="string",
66                     default=".")
67   parser.add_option("--instance-directory",
68                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
69                     action="store",
70                     dest="instance_directory",
71                     type="string",
72                     default=".")
73   parser.add_option("--solver",
74                     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
75                     action="store",
76                     dest="solver_type",
77                     type="string",
78                     default="cplex")
79   parser.add_option("--solver-manager",
80                     help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
81                     action="store",
82                     dest="solver_manager_type",
83                     type="string",
84                     default="serial")
85   parser.add_option("--scenario-solver-options",
86                     help="Solver options for all PH scenario sub-problems",
87                     action="append",
88                     dest="scenario_solver_options",
89                     type="string",
90                     default=[])
91   parser.add_option("--scenario-mipgap",
92                     help="Specifies the mipgap for all PH scenario sub-problems",
93                     action="store",
94                     dest="scenario_mipgap",
95                     type="float",
96                     default=None)
97   parser.add_option("--ef-solver-options",
98                     help="Solver options for the extension form problem",
99                     action="append",
100                     dest="ef_solver_options",
101                     type="string",
102                     default=[])
103   parser.add_option("--ef-mipgap",
104                     help="Specifies the mipgap for the EF solve",
105                     action="store",
106                     dest="ef_mipgap",
107                     type="float",
108                     default=None)
109   parser.add_option("--max-iterations",
110                     help="The maximal number of PH iterations. Default is 100.",
111                     action="store",
112                     dest="max_iterations",
113                     type="int",
114                     default=100)
115   parser.add_option("--default-rho",
116                     help="The default (global) rho for all blended variables. Default is 1.",
117                     action="store",
118                     dest="default_rho",
119                     type="float",
120                     default=1.0)
121   parser.add_option("--rho-cfgfile",
122                     help="The name of a configuration script to compute PH rho values. Default is None.",
123                     action="store",
124                     dest="rho_cfgfile",
125                     type="string",
126                     default=None)
127   parser.add_option("--bounds-cfgfile",
128                     help="The name of a configuration script to set variable bound values. Default is None.",
129                     action="store",
130                     dest="bounds_cfgfile",
131                     default=None)
132   parser.add_option("--enable-termdiff-convergence",
133                     help="Terminate PH based on the termdiff convergence metric. Default is True.",
134                     action="store_true",
135                     dest="enable_termdiff_convergence",
136                     default=True)
137   parser.add_option("--enable-normalized-termdiff-convergence",
138                     help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
139                     action="store_true",
140                     dest="enable_normalized_termdiff_convergence",
141                     default=False)
142   parser.add_option("--termdiff-threshold",
143                     help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
144                     action="store",
145                     dest="termdiff_threshold",
146                     type="float",
147                     default=0.01)
148   parser.add_option("--enable-free-discrete-count-convergence",
149                     help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
150                     action="store_true",
151                     dest="enable_free_discrete_count_convergence",
152                     default=False)
153   parser.add_option("--free-discrete-count-threshold",
154                     help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
155                     action="store",
156                     dest="free_discrete_count_threshold",
157                     type="float",
158                     default=20)
159   parser.add_option("--enable-ww-extensions",
160                     help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
161                     action="store_true",
162                     dest="enable_ww_extensions",
163                     default=False)
164   parser.add_option("--ww-extension-cfgfile",
165                     help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
166                     action="store",
167                     dest="ww_extension_cfgfile",
168                     type="string",
169                     default="")
170   parser.add_option("--ww-extension-suffixfile",
171                     help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
172                     action="store",
173                     dest="ww_extension_suffixfile",
174                     type="string",
175                     default="")
176   parser.add_option("--user-defined-extension",
177                     help="The name of a python module specifying a user-defined PH extension plugin.",
178                     action="store",
179                     dest="user_defined_extension",
180                     type="string",
181                     default=None)
182   parser.add_option("--write-ef",
183                     help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
184                     action="store_true",
185                     dest="write_ef",
186                     default=False)
187   parser.add_option("--solve-ef",
188                     help="Following write of the extensive form model, solve it.",
189                     action="store_true",
190                     dest="solve_ef",
191                     default=False)
192   parser.add_option("--ef-output-file",
193                     help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
194                     action="store",
195                     dest="ef_output_file",
196                     type="string",
197                     default="efout.lp")
198   parser.add_option("--suppress-continuous-variable-output",
199                     help="Eliminate PH-related output involving continuous variables.",
200                     action="store_true",
201                     dest="suppress_continuous_variable_output",
202                     default=False)
203   parser.add_option("--keep-solver-files",
204                     help="Retain temporary input and output files for scenario sub-problem solves",
205                     action="store_true",
206                     dest="keep_solver_files",
207                     default=False)
208   parser.add_option("--output-solver-logs",
209                     help="Output solver logs during scenario sub-problem solves",
210                     action="store_true",
211                     dest="output_solver_logs",
212                     default=False)
213   parser.add_option("--output-ef-solver-log",
214                     help="Output solver log during the extensive form solve",
215                     action="store_true",
216                     dest="output_ef_solver_log",
217                     default=False)
218   parser.add_option("--output-solver-results",
219                     help="Output solutions obtained after each scenario sub-problem solve",
220                     action="store_true",
221                     dest="output_solver_results",
222                     default=False)
223   parser.add_option("--output-times",
224                     help="Output timing statistics for various PH components",
225                     action="store_true",
226                     dest="output_times",
227                     default=False)
228   parser.add_option("--disable-warmstarts",
229                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
230                     action="store_true",
231                     dest="disable_warmstarts",
232                     default=False)
233   parser.add_option("--drop-proximal-terms",
234                     help="Eliminate proximal terms (i.e., the quadratic penalty terms) from the weighted PH objective. Default is False.",
235                     action="store_true",
236                     dest="drop_proximal_terms",
237                     default=False)
238   parser.add_option("--retain-quadratic-binary-terms",
239                     help="Do not linearize PH objective terms involving binary decision variables",
240                     action="store_true",
241                     dest="retain_quadratic_binary_terms",
242                     default=False)
243   parser.add_option("--linearize-nonbinary-penalty-terms",
244                     help="Approximate the PH quadratic term for non-binary variables with a piece-wise linear function, using the supplied number of equal-length pieces from each bound to the average",
245                     action="store",
246                     dest="linearize_nonbinary_penalty_terms",
247                     type="int",
248                     default=0)
249   parser.add_option("--breakpoint-strategy",
250                     help="Specify the strategy to distribute breakpoints on the [lb, ub] interval of each variable when linearizing. 0 indicates uniform distribution. 1 indicates breakpoints at the node min and max, uniformly in-between. 2 indicates more aggressive concentration of breakpoints near the observed node min/max.",
251                     action="store",
252                     dest="breakpoint_strategy",
253                     type="int",
254                     default=0)
255   parser.add_option("--checkpoint-interval",
256                     help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
257                     action="store",
258                     dest="checkpoint_interval",
259                     type="int",
260                     default=0)
261   parser.add_option("--restore-from-checkpoint",
262                     help="The name of the checkpoint file from which PH should be initialized. Default is \"\", indicating no checkpoint restoration",
263                     action="store",
264                     dest="restore_from_checkpoint",
265                     type="string",
266                     default="")
267   parser.add_option("--profile",
268                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
269                     action="store",
270                     dest="profile",
271                     type="int",
272                     default=0)
273   parser.add_option("--enable-gc",
274                     help="Enable the python garbage collecter. Default is True.",
275                     action="store_true",
276                     dest="enable_gc",
277                     default=True)
278
279   parser.usage=usage_string
280
281   return parser
282
283#
284# Create the reference model / instance and scenario tree instance for PH.
285#
286
287def load_reference_and_scenario_models(options):
288
289   #
290   # create and populate the reference model/instance pair.
291   #
292
293   reference_model = None
294   reference_instance = None
295
296   try:
297      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
298      if options.verbose is True:
299         print "Scenario reference model filename="+reference_model_filename
300      model_import = pyutilib.misc.import_file(reference_model_filename)
301      if "model" not in dir(model_import):
302         print ""
303         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
304         return None, None, None
305
306      if model_import.model is None:
307         print ""
308         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
309         return None, None, None
310 
311      reference_model = model_import.model
312   except IOError:
313      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
314      return None, None, None
315
316   try:
317      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
318      if options.verbose is True:
319         print "Scenario reference instance filename="+reference_instance_filename
320      reference_instance = reference_model.create(reference_instance_filename)
321   except IOError:
322      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
323      return None, None, None
324
325   #
326   # create and populate the scenario tree model
327   #
328
329   from coopr.pysp.util.scenariomodels import scenario_tree_model
330   scenario_tree_instance = None
331
332   try:
333      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
334      if options.verbose is True:
335         print "Scenario tree instance filename="+scenario_tree_instance_filename
336      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
337   except IOError:
338      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
339      return None, None, None
340   
341   #
342   # construct the scenario tree
343   #
344   scenario_tree = ScenarioTree(model=reference_instance,
345                                nodes=scenario_tree_instance.Nodes,
346                                nodechildren=scenario_tree_instance.Children,
347                                nodestages=scenario_tree_instance.NodeStage,
348                                nodeprobabilities=scenario_tree_instance.ConditionalProbability,
349                                stages=scenario_tree_instance.Stages,
350                                stagevariables=scenario_tree_instance.StageVariables,
351                                stagecostvariables=scenario_tree_instance.StageCostVariable,
352                                scenarios=scenario_tree_instance.Scenarios,
353                                scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
354                                scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
355
356   return reference_model, reference_instance, scenario_tree
357
358#
359# Create a PH object from a checkpoint.
360#
361def create_ph_from_checkpoint(options):
362
363   # we need to load the reference model, as pickle doesn't save contents of .py files!
364   try:
365      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
366      if options.verbose is True:
367         print "Scenario reference model filename="+reference_model_filename
368      model_import = pyutilib.misc.import_file(reference_model_filename)
369      if "model" not in dir(model_import):
370         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
371         return
372
373      if model_import.model is None:
374         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
375         return None
376 
377      reference_model = model_import.model
378   except IOError:
379      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
380      return None
381
382   # import the saved state
383     
384   try:
385      checkpoint_file = open(options.restore_from_checkpoint,"r")
386      ph = pickle.load(checkpoint_file)
387      checkpoint_file.close()
388     
389   except IOError, msg:
390      raise RuntimeError, msg
391
392   # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
393
394   raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
395
396   return ph
397
398#
399# Create a PH object from a checkpoint.
400#
401def create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree):
402
403   #
404   # print the input tree for validation/information purposes.
405   #
406   if options.verbose is True:
407      scenario_tree.pprint()
408
409   #
410   # validate the tree prior to doing anything serious
411   #
412   print ""
413   if scenario_tree.validate() is False:
414      print "***ERROR: Scenario tree is invalid****"
415      return None
416   else:
417      if options.verbose is True:
418         print "Scenario tree is valid!"
419   print ""
420
421   #
422   # if any of the ww extension configuration options are specified without the
423   # ww extension itself being enabled, halt and warn the user - this has led
424   # to confusion in the past, and will save user support time.
425   #
426   if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
427      print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
428      return None
429
430   if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
431      print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"         
432      return None
433
434   #
435   # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
436   #
437   if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
438      print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
439      return None
440
441   #
442   # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
443   # order only matters if both are specified.
444   #
445   if options.enable_ww_extensions is True:
446
447      from coopr.pysp import wwphextension
448
449      plugin = ExtensionPoint(IPHExtension)
450      if len(options.ww_extension_cfgfile) > 0:
451         plugin.service()._configuration_filename = options.ww_extension_cfgfile
452      if len(options.ww_extension_suffixfile) > 0:
453         plugin.service()._suffix_filename = options.ww_extension_suffixfile
454
455   if options.user_defined_extension is not None:
456      print "Trying to import user-defined PH extension module="+options.user_defined_extension
457      # JPW removed the exception handling logic, as the module importer
458      # can raise a broad array of exceptions.
459      __import__(options.user_defined_extension)
460      print "Module successfully loaded"
461
462   #
463   # construct the convergence "computer" class.
464   #
465   converger = None
466   # go with the non-defaults first, and then with the default.
467   if options.enable_free_discrete_count_convergence is True:
468      converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
469   elif options.enable_normalized_termdiff_convergence is True:
470      converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
471   else:
472      converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
473
474   
475   #
476   # construct and initialize PH
477   #
478   ph = ProgressiveHedging(max_iterations=options.max_iterations, \
479                           rho=options.default_rho, \
480                           rho_setter=options.rho_cfgfile, \
481                           bounds_setter=options.bounds_cfgfile, \
482                           solver=options.solver_type, \
483                           solver_manager=options.solver_manager_type, \
484                           scenario_solver_options=options.scenario_solver_options, \
485                           scenario_mipgap=options.scenario_mipgap, \
486                           keep_solver_files=options.keep_solver_files, \
487                           output_solver_log=options.output_solver_logs, \
488                           output_solver_results=options.output_solver_results, \
489                           verbose=options.verbose, \
490                           report_solutions=options.report_solutions, \
491                           report_weights=options.report_weights, \
492                           output_times=options.output_times, \
493                           disable_warmstarts=options.disable_warmstarts,
494                           drop_proximal_terms=options.drop_proximal_terms,
495                           retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
496                           linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
497                           breakpoint_strategy=options.breakpoint_strategy, \
498                           checkpoint_interval=options.checkpoint_interval)
499
500   ph.initialize(scenario_data_directory_name=os.path.expanduser(options.instance_directory), \
501                 model=reference_model, \
502                 model_instance=reference_instance, \
503                 scenario_tree=scenario_tree, \
504                 converger=converger)
505
506   if options.suppress_continuous_variable_output is True:
507      ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.
508
509   return ph
510
511#
512# Given a PH object, execute it and optionally solve the EF at the end.
513#
514
515def run_ph(options, ph):
516
517   #
518   # at this point, we have an initialized PH object by some means.
519   #
520   start_time = time.time()
521
522   #
523   # kick off the solve
524   #
525   ph.solve()
526
527   end_time = time.time()
528
529   print ""
530   print "Total PH execution time=%8.2f seconds" %(end_time - start_time)
531   print ""
532   if options.output_times is True:
533      ph.print_time_stats()
534
535   #
536   # write the extensive form, accounting for any fixed variables.
537   #
538   if (options.write_ef is True) or (options.solve_ef is True):
539      print ""
540      print "Writing EF for remainder problem"
541      print ""
542      write_ef(ph._scenario_tree, ph._instances, os.path.expanduser(options.ef_output_file))
543
544   #
545   # solve the extensive form.
546   #
547   if options.solve_ef is True:
548      print ""
549      print "Solving extensive form written to file="+os.path.expanduser(options.ef_output_file)
550      print ""
551
552      ef_solver = SolverFactory(options.solver_type)
553      if ef_solver is None:
554         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
555      if len(options.ef_solver_options) > 0:
556         print "Initializing ef solver with options="+str(options.ef_solver_options)         
557         ef_solver.set_options("".join(options.ef_solver_options))
558      if options.ef_mipgap is not None:
559         if (options.ef_mipgap < 0.0) or (options.ef_mipgap > 1.0):
560            raise ValueError, "Value of the mipgap parameter for the EF solve must be on the unit interval; value specified=" + `options.ef_mipgap`
561         else:
562            ef_solver.mipgap = options.ef_mipgap
563
564      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
565      if ef_solver is None:
566         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
567
568      print "Queuing extensive form solve"
569      ef_action_handle = ef_solver_manager.queue(os.path.expanduser(options.ef_output_file), opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
570      print "Waiting for extensive form solve"
571      ef_results = ef_solver_manager.wait_for(ef_action_handle)
572      print "Extensive form solve results:"
573      ef_results.write(num=1)
574   
575#
576# The main PH initialization / runner routine.
577#
578
579def exec_ph(options):
580
581   ph = None
582
583   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
584   if len(options.restore_from_checkpoint) > 0:
585
586      ph = create_ph_from_checkpoint(options)
587     
588   else:
589
590      reference_model, reference_instance, scenario_tree = load_reference_and_scenario_models(options)
591      if reference_model is None or reference_instance is None or scenario_tree is None:
592         return
593      ph = create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree)
594
595   if ph is None:
596      print "***FAILED TO CREATE PH OBJECT"
597      return
598
599   run_ph(options, ph)
600
601def run(args=None):
602
603    #
604    # Top-level command that executes the extensive form writer.
605    # This is segregated from run_ef_writer to enable profiling.
606    #
607
608    #
609    # Parse command-line options.
610    #
611    try:
612       ph_options_parser = construct_ph_options_parser("runph [options]")
613       (options, args) = ph_options_parser.parse_args(args=args)
614    except SystemExit:
615       # the parser throws a system exit if "-h" is specified - catch
616       # it to exit gracefully.
617       return
618
619    # for a one-pass execution, garbage collection doesn't make
620    # much sense - so it is disabled by default. Because: It drops
621    # the run-time by a factor of 3-4 on bigger instances.
622    if options.enable_gc is False:
623       gc.disable()
624    else:
625       gc.enable()
626
627    if options.profile > 0:
628        #
629        # Call the main PH routine with profiling.
630        #
631        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
632        tmp = cProfile.runctx('exec_ph(options)',globals(),locals(),tfile)
633        p = pstats.Stats(tfile).strip_dirs()
634        p.sort_stats('time', 'cum')
635        p = p.print_stats(options.profile)
636        p.print_callers(options.profile)
637        p.print_callees(options.profile)
638        p = p.sort_stats('cum','calls')
639        p.print_stats(options.profile)
640        p.print_callers(options.profile)
641        p.print_callees(options.profile)
642        p = p.sort_stats('calls')
643        p.print_stats(options.profile)
644        p.print_callers(options.profile)
645        p.print_callees(options.profile)
646        pyutilib.services.TempfileManager.clear_tempfiles()
647        ans = [tmp, None]
648    else:
649        #
650        # Call the main PH routine without profiling.
651        #
652        ans = exec_ph(options)
653
654    gc.enable()
655   
656    return ans
657
Note: See TracBrowser for help on using the repository browser.