source: coopr.pysp/trunk/coopr/pysp/phinit.py @ 2418

Last change on this file since 2418 was 2418, checked in by jwatson, 9 years ago

More restructuring and simplification of the PySP extensive writer code, which is now rather compact.

  • Property svn:executable set to *
File size: 27.2 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20
21# garbage collection control.
22import gc
23
24# for profiling
25import cProfile
26import pstats
27
28# for serializing
29import pickle
30
31from coopr.pysp.convergence import *
32from coopr.pysp.scenariotree import *
33from coopr.pysp.ph import *
34from coopr.pysp.ef import *
35from coopr.opt.base import SolverFactory
36from coopr.opt.parallel import SolverManagerFactory
37
38#
39# utility method to construct an option parser for ph arguments, to be
40# supplied as an argument to the runph method.
41#
42
43def construct_ph_options_parser(usage_string):
44
45   parser = OptionParser()
46   parser.add_option("--verbose",
47                     help="Generate verbose output for both initialization and execution. Default is False.",
48                     action="store_true",
49                     dest="verbose",
50                     default=False)
51   parser.add_option("--report-solutions",
52                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
53                     action="store_true",
54                     dest="report_solutions",
55                     default=False)
56   parser.add_option("--report-weights",
57                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
58                     action="store_true",
59                     dest="report_weights",
60                     default=False)
61   parser.add_option("--model-directory",
62                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
63                     action="store",
64                     dest="model_directory",
65                     type="string",
66                     default=".")
67   parser.add_option("--instance-directory",
68                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
69                     action="store",
70                     dest="instance_directory",
71                     type="string",
72                     default=".")
73   parser.add_option("--solver",
74                     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
75                     action="store",
76                     dest="solver_type",
77                     type="string",
78                     default="cplex")
79   parser.add_option("--solver-manager",
80                     help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
81                     action="store",
82                     dest="solver_manager_type",
83                     type="string",
84                     default="serial")
85   parser.add_option("--scenario-solver-options",
86                     help="Solver options for all PH scenario sub-problems",
87                     action="append",
88                     dest="scenario_solver_options",
89                     type="string",
90                     default=[])
91   parser.add_option("--scenario-mipgap",
92                     help="Specifies the mipgap for all PH scenario sub-problems",
93                     action="store",
94                     dest="scenario_mipgap",
95                     type="float",
96                     default=None)
97   parser.add_option("--ef-solver-options",
98                     help="Solver options for the extension form problem",
99                     action="append",
100                     dest="ef_solver_options",
101                     type="string",
102                     default=[])
103   parser.add_option("--ef-mipgap",
104                     help="Specifies the mipgap for the EF solve",
105                     action="store",
106                     dest="ef_mipgap",
107                     type="float",
108                     default=None)
109   parser.add_option("--max-iterations",
110                     help="The maximal number of PH iterations. Default is 100.",
111                     action="store",
112                     dest="max_iterations",
113                     type="int",
114                     default=100)
115   parser.add_option("--default-rho",
116                     help="The default (global) rho for all blended variables. Default is 1.",
117                     action="store",
118                     dest="default_rho",
119                     type="float",
120                     default=1.0)
121   parser.add_option("--rho-cfgfile",
122                     help="The name of a configuration script to compute PH rho values. Default is None.",
123                     action="store",
124                     dest="rho_cfgfile",
125                     type="string",
126                     default=None)
127   parser.add_option("--bounds-cfgfile",
128                     help="The name of a configuration script to set variable bound values. Default is None.",
129                     action="store",
130                     dest="bounds_cfgfile",
131                     default=None)
132   parser.add_option("--enable-termdiff-convergence",
133                     help="Terminate PH based on the termdiff convergence metric. Default is True.",
134                     action="store_true",
135                     dest="enable_termdiff_convergence",
136                     default=True)
137   parser.add_option("--enable-normalized-termdiff-convergence",
138                     help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
139                     action="store_true",
140                     dest="enable_normalized_termdiff_convergence",
141                     default=False)
142   parser.add_option("--termdiff-threshold",
143                     help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
144                     action="store",
145                     dest="termdiff_threshold",
146                     type="float",
147                     default=0.01)
148   parser.add_option("--enable-free-discrete-count-convergence",
149                     help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
150                     action="store_true",
151                     dest="enable_free_discrete_count_convergence",
152                     default=False)
153   parser.add_option("--free-discrete-count-threshold",
154                     help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
155                     action="store",
156                     dest="free_discrete_count_threshold",
157                     type="float",
158                     default=20)
159   parser.add_option("--enable-ww-extensions",
160                     help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
161                     action="store_true",
162                     dest="enable_ww_extensions",
163                     default=False)
164   parser.add_option("--ww-extension-cfgfile",
165                     help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
166                     action="store",
167                     dest="ww_extension_cfgfile",
168                     type="string",
169                     default="")
170   parser.add_option("--ww-extension-suffixfile",
171                     help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
172                     action="store",
173                     dest="ww_extension_suffixfile",
174                     type="string",
175                     default="")
176   parser.add_option("--user-defined-extension",
177                     help="The name of a python module specifying a user-defined PH extension plugin.",
178                     action="store",
179                     dest="user_defined_extension",
180                     type="string",
181                     default=None)
182   parser.add_option("--write-ef",
183                     help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
184                     action="store_true",
185                     dest="write_ef",
186                     default=False)
187   parser.add_option("--solve-ef",
188                     help="Following write of the extensive form model, solve it.",
189                     action="store_true",
190                     dest="solve_ef",
191                     default=False)
192   parser.add_option("--ef-output-file",
193                     help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
194                     action="store",
195                     dest="ef_output_file",
196                     type="string",
197                     default="efout.lp")
198   parser.add_option("--suppress-continuous-variable-output",
199                     help="Eliminate PH-related output involving continuous variables.",
200                     action="store_true",
201                     dest="suppress_continuous_variable_output",
202                     default=False)
203   parser.add_option("--keep-solver-files",
204                     help="Retain temporary input and output files for scenario sub-problem solves",
205                     action="store_true",
206                     dest="keep_solver_files",
207                     default=False)
208   parser.add_option("--output-solver-logs",
209                     help="Output solver logs during scenario sub-problem solves",
210                     action="store_true",
211                     dest="output_solver_logs",
212                     default=False)
213   parser.add_option("--output-ef-solver-log",
214                     help="Output solver log during the extensive form solve",
215                     action="store_true",
216                     dest="output_ef_solver_log",
217                     default=False)
218   parser.add_option("--output-solver-results",
219                     help="Output solutions obtained after each scenario sub-problem solve",
220                     action="store_true",
221                     dest="output_solver_results",
222                     default=False)
223   parser.add_option("--output-times",
224                     help="Output timing statistics for various PH components",
225                     action="store_true",
226                     dest="output_times",
227                     default=False)
228   parser.add_option("--disable-warmstarts",
229                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
230                     action="store_true",
231                     dest="disable_warmstarts",
232                     default=False)
233   parser.add_option("--drop-proximal-terms",
234                     help="Eliminate proximal terms (i.e., the quadratic penalty terms) from the weighted PH objective. Default is False.",
235                     action="store_true",
236                     dest="drop_proximal_terms",
237                     default=False)
238   parser.add_option("--retain-quadratic-binary-terms",
239                     help="Do not linearize PH objective terms involving binary decision variables",
240                     action="store_true",
241                     dest="retain_quadratic_binary_terms",
242                     default=False)
243   parser.add_option("--linearize-nonbinary-penalty-terms",
244                     help="Approximate the PH quadratic term for non-binary variables with a piece-wise linear function, using the supplied number of equal-length pieces from each bound to the average",
245                     action="store",
246                     dest="linearize_nonbinary_penalty_terms",
247                     type="int",
248                     default=0)
249   parser.add_option("--breakpoint-strategy",
250                     help="Specify the strategy to distribute breakpoints on the [lb, ub] interval of each variable when linearizing. 0 indicates uniform distribution. 1 indicates breakpoints at the node min and max, uniformly in-between. 2 indicates more aggressive concentration of breakpoints near the observed node min/max.",
251                     action="store",
252                     dest="breakpoint_strategy",
253                     type="int",
254                     default=0)
255   parser.add_option("--checkpoint-interval",
256                     help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
257                     action="store",
258                     dest="checkpoint_interval",
259                     type="int",
260                     default=0)
261   parser.add_option("--restore-from-checkpoint",
262                     help="The name of the checkpoint file from which PH should be initialized. Default is \"\", indicating no checkpoint restoration",
263                     action="store",
264                     dest="restore_from_checkpoint",
265                     type="string",
266                     default="")
267   parser.add_option("--profile",
268                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
269                     action="store",
270                     dest="profile",
271                     type="int",
272                     default=0)
273   parser.add_option("--disable-gc",
274                     help="Disable the python garbage collecter. Default is False.",
275                     action="store_true",
276                     dest="disable_gc",
277                     default=False)
278
279   parser.usage=usage_string
280
281   return parser
282
283#
284# Create the reference model / instance and scenario tree instance for PH.
285#
286
287def load_reference_and_scenario_models(options):
288
289   #
290   # create and populate the reference model/instance pair.
291   #
292
293   reference_model = None
294   reference_instance = None
295
296   try:
297      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
298      if options.verbose is True:
299         print "Scenario reference model filename="+reference_model_filename
300      model_import = pyutilib.misc.import_file(reference_model_filename)
301      if "model" not in dir(model_import):
302         print ""
303         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
304         return None, None, None, None
305
306      if model_import.model is None:
307         print ""
308         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
309         return None, None, None, None
310 
311      reference_model = model_import.model
312   except IOError:
313      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
314      return None, None, None, None
315
316   try:
317      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
318      if options.verbose is True:
319         print "Scenario reference instance filename="+reference_instance_filename
320      reference_instance = reference_model.create(reference_instance_filename)
321   except IOError:
322      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
323      return None, None, None, None
324
325   #
326   # create and populate the scenario tree model
327   #
328
329   from coopr.pysp.util.scenariomodels import scenario_tree_model
330   scenario_tree_instance = None
331
332   try:
333      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
334      if options.verbose is True:
335         print "Scenario tree instance filename="+scenario_tree_instance_filename
336      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
337   except IOError:
338      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
339      return None, None, None, None
340   
341   #
342   # construct the scenario tree
343   #
344   scenario_tree = ScenarioTree(scenarioinstance=reference_instance,
345                                scenariotreeinstance=scenario_tree_instance)
346
347   return reference_model, reference_instance, scenario_tree, scenario_tree_instance
348
349#
350# Create a PH object from a checkpoint.
351#
352def create_ph_from_checkpoint(options):
353
354   # we need to load the reference model, as pickle doesn't save contents of .py files!
355   try:
356      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
357      if options.verbose is True:
358         print "Scenario reference model filename="+reference_model_filename
359      model_import = pyutilib.misc.import_file(reference_model_filename)
360      if "model" not in dir(model_import):
361         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
362         return
363
364      if model_import.model is None:
365         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
366         return None
367 
368      reference_model = model_import.model
369   except IOError:
370      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
371      return None
372
373   # import the saved state
374     
375   try:
376      checkpoint_file = open(options.restore_from_checkpoint,"r")
377      ph = pickle.load(checkpoint_file)
378      checkpoint_file.close()
379     
380   except IOError, msg:
381      raise RuntimeError, msg
382
383   # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
384
385   raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
386
387   return ph
388
389#
390# Create a PH object from a checkpoint.
391#
392def create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree):
393
394   #
395   # print the input tree for validation/information purposes.
396   #
397   if options.verbose is True:
398      scenario_tree.pprint()
399
400   #
401   # validate the tree prior to doing anything serious
402   #
403   print ""
404   if scenario_tree.validate() is False:
405      print "***ERROR: Scenario tree is invalid****"
406      return None
407   else:
408      if options.verbose is True:
409         print "Scenario tree is valid!"
410   print ""
411
412   #
413   # if any of the ww extension configuration options are specified without the
414   # ww extension itself being enabled, halt and warn the user - this has led
415   # to confusion in the past, and will save user support time.
416   #
417   if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
418      print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
419      return None
420
421   if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
422      print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"         
423      return None
424
425   #
426   # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
427   #
428   if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
429      print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
430      return None
431
432   #
433   # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
434   # order only matters if both are specified.
435   #
436   if options.enable_ww_extensions is True:
437
438      from coopr.pysp import wwphextension
439
440      plugin = ExtensionPoint(IPHExtension)
441      if len(options.ww_extension_cfgfile) > 0:
442         plugin.service()._configuration_filename = options.ww_extension_cfgfile
443      if len(options.ww_extension_suffixfile) > 0:
444         plugin.service()._suffix_filename = options.ww_extension_suffixfile
445
446   if options.user_defined_extension is not None:
447      print "Trying to import user-defined PH extension module="+options.user_defined_extension
448      # JPW removed the exception handling logic, as the module importer
449      # can raise a broad array of exceptions.
450      __import__(options.user_defined_extension)
451      print "Module successfully loaded"
452
453   #
454   # construct the convergence "computer" class.
455   #
456   converger = None
457   # go with the non-defaults first, and then with the default.
458   if options.enable_free_discrete_count_convergence is True:
459      converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
460   elif options.enable_normalized_termdiff_convergence is True:
461      converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
462   else:
463      converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
464
465   
466   #
467   # construct and initialize PH
468   #
469   ph = ProgressiveHedging(max_iterations=options.max_iterations, \
470                           rho=options.default_rho, \
471                           rho_setter=options.rho_cfgfile, \
472                           bounds_setter=options.bounds_cfgfile, \
473                           solver=options.solver_type, \
474                           solver_manager=options.solver_manager_type, \
475                           scenario_solver_options=options.scenario_solver_options, \
476                           scenario_mipgap=options.scenario_mipgap, \
477                           keep_solver_files=options.keep_solver_files, \
478                           output_solver_log=options.output_solver_logs, \
479                           output_solver_results=options.output_solver_results, \
480                           verbose=options.verbose, \
481                           report_solutions=options.report_solutions, \
482                           report_weights=options.report_weights, \
483                           output_times=options.output_times, \
484                           disable_warmstarts=options.disable_warmstarts,
485                           drop_proximal_terms=options.drop_proximal_terms,
486                           retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
487                           linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
488                           breakpoint_strategy=options.breakpoint_strategy, \
489                           checkpoint_interval=options.checkpoint_interval)
490
491   ph.initialize(scenario_data_directory_name=os.path.expanduser(options.instance_directory), \
492                 model=reference_model, \
493                 model_instance=reference_instance, \
494                 scenario_tree=scenario_tree, \
495                 converger=converger)
496
497   if options.suppress_continuous_variable_output is True:
498      ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.
499
500   return ph
501
502#
503# Given a PH object, execute it and optionally solve the EF at the end.
504#
505
506def run_ph(options, ph):
507
508   #
509   # at this point, we have an initialized PH object by some means.
510   #
511   start_time = time.time()
512
513   #
514   # kick off the solve
515   #
516   ph.solve()
517
518   end_time = time.time()
519
520   print ""
521   print "Total PH execution time=%8.2f seconds" %(end_time - start_time)
522   print ""
523   if options.output_times is True:
524      ph.print_time_stats()
525
526   #
527   # write the extensive form, accounting for any fixed variables.
528   #
529   if (options.write_ef is True) or (options.solve_ef is True):
530      print ""
531      print "Writing EF for remainder problem"
532      print ""
533      create_and_write_ef(ph._scenario_tree, ph._instances, os.path.expanduser(options.ef_output_file))
534
535   #
536   # solve the extensive form.
537   #
538   if options.solve_ef is True:
539      print ""
540      print "Solving extensive form written to file="+os.path.expanduser(options.ef_output_file)
541      print ""
542
543      ef_solver = SolverFactory(options.solver_type)
544      if ef_solver is None:
545         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
546      if len(options.ef_solver_options) > 0:
547         print "Initializing ef solver with options="+str(options.ef_solver_options)         
548         ef_solver.set_options("".join(options.ef_solver_options))
549      if options.ef_mipgap is not None:
550         if (options.ef_mipgap < 0.0) or (options.ef_mipgap > 1.0):
551            raise ValueError, "Value of the mipgap parameter for the EF solve must be on the unit interval; value specified=" + `options.ef_mipgap`
552         else:
553            ef_solver.mipgap = options.ef_mipgap
554
555      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
556      if ef_solver is None:
557         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
558
559      print "Queuing extensive form solve"
560      ef_action_handle = ef_solver_manager.queue(os.path.expanduser(options.ef_output_file), opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
561      print "Waiting for extensive form solve"
562      ef_results = ef_solver_manager.wait_for(ef_action_handle)
563      print "Extensive form solve results:"
564      ef_results.write(num=1)
565   
566#
567# The main PH initialization / runner routine.
568#
569
570def exec_ph(options):
571
572   ph = None
573
574   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
575   if len(options.restore_from_checkpoint) > 0:
576
577      ph = create_ph_from_checkpoint(options)
578     
579   else:
580
581      reference_model, reference_instance, scenario_tree, scenario_tree_instance = load_reference_and_scenario_models(options)
582      if reference_model is None or reference_instance is None or scenario_tree is None:
583         return
584      ph = create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree)
585
586   if ph is None:
587      print "***FAILED TO CREATE PH OBJECT"
588      return
589
590   run_ph(options, ph)
591
592def run(args=None):
593
594    #
595    # Top-level command that executes the extensive form writer.
596    # This is segregated from run_ef_writer to enable profiling.
597    #
598
599    #
600    # Parse command-line options.
601    #
602    try:
603       ph_options_parser = construct_ph_options_parser("runph [options]")
604       (options, args) = ph_options_parser.parse_args(args=args)
605    except SystemExit:
606       # the parser throws a system exit if "-h" is specified - catch
607       # it to exit gracefully.
608       return
609
610    if options.disable_gc is True:
611       gc.disable()
612    else:
613       gc.enable()
614
615    if options.profile > 0:
616        #
617        # Call the main PH routine with profiling.
618        #
619        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
620        tmp = cProfile.runctx('exec_ph(options)',globals(),locals(),tfile)
621        p = pstats.Stats(tfile).strip_dirs()
622        p.sort_stats('time', 'cum')
623        p = p.print_stats(options.profile)
624        p.print_callers(options.profile)
625        p.print_callees(options.profile)
626        p = p.sort_stats('cum','calls')
627        p.print_stats(options.profile)
628        p.print_callers(options.profile)
629        p.print_callees(options.profile)
630        p = p.sort_stats('calls')
631        p.print_stats(options.profile)
632        p.print_callers(options.profile)
633        p.print_callees(options.profile)
634        pyutilib.services.TempfileManager.clear_tempfiles()
635        ans = [tmp, None]
636    else:
637        #
638        # Call the main PH routine without profiling.
639        #
640        ans = exec_ph(options)
641
642    gc.enable()
643   
644    return ans
Note: See TracBrowser for help on using the repository browser.