source: coopr.pysp/stable/coopr/pysp/phinit.py @ 3286

Last change on this file since 3286 was 3286, checked in by wehart, 10 years ago

Merged revisions 3187-3285 via svnmerge from
https://software.sandia.gov/svn/public/coopr/coopr.pysp/trunk

........

r3201 | jwatson | 2010-10-29 13:18:17 -0600 (Fri, 29 Oct 2010) | 3 lines


Inverting order of .dat files in PySP when loading from a node representation - now root-to-leaf, instead of leaf-to-root. This allows for deeper-in-the-tree nodes to over-write parameter values defined higher in the tree, which is a more "expected" behavior than the converse. The real answer is to throw an exception if a parameter is re-defined, but we're not there yet.

........

r3217 | jwatson | 2010-11-05 11:29:42 -0600 (Fri, 05 Nov 2010) | 3 lines


Various updates to support heteogeneous index sets in PH for different nodes in the scenario tree - more work / testing remains.

........

r3218 | jwatson | 2010-11-05 12:35:51 -0600 (Fri, 05 Nov 2010) | 3 lines


More changes associated with generalizing the PySP index structures from per-stage to per-node.

........

r3220 | jwatson | 2010-11-05 20:28:29 -0600 (Fri, 05 Nov 2010) | 3 lines


Various fixes to the WW PH extension, bringing it in compliance to previous commit changes.

........

r3221 | jwatson | 2010-11-05 20:43:40 -0600 (Fri, 05 Nov 2010) | 1 line


Removing some older PySP test problems from the repository

........

r3222 | jwatson | 2010-11-05 20:55:15 -0600 (Fri, 05 Nov 2010) | 1 line


Moving PySP forestry example to local sandbox, to streamline distribution.

........

r3261 | jwatson | 2010-11-29 15:26:46 -0700 (Mon, 29 Nov 2010) | 9 lines


Adding two options to the runef and runph pysp scripts, to facilitate scenario downsampling - the case where you have a big tree, but you don't want to use it all.


The options are:
--scenario-tree-downsample-fraction=X
--scenario-tree-random-seed


The options are fairly self-explanatory - the only possible nuance is that the downsample fraction is the fraction of scenarios retained.

........

r3263 | jwatson | 2010-12-01 10:47:21 -0700 (Wed, 01 Dec 2010) | 3 lines


Corrected issue with cvar generation introduced a while back.

........

r3264 | jwatson | 2010-12-01 11:16:01 -0700 (Wed, 01 Dec 2010) | 3 lines


Adding PySP extensive form tests.

........

r3265 | wehart | 2010-12-01 13:19:56 -0700 (Wed, 01 Dec 2010) | 2 lines


Auxmenting the filter

........

r3266 | wehart | 2010-12-01 13:58:59 -0700 (Wed, 01 Dec 2010) | 2 lines


Adding further diagnostics to the filter.

........

r3267 | wehart | 2010-12-01 14:12:15 -0700 (Wed, 01 Dec 2010) | 2 lines


Another attempt to fix this filter...

........

r3271 | wehart | 2010-12-01 15:51:23 -0700 (Wed, 01 Dec 2010) | 4 lines


Simplifying filter.


The error was really a Python 2.5 portability issue in EF. :(

........

r3275 | jwatson | 2010-12-02 19:35:52 -0700 (Thu, 02 Dec 2010) | 3 lines


Fixing Python 2.5-related issue with string.translate in phutils.py - using None as the translation table is not allowed in Python 2.5.

........

  • Property svn:executable set to *
File size: 31.4 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import os
13import sys
14
15from optparse import OptionParser, OptionGroup
16
17import pyutilib.services
18import pyutilib.misc
19
20# garbage collection control.
21import gc
22
23# for profiling
24try:
25    import cProfile as profile
26except ImportError:
27    import profile
28import pstats
29
30# for serializing
31import pickle
32
33from coopr.pysp.convergence import *
34from coopr.pysp.scenariotree import *
35from coopr.pysp.ph import *
36from coopr.pysp.ef import *
37from coopr.opt.base import SolverFactory
38from coopr.opt.parallel import SolverManagerFactory
39
40from pyutilib.component.core import ExtensionPoint
41from coopr.pysp.solutionwriter import ISolutionWriterExtension
42
43#
44# utility method to construct an option parser for ph arguments,
45# to be supplied as an argument to the runph method.
46#
47
48def construct_ph_options_parser(usage_string):
49
50   parser = OptionParser()
51   parser.usage = usage_string
52
53   # NOTE: these groups should eventually be queried from the PH, scenario tree, etc. classes (to facilitate re-use).
54   inputOpts        = OptionGroup( parser, 'Input Options' )
55   scenarioTreeOpts = OptionGroup( parser, 'Scenario Tree Options' )
56   phOpts           = OptionGroup( parser, 'PH Options' )
57   solverOpts       = OptionGroup( parser, 'Solver Options' )
58   postprocessOpts  = OptionGroup( parser, 'Postprocessing Options' )   
59   outputOpts       = OptionGroup( parser, 'Output Options' )
60   otherOpts        = OptionGroup( parser, 'Other Options' )
61   parser.add_option_group( inputOpts )
62   parser.add_option_group( scenarioTreeOpts )
63   parser.add_option_group( phOpts )
64   parser.add_option_group( solverOpts )
65   parser.add_option_group( postprocessOpts )   
66   parser.add_option_group( outputOpts )
67   parser.add_option_group( otherOpts )
68
69   inputOpts.add_option('-m','--model-directory',
70     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
71     action="store",
72     dest="model_directory",
73     type="string",
74     default=".")
75   inputOpts.add_option('-i','--instance-directory',
76     help="The directory in which all instance (reference and scenario) definitions are stored. Default is '.'.",
77     action="store",
78     dest="instance_directory",
79     type="string",
80     default=".")
81   inputOpts.add_option('--bounds-cfgfile',
82     help="The name of a configuration script to set variable bound values. Default is None.",
83     action="store",
84     dest="bounds_cfgfile",
85     default=None)
86
87   scenarioTreeOpts.add_option('--scenario-tree-seed',
88     help="The random seed associated with manipulation operations on the scenario tree (e.g., down-sampling). Default is 0, indicating unassigned.",
89     action="store",
90     dest="scenario_tree_random_seed",
91     type="int",
92     default=None)
93   scenarioTreeOpts.add_option('--scenario-tree-downsample-fraction',
94     help="The proportion of the scenarios in the scenario tree that are actually used. Specific scenarios are selected at random. Default is 1.0, indicating no down-sampling.",
95     action="store",
96     dest="scenario_tree_downsample_fraction",
97     type="float",
98     default=1.0)   
99
100   phOpts.add_option('-r','--default-rho',
101     help="The default (global) rho for all blended variables. Default is 1.",
102     action="store",
103     dest="default_rho",
104     type="float",
105     default=1.0)
106   phOpts.add_option('--rho-cfgfile',
107     help="The name of a configuration script to compute PH rho values. Default is None.",
108     action="store",
109     dest="rho_cfgfile",
110     type="string",
111     default=None)
112   phOpts.add_option('--max-iterations',
113     help="The maximal number of PH iterations. Default is 100.",
114     action="store",
115     dest="max_iterations",
116     type="int",
117     default=100)
118   phOpts.add_option('--termdiff-threshold',
119     help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
120     action="store",
121     dest="termdiff_threshold",
122     type="float",
123     default=0.01)
124   phOpts.add_option('--enable-free-discrete-count-convergence',
125     help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
126     action="store_true",
127     dest="enable_free_discrete_count_convergence",
128     default=False)
129   phOpts.add_option('--enable-normalized-termdiff-convergence',
130     help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
131     action="store_true",
132     dest="enable_normalized_termdiff_convergence",
133     default=False)
134   phOpts.add_option('--enable-termdiff-convergence',
135     help="Terminate PH based on the termdiff convergence metric. Default is True.",
136     action="store_true",
137     dest="enable_termdiff_convergence",
138     default=True)
139   phOpts.add_option('--free-discrete-count-threshold',
140     help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
141     action="store",
142     dest="free_discrete_count_threshold",
143     type="float",
144     default=20)
145   phOpts.add_option('--linearize-nonbinary-penalty-terms',
146     help="Approximate the PH quadratic term for non-binary variables with a piece-wise linear function, using the supplied number of equal-length pieces from each bound to the average",
147     action="store",
148     dest="linearize_nonbinary_penalty_terms",
149     type="int",
150     default=0)
151   phOpts.add_option('--breakpoint-strategy',
152     help="Specify the strategy to distribute breakpoints on the [lb, ub] interval of each variable when linearizing. 0 indicates uniform distribution. 1 indicates breakpoints at the node min and max, uniformly in-between. 2 indicates more aggressive concentration of breakpoints near the observed node min/max.",
153     action="store",
154     dest="breakpoint_strategy",
155     type="int",
156     default=0)   
157   phOpts.add_option('--retain-quadratic-binary-terms',
158     help="Do not linearize PH objective terms involving binary decision variables",
159     action="store_true",
160     dest="retain_quadratic_binary_terms",
161     default=False)
162   phOpts.add_option('--drop-proximal-terms',
163     help="Eliminate proximal terms (i.e., the quadratic penalty terms) from the weighted PH objective. Default is False.",
164     action="store_true",
165     dest="drop_proximal_terms",
166     default=False)
167   phOpts.add_option('--enable-ww-extensions',
168     help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
169     action="store_true",
170     dest="enable_ww_extensions",
171     default=False)
172   phOpts.add_option('--ww-extension-cfgfile',
173     help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
174     action="store",
175     dest="ww_extension_cfgfile",
176     type="string",
177     default="")
178   phOpts.add_option('--ww-extension-suffixfile',
179     help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
180     action="store",
181     dest="ww_extension_suffixfile",
182     type="string",
183     default="")
184   phOpts.add_option('--user-defined-extension',
185     help="The name of a python module specifying a user-defined PH extension plugin.",
186     action="store",
187     dest="user_defined_extension",
188     type="string",
189     default=None)
190   phOpts.add_option("--linearize-expressions",
191     help="EXPERIMENTAL: An option intended for use on linear or mixed-integer models " \
192          "in which expression trees in a model (constraints or objectives) are compacted " \
193          "into a more memory-efficient and concise form. The trees themselves are eliminated. ",
194     action="store_true",
195     dest="linearize_expressions",
196     default=False)
197   phOpts.add_option("--simplify-expressions",
198     help="Enable expression simplification during both model instance creation and any " \
199          "subsequent modifications of the model, e.g., during manipulation of the objective.",
200     action="store_true",
201     dest="simplify_expressions",
202     default=False)
203
204   solverOpts.add_option('--scenario-mipgap',
205     help="Specifies the mipgap for all PH scenario sub-problems",
206     action="store",
207     dest="scenario_mipgap",
208     type="float",
209     default=None)
210   solverOpts.add_option('--scenario-solver-options',
211     help="Solver options for all PH scenario sub-problems",
212     action="append",
213     dest="scenario_solver_options",
214     type="string",
215     default=[])
216   solverOpts.add_option('--solver',
217     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
218     action="store",
219     dest="solver_type",
220     type="string",
221     default="cplex")
222   solverOpts.add_option('--solver-manager',
223     help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
224     action="store",
225     dest="solver_manager_type",
226     type="string",
227     default="serial")
228   solverOpts.add_option('--disable-warmstarts',
229     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
230     action="store_true",
231     dest="disable_warmstarts",
232     default=False)
233
234   postprocessOpts.add_option('--ef-output-file',
235     help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
236     action="store",
237     dest="ef_output_file",
238     type="string",
239     default="efout.lp")
240   postprocessOpts.add_option('--solve-ef',
241     help="Following write of the extensive form model, solve it.",
242     action="store_true",
243     dest="solve_ef",
244     default=False)
245   postprocessOpts.add_option('--ef-mipgap',
246     help="Specifies the mipgap for the EF solve",
247     action="store",
248     dest="ef_mipgap",
249     type="float",
250     default=None)
251   postprocessOpts.add_option('--ef-solver-options',
252     help="Solver options for the extension form problem",
253     action="append",
254     dest="ef_solver_options",
255     type="string",
256     default=[])   
257   postprocessOpts.add_option('--output-ef-solver-log',
258     help="Output solver log during the extensive form solve",
259     action="store_true",
260     dest="output_ef_solver_log",
261     default=False)
262   
263   outputOpts.add_option('--output-scenario-tree-solution',
264     help="Report the full solution (even leaves) in scenario tree format upon termination. Values represent averages, so convergence is not an issue. Default is False.",
265     action="store_true",
266     dest="output_scenario_tree_solution",
267     default=False)
268   outputOpts.add_option('--output-solver-logs',
269     help="Output solver logs during scenario sub-problem solves",
270     action="store_true",
271     dest="output_solver_logs",
272     default=False)
273   outputOpts.add_option('--output-solver-results',
274     help="Output solutions obtained after each scenario sub-problem solve",
275     action="store_true",
276     dest="output_solver_results",
277     default=False)
278   outputOpts.add_option('--output-times',
279     help="Output timing statistics for various PH components",
280     action="store_true",
281     dest="output_times",
282     default=False)
283   outputOpts.add_option('--report-only-statistics',
284     help="When reporting solutions (if enabled), only output per-variable statistics - not the individual scenario values. Default is False.",
285     action="store_true",
286     dest="report_only_statistics",
287     default=False)
288   outputOpts.add_option('--report-solutions',
289     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
290     action="store_true",
291     dest="report_solutions",
292     default=False)
293   outputOpts.add_option('--report-weights',
294     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
295     action="store_true",
296     dest="report_weights",
297     default=False)
298   outputOpts.add_option('--restore-from-checkpoint',
299     help="The name of the checkpoint file from which PH should be initialized. Default is \"\", indicating no checkpoint restoration",
300     action="store",
301     dest="restore_from_checkpoint",
302     type="string",
303     default="")
304   outputOpts.add_option('--solution-writer',
305     help="The plugin invoked to write the scenario tree solution. Defaults to the empty list.",
306     action="append",
307     dest="solution_writer",
308     type="string",
309     default = [])
310   outputOpts.add_option('--suppress-continuous-variable-output',
311     help="Eliminate PH-related output involving continuous variables.",
312     action="store_true",
313     dest="suppress_continuous_variable_output",
314     default=False)
315   outputOpts.add_option('--verbose',
316     help="Generate verbose output for both initialization and execution. Default is False.",
317     action="store_true",
318     dest="verbose",
319     default=False)
320   outputOpts.add_option('--write-ef',
321     help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
322     action="store_true",
323     dest="write_ef",
324     default=False)
325
326   otherOpts.add_option('--disable-gc',
327     help="Disable the python garbage collecter. Default is False.",
328     action="store_true",
329     dest="disable_gc",
330     default=False)
331   otherOpts.add_option('--keep-solver-files',
332     help="Retain temporary input and output files for scenario sub-problem solves",
333     action="store_true",
334     dest="keep_solver_files",
335     default=False)
336   otherOpts.add_option('--profile',
337     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
338     action="store",
339     dest="profile",
340     type="int",
341     default=0)
342   otherOpts.add_option('--checkpoint-interval',
343     help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
344     action="store",
345     dest="checkpoint_interval",
346     type="int",
347     default=0)
348   otherOpts.add_option('--traceback',
349     help="When an exception is thrown, show the entire call stack. Ignored if profiling is enabled. Default is False.",
350     action="store_true",
351     dest="traceback",
352     default=False)   
353
354   return parser
355
356#
357# Create the reference model / instance and scenario tree instance for PH.
358# IMPT: This method should be moved into a more generic module - it has nothing
359#       to do with PH, and is used elsewhere (by routines that shouldn't have
360#       to know about PH).
361#
362
363def load_reference_and_scenario_models(options):
364
365   #
366   # create and populate the reference model/instance pair.
367   #
368
369   reference_model = None
370   reference_instance = None
371
372   try:
373      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
374      if options.verbose is True:
375         print "Scenario reference model filename="+reference_model_filename
376      model_import = pyutilib.misc.import_file(reference_model_filename)
377      if "model" not in dir(model_import):
378         print ""
379         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
380         return None, None, None, None
381
382      if model_import.model is None:
383         print ""
384         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
385         return None, None, None, None
386
387      reference_model = model_import.model
388   except IOError:
389      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
390      return None, None, None, None
391
392   try:
393      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
394      if options.verbose is True:
395         print "Scenario reference instance filename="+reference_instance_filename
396      reference_instance = reference_model.create(reference_instance_filename, preprocess=False, simplify=options.simplify_expressions)
397      # IMPT: disable canonical representation construction for ASL solvers.
398      #       this is a hack, in that we need to address encodings and
399      #       the like at a more general level.
400      if options.solver_type == "asl":
401         reference_instance.skip_canonical_repn = True
402      else:
403         reference_instance.preprocess()
404     
405   except IOError:
406      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
407      return None, None, None, None
408
409   #
410   # create and populate the scenario tree model
411   #
412
413   from coopr.pysp.util.scenariomodels import scenario_tree_model
414   scenario_tree_instance = None
415
416   try:
417      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
418      if options.verbose is True:
419         print "Scenario tree instance filename="+scenario_tree_instance_filename
420      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename, simplify=options.simplify_expressions)
421   except IOError:
422      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
423      return None, None, None, None
424
425   #
426   # construct the scenario tree
427   #
428   scenario_tree = ScenarioTree(scenarioinstance=reference_instance,
429                                scenariotreeinstance=scenario_tree_instance)
430
431   #
432   # compress/down-sample the scenario tree, if operation is required.
433   #
434   if options.scenario_tree_downsample_fraction < 1.0:
435       
436      scenario_tree.downsample(options.scenario_tree_downsample_fraction, options.scenario_tree_random_seed, options.verbose)
437
438   return reference_model, reference_instance, scenario_tree, scenario_tree_instance
439
440#
441# Create a PH object from a (pickl) checkpoint. Experimental at the moment.
442#
443def create_ph_from_checkpoint(options):
444
445   # we need to load the reference model, as pickle doesn't save contents of .py files!
446   try:
447      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
448      if options.verbose is True:
449         print "Scenario reference model filename="+reference_model_filename
450      model_import = pyutilib.misc.import_file(reference_model_filename)
451      if "model" not in dir(model_import):
452         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
453         return
454
455      if model_import.model is None:
456         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
457         return None
458
459      reference_model = model_import.model
460   except IOError:
461      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
462      return None
463
464   # import the saved state
465
466   try:
467      checkpoint_file = open(options.restore_from_checkpoint,"r")
468      ph = pickle.load(checkpoint_file)
469      checkpoint_file.close()
470
471   except IOError, msg:
472      raise RuntimeError, msg
473
474   # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
475
476   raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
477
478   return ph
479
480#
481# Create a PH object from scratch.
482#
483
484def create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree):
485
486   #
487   # print the input tree for validation/information purposes.
488   #
489   if options.verbose is True:
490      scenario_tree.pprint()
491
492   #
493   # validate the tree prior to doing anything serious
494   #
495   if scenario_tree.validate() is False:
496      print "***ERROR: Scenario tree is invalid****"
497      return None
498   else:
499      if options.verbose is True:
500         print "Scenario tree is valid!"
501
502   #
503   # if any of the ww extension configuration options are specified without the
504   # ww extension itself being enabled, halt and warn the user - this has led
505   # to confusion in the past, and will save user support time.
506   #
507   if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
508      print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
509      return None
510
511   if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
512      print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"
513      return None
514
515   #
516   # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
517   #
518   if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
519      print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
520      return None
521
522   #
523   # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
524   # order only matters if both are specified.
525   #
526   if options.enable_ww_extensions is True:
527
528      from coopr.pysp import wwphextension
529
530      plugin = ExtensionPoint(IPHExtension)
531      if len(options.ww_extension_cfgfile) > 0:
532         plugin.service()._configuration_filename = options.ww_extension_cfgfile
533      if len(options.ww_extension_suffixfile) > 0:
534         plugin.service()._suffix_filename = options.ww_extension_suffixfile
535
536   if options.user_defined_extension is not None:
537      print "Trying to import user-defined PH extension module="+options.user_defined_extension
538      # JPW removed the exception handling logic, as the module importer
539      # can raise a broad array of exceptions.
540      __import__(options.user_defined_extension)
541      print "Module successfully loaded"
542
543   #
544   # construct the convergence "computer" class.
545   #
546   converger = None
547   # go with the non-defaults first, and then with the default.
548   if options.enable_free_discrete_count_convergence is True:
549      converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
550   elif options.enable_normalized_termdiff_convergence is True:
551      converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)
552   else:
553      converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)
554
555
556   #
557   # construct and initialize PH
558   #
559   ph = ProgressiveHedging(max_iterations=options.max_iterations, \
560                           rho=options.default_rho, \
561                           rho_setter=options.rho_cfgfile, \
562                           bounds_setter=options.bounds_cfgfile, \
563                           solver=options.solver_type, \
564                           solver_manager=options.solver_manager_type, \
565                           output_scenario_tree_solution=options.output_scenario_tree_solution, \
566                           scenario_solver_options=options.scenario_solver_options, \
567                           scenario_mipgap=options.scenario_mipgap, \
568                           keep_solver_files=options.keep_solver_files, \
569                           output_solver_log=options.output_solver_logs, \
570                           output_solver_results=options.output_solver_results, \
571                           verbose=options.verbose, \
572                           report_solutions=options.report_solutions, \
573                           report_weights=options.report_weights, \
574                           report_only_statistics=options.report_only_statistics, \
575                           output_times=options.output_times, \
576                           disable_warmstarts=options.disable_warmstarts,
577                           drop_proximal_terms=options.drop_proximal_terms,
578                           retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
579                           linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
580                           breakpoint_strategy=options.breakpoint_strategy, \
581                           checkpoint_interval=options.checkpoint_interval, \
582                           simplify_expressions=options.simplify_expressions)
583
584   ph.initialize(scenario_data_directory_name=os.path.expanduser(options.instance_directory), \
585                 model=reference_model, \
586                 model_instance=reference_instance, \
587                 scenario_tree=scenario_tree, \
588                 converger=converger, \
589                 linearize=options.linearize_expressions)
590
591   if options.suppress_continuous_variable_output is True:
592      ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.
593
594   return ph
595
596
597
598
599#
600# Given a PH object, execute it and optionally solve the EF at the end.
601#
602
603def run_ph(options, ph):
604
605   #
606   # at this point, we have an initialized PH object by some means.
607   #
608   start_time = time.time()
609
610   #
611   # kick off the solve
612   #
613   ph.solve()
614
615   end_time = time.time()
616
617   print ""
618   print "Total PH execution time=%8.2f seconds" %(end_time - start_time)
619   print ""
620   if options.output_times is True:
621      ph.print_time_stats()
622
623   solution_writer_plugins = ExtensionPoint(ISolutionWriterExtension)
624   for plugin in solution_writer_plugins:
625      plugin.write(ph._scenario_tree, "ph")
626
627   # store the binding instance, if created, in order to load
628   # the solution back into the scenario tree.
629   binding_instance = None
630
631   #
632   # write the extensive form, accounting (implicitly) for any fixed variables.
633   #
634   if (options.write_ef is True) or (options.solve_ef is True):
635      print ""
636      print "Writing EF for remainder problem"
637      print ""
638      binding_instance = create_and_write_ef(ph._scenario_tree, ph._instances, os.path.expanduser(options.ef_output_file))
639
640   #
641   # solve the extensive form and load the solution back into the PH scenario tree.
642   # contents from the PH solve will obviously be over-written!
643   #
644   if options.solve_ef is True:
645      print ""
646      print "Solving extensive form written to file="+os.path.expanduser(options.ef_output_file)
647      print ""
648
649      ef_solver = SolverFactory(options.solver_type)
650      if ef_solver is None:
651         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
652      if len(options.ef_solver_options) > 0:
653         print "Initializing ef solver with options="+str(options.ef_solver_options)
654         ef_solver.set_options("".join(options.ef_solver_options))
655      if options.ef_mipgap is not None:
656         if (options.ef_mipgap < 0.0) or (options.ef_mipgap > 1.0):
657            raise ValueError, "Value of the mipgap parameter for the EF solve must be on the unit interval; value specified=" + `options.ef_mipgap`
658         else:
659            ef_solver.mipgap = options.ef_mipgap
660
661      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
662      if ef_solver is None:
663         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
664
665      print "Queuing extensive form solve"
666      ef_action_handle = ef_solver_manager.queue(os.path.expanduser(options.ef_output_file), opt=ef_solver, tee=options.output_ef_solver_log)
667      print "Waiting for extensive form solve"
668      ef_results = ef_solver_manager.wait_for(ef_action_handle)
669
670      load_ef_solution(ef_results, binding_instance, ph._instances)
671      ph._scenario_tree.snapshotSolutionFromInstances(ph._instances)
672
673      print ""
674      print "Extensive form solution:"
675      ph._scenario_tree.pprintSolution()
676      print ""
677      print "Extensive form costs:"
678      ph._scenario_tree.pprintCosts(ph._instances)
679
680      solution_writer_plugins = ExtensionPoint(ISolutionWriterExtension)
681      for plugin in solution_writer_plugins:
682         plugin.write(ph._scenario_tree, "postphef")
683
684#
685# The main PH initialization / runner routine. Really only branches based on
686# the construction source - a checkpoint or from scratch.
687#
688
689def exec_ph(options):
690
691   ph = None
692
693   # validate the solution writer plugin exists, to avoid a lot of wasted work.
694   for solution_writer_name in options.solution_writer:
695      print "Trying to import solution writer="+solution_writer_name
696      __import__(solution_writer_name)
697      print "Module successfully loaded"
698
699   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
700   if len(options.restore_from_checkpoint) > 0:
701      ph = create_ph_from_checkpoint(options)
702
703   else:
704      reference_model, reference_instance, scenario_tree, scenario_tree_instance = load_reference_and_scenario_models(options)
705      if reference_model is None or reference_instance is None or scenario_tree is None:
706         return
707      ph = create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree)
708
709   if ph is None:
710      print "***FAILED TO CREATE PH OBJECT"
711      return
712
713   run_ph(options, ph)
714
715#
716# the main driver routine for the runph script.
717#
718
719def run(args=None):
720
721    #
722    # Top-level command that executes the extensive form writer.
723    # This is segregated from run_ef_writer to enable profiling.
724    #
725
726    #
727    # Parse command-line options.
728    #
729    try:
730       ph_options_parser = construct_ph_options_parser("runph [options]")
731       (options, args) = ph_options_parser.parse_args(args=args)
732    except SystemExit:
733       # the parser throws a system exit if "-h" is specified - catch
734       # it to exit gracefully.
735       return
736    #
737    # Control the garbage collector - more critical than I would like at the moment.
738    #
739
740    if options.disable_gc is True:
741       gc.disable()
742    else:
743       gc.enable()
744
745    #
746    # Run PH - precise invocation depends on whether we want profiling output.
747    #
748
749    # if an exception is triggered and traceback is enabled, 'ans' won't
750    # have a value and the return statement from this function will flag
751    # an error, masking the stack trace that you really want to see.
752    ans = None       
753
754    if options.profile > 0:
755        #
756        # Call the main PH routine with profiling.
757        #
758        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
759        tmp = profile.runctx('exec_ph(options)',globals(),locals(),tfile)
760        p = pstats.Stats(tfile).strip_dirs()
761        p.sort_stats('time', 'cum')
762        p = p.print_stats(options.profile)
763        p.print_callers(options.profile)
764        p.print_callees(options.profile)
765        p = p.sort_stats('cum','calls')
766        p.print_stats(options.profile)
767        p.print_callers(options.profile)
768        p.print_callees(options.profile)
769        p = p.sort_stats('calls')
770        p.print_stats(options.profile)
771        p.print_callers(options.profile)
772        p.print_callees(options.profile)
773        pyutilib.services.TempfileManager.clear_tempfiles()
774        ans = [tmp, None]
775    else:
776        #
777        # Call the main PH routine without profiling.
778        #
779
780        if options.traceback is True:
781           ans = exec_ph(options)
782        else:
783           try:
784              ans = exec_ph(options)
785           except ValueError, str:
786              print "VALUE ERROR:"
787              print str
788           except TypeError, str:
789              print "TYPE ERROR:"
790              print str
791           except NameError, str:
792              print "NAME ERROR:"
793              print str                           
794           except IOError, str:
795              print "IO ERROR:"
796              print str
797           except pyutilib.common.ApplicationError, str:
798              print "APPLICATION ERROR:"
799              print str
800           except RuntimeError, str:
801              print "RUN-TIME ERROR:"   
802              print str       
803           except:
804              print "Encountered unhandled exception"
805              traceback.print_exc()
806
807    gc.enable()
808
809    return ans
Note: See TracBrowser for help on using the repository browser.