source: trunk/coopr/pysp/ph_script.py @ 1768

Last change on this file since 1768 was 1768, checked in by wehart, 11 years ago

Rework of Coopr to use the new PyUtilib? package decomposition.

NOTE: to use Coopr with this update, we need to work with a new version of coopr_install.

  • Property svn:executable set to *
File size: 21.0 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20
21# garbage collection control.
22import gc
23
24# for profiling
25import cProfile
26import pstats
27
28# for serializing
29import pickle
30
31from coopr.pysp.convergence import *
32from coopr.pysp.scenariotree import *
33from coopr.pysp.ph import *
34from coopr.pysp.ef import *
35from coopr.opt.base import SolverFactory
36from coopr.opt.parallel import SolverManagerFactory
37
38#
39# Setup command-line options
40#
41
42parser = OptionParser()
43parser.add_option("--verbose",
44                  help="Generate verbose output for both initialization and execution.",
45                  action="store_true",
46                  dest="verbose",
47                  default=False)
48parser.add_option("--model-directory",
49                  help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
50                  action="store",
51                  dest="model_directory",
52                  type="string",
53                  default=".")
54parser.add_option("--instance-directory",
55                  help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
56                  action="store",
57                  dest="instance_directory",
58                  type="string",
59                  default=".")
60parser.add_option("--solver",
61                  help="The type of solver used to solve scenario sub-problems. Default is cplex.",
62                  action="store",
63                  dest="solver_type",
64                  type="string",
65                  default="cplex")
66parser.add_option("--solver-manager",
67                  help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
68                  action="store",
69                  dest="solver_manager_type",
70                  type="string",
71                  default="serial")
72parser.add_option("--max-iterations",
73                  help="The maximal number of PH iterations. Default is 100.",
74                  action="store",
75                  dest="max_iterations",
76                  type="int",
77                  default=100)
78parser.add_option("--default-rho",
79                  help="The default (global) rho for all blended variables. Default is 1.",
80                  action="store",
81                  dest="default_rho",
82                  type="float",
83                  default=1.0)
84parser.add_option("--rho-cfgfile",
85                  help="The name of a configuration script to compute PH rho values. Default is None.",
86                  action="store",
87                  dest="rho_cfgfile",
88                  type="string",
89                  default=None)
90parser.add_option("--enable-termdiff-convergence",
91                  help="Terminate PH based on the termdiff convergence metric. Default is True.",
92                  action="store_true",
93                  dest="enable_termdiff_convergence",
94                  default=True)
95parser.add_option("--enable-normalized-termdiff-convergence",
96                  help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
97                  action="store_true",
98                  dest="enable_normalized_termdiff_convergence",
99                  default=False)
100parser.add_option("--termdiff-threshold",
101                  help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
102                  action="store",
103                  dest="termdiff_threshold",
104                  type="float",
105                  default=0.01)
106parser.add_option("--enable-free-discrete-count-convergence",
107                  help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
108                  action="store_true",
109                  dest="enable_free_discrete_count_convergence",
110                  default=False)
111parser.add_option("--free-discrete-count-threshold",
112                  help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
113                  action="store",
114                  dest="free_discrete_count_threshold",
115                  type="float",
116                  default=20)
117parser.add_option("--enable-ww-extensions",
118                  help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
119                  action="store_true",
120                  dest="enable_ww_extensions",
121                  default=False)
122parser.add_option("--ww-extension-cfgfile",
123                  help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
124                  action="store",
125                  dest="ww_extension_cfgfile",
126                  type="string",
127                  default="wwph.cfg")
128parser.add_option("--ww-extension-suffixfile",
129                  help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
130                  action="store",
131                  dest="ww_extension_suffixfile",
132                  type="string",
133                  default="wwph.suffixes")
134parser.add_option("--write-ef",
135                  help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
136                  action="store_true",
137                  dest="write_ef",
138                  default=False)
139parser.add_option("--solve-ef",
140                  help="Following write of the extensive form model, solve it.",
141                  action="store_true",
142                  dest="solve_ef",
143                  default=False)
144parser.add_option("--ef-output-file",
145                  help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
146                  action="store",
147                  dest="ef_output_file",
148                  type="string",
149                  default="efout.lp")
150parser.add_option("--suppress-continuous-variable-output",
151                  help="Eliminate PH-related output involving continuous variables.",
152                  action="store_true",
153                  dest="suppress_continuous_variable_output",
154                  default=False)
155parser.add_option("--keep-solver-files",
156                  help="Retain temporary input and output files for scenario sub-problem solves",
157                  action="store_true",
158                  dest="keep_solver_files",
159                  default=False)
160parser.add_option("--output-solver-logs",
161                  help="Output solver logs during scenario sub-problem solves",
162                  action="store_true",
163                  dest="output_solver_logs",
164                  default=False)
165parser.add_option("--output-ef-solver-log",
166                  help="Output solver log during the extensive form solve",
167                  action="store_true",
168                  dest="output_ef_solver_log",
169                  default=False)
170parser.add_option("--output-solver-results",
171                  help="Output solutions obtained after each scenario sub-problem solve",
172                  action="store_true",
173                  dest="output_solver_results",
174                  default=False)
175parser.add_option("--output-times",
176                  help="Output timing statistics for various PH components",
177                  action="store_true",
178                  dest="output_times",
179                  default=False)
180parser.add_option("--disable-warmstarts",
181                  help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1",
182                  action="store_true",
183                  dest="disable_warmstarts",
184                  default=False)
185parser.add_option("--retain-quadratic-binary-terms",
186                  help="Do not linearize PH objective terms involving binary decision variables",
187                  action="store_true",
188                  dest="retain_quadratic_binary_terms",
189                  default=False)
190parser.add_option("--checkpoint-interval",
191                  help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
192                  action="store",
193                  dest="checkpoint_interval",
194                  type="int",
195                  default=0)
196parser.add_option("--restore-from-checkpoint",
197                  help="The name of the checkpoint file from which PH should be initialized. Default is "", indicating no checkpoint restoration",
198                  action="store",
199                  dest="restore_from_checkpoint",
200                  type="string",
201                  default="")
202parser.add_option("--profile",
203                  help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
204                  action="store",
205                  dest="profile",
206                  type="int",
207                  default=0)
208parser.add_option("--enable-gc",
209                  help="Enable the python garbage collecter.",
210                  action="store_true",
211                  dest="enable_gc",
212                  default=False)
213
214
215parser.usage="runph [options]"
216
217#
218# The main PH initialization / runner routine.
219#
220
221def run_ph(options, args):
222
223   start_time = time.time()
224
225   ph = None
226
227   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
228   if len(options.restore_from_checkpoint) > 0:
229
230      # we need to load the reference model, as pickle doesn't save contents of .py files!
231      try:
232         reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
233         if options.verbose is True:
234            print "Scenario reference model filename="+reference_model_filename
235         model_import = pyutilib.misc.import_file(reference_model_filename)
236         if "model" not in dir(model_import):
237            print ""
238            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
239            return
240
241         if model_import.model is None:
242            print ""
243            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
244            return
245 
246         reference_model = model_import.model
247      except IOError:
248         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
249         return     
250
251      # import the saved state
252     
253      try:
254         checkpoint_file = open(options.restore_from_checkpoint,"r")
255         ph = pickle.load(checkpoint_file)
256         checkpoint_file.close()
257         
258#         print "PH=",ph
259#         ph._scenario_tree.pprint()         
260      except IOError, msg:
261         raise RuntimeError, msg
262
263      # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
264
265      raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
266     
267   else:
268      #
269      # create and populate the reference model/instance pair.
270      #
271
272      reference_model = None
273      reference_instance = None
274
275      try:
276         reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
277         if options.verbose is True:
278            print "Scenario reference model filename="+reference_model_filename
279         model_import = pyutilib.misc.import_file(reference_model_filename)
280         if "model" not in dir(model_import):
281            print ""
282            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
283            return
284
285         if model_import.model is None:
286            print ""
287            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
288            return
289 
290         reference_model = model_import.model
291      except IOError:
292         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
293         return
294
295      try:
296         reference_instance_filename = options.instance_directory+os.sep+"ReferenceModel.dat"
297         if options.verbose is True:
298            print "Scenario reference instance filename="+reference_instance_filename
299         reference_instance = reference_model.create(reference_instance_filename)
300      except IOError:
301         print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
302         return     
303
304      #
305      # create and populate the scenario tree model
306      #
307
308      scenario_tree_model = None
309      scenario_tree_instance = None
310
311      try:
312         scenario_tree_model_filename = options.model_directory+os.sep+"ScenarioStructure.py"
313         if options.verbose is True:
314            print "Scenario tree model filename="+scenario_tree_model_filename
315         scenario_tree_import = pyutilib.misc.import_file(scenario_tree_model_filename)
316         scenario_tree_model = scenario_tree_import.model
317      except IOError:
318         print "***ERROR: Failed to load scenario tree reference model from file="+scenario_tree_model_filename
319         return   
320
321      try:
322         scenario_tree_instance_filename = options.instance_directory+os.sep+"ScenarioStructure.dat"
323         if options.verbose is True:
324            print "Scenario tree instance filename="+scenario_tree_instance_filename
325         scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
326      except IOError:
327         print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
328         return
329   
330      #
331      # construct the scenario tree
332      #
333      scenario_tree = ScenarioTree(model=reference_instance,
334                                   nodes=scenario_tree_instance.Nodes,
335                                   nodechildren=scenario_tree_instance.Children,
336                                   nodestages=scenario_tree_instance.NodeStage,
337                                   nodeprobabilities=scenario_tree_instance.ConditionalProbability,
338                                   stages=scenario_tree_instance.Stages,
339                                   stagevariables=scenario_tree_instance.StageVariables,
340                                   stagecostvariables=scenario_tree_instance.StageCostVariable,
341                                   scenarios=scenario_tree_instance.Scenarios,
342                                   scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
343                                   scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
344
345      #
346      # print the input tree for validation/information purposes.
347      #
348      if options.verbose is True:
349         scenario_tree.pprint()
350
351      #
352      # validate the tree prior to doing anything serious
353      #
354      print ""
355      if scenario_tree.validate() is False:
356         print "***ERROR: Scenario tree is invalid****"
357         return
358      else:
359         if options.verbose is True:
360            print "Scenario tree is valid!"
361      print ""
362
363      #
364      # deal with any plugins (we currently only deal with one!)
365      #
366      if options.enable_ww_extensions is True:
367
368         from coopr.pysp import wwphextension
369
370         plugin = ExtensionPoint(IPHExtension)
371         plugin.service()._configuration_filename = options.ww_extension_cfgfile
372         plugin.service()._suffix_filename = options.ww_extension_suffixfile
373
374      #
375      # construct the convergence "computer" class.
376      #
377      converger = None
378      # go with the non-defaults first, and then with the default.
379      if options.enable_free_discrete_count_convergence is True:
380         converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
381      elif options.enable_normalized_termdiff_convergence is True:
382         converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
383      else:
384         converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
385
386     
387      #
388      # construct and initialize PH
389      #
390      ph = ProgressiveHedging(max_iterations=options.max_iterations, \
391                              rho=options.default_rho, \
392                              rho_setter=options.rho_cfgfile, \
393                              solver=options.solver_type, \
394                              solver_manager=options.solver_manager_type, \
395                              keep_solver_files=options.keep_solver_files, \
396                              output_solver_log=options.output_solver_logs, \
397                              output_solver_results=options.output_solver_results, \
398                              verbose=options.verbose, \
399                              output_times=options.output_times, \
400                              disable_warmstarts=options.disable_warmstarts,
401                              retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
402                              checkpoint_interval=options.checkpoint_interval)
403   
404      ph.initialize(scenario_data_directory_name=options.instance_directory, \
405                    model=reference_model, \
406                    model_instance=reference_instance, \
407                    scenario_tree=scenario_tree, \
408                    converger=converger)
409
410      if options.suppress_continuous_variable_output is True:
411         ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
412
413   #
414   # at this point, we have a PH object by some means.
415   #
416
417   #
418   # kick off the solve
419   #
420   ph.solve()
421
422   print ""
423   print "DONE..."
424
425   end_time = time.time()
426
427   print ""
428   print "Total execution time=%8.2f seconds" %(end_time - start_time)
429   print ""
430   if options.output_times is True:
431      ph.print_time_stats()
432
433   #
434   # write the extensive form, accounting for any fixed variables.
435   #
436   if (options.write_ef is True) or (options.solve_ef is True):
437      print ""
438      print "Writing EF for remainder problem"
439      print ""
440      write_ef(ph._scenario_tree, ph._instances, options.ef_output_file)
441
442   #
443   # solve the extensive form.
444   #
445   if options.solve_ef is True:
446      print ""
447      print "Solving extensive form written to file="+options.ef_output_file
448      print ""
449
450      ef_solver = SolverFactory(options.solver_type)
451      if ef_solver is None:
452         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
453
454      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
455      if ef_solver is None:
456         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
457
458      print "Queuing extensive form solve"
459      ef_action_handle = ef_solver_manager.queue(options.ef_output_file, opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
460      print "Waiting for extensive form solve"
461      ef_results = ef_solver_manager.wait_for(ef_action_handle)
462      print "Extensive form solve results:"
463      print ef_results
464     
465
466def run(args=None):
467
468    #
469    # Top-level command that executes the extensive form writer.
470    # This is segregated from run_ef_writer to enable profiling.
471    #
472
473    #
474    # Parse command-line options.
475    #
476    try:
477       (options, args) = parser.parse_args(args=args)
478    except SystemExit:
479       # the parser throws a system exit if "-h" is specified - catch
480       # it to exit gracefully.
481       return
482
483    # for a one-pass execution, garbage collection doesn't make
484    # much sense - so it is disabled by default. Because: It drops
485    # the run-time by a factor of 3-4 on bigger instances.
486    if options.enable_gc is False:
487       gc.disable()       
488
489    if options.profile > 0:
490        #
491        # Call the main PH routine with profiling.
492        #
493        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
494        tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
495        p = pstats.Stats(tfile).strip_dirs()
496        p.sort_stats('time', 'cum')
497        options.profile = eval(options.profile)
498        p = p.print_stats(options.profile)
499        p.print_callers(options.profile)
500        p.print_callees(options.profile)
501        p = p.sort_stats('cum','calls')
502        p.print_stats(options.profile)
503        p.print_callers(options.profile)
504        p.print_callees(options.profile)
505        p = p.sort_stats('calls')
506        p.print_stats(options.profile)
507        p.print_callers(options.profile)
508        p.print_callees(options.profile)
509        pyutilib.services.TempfileManager.clear_tempfiles()
510        ans = [tmp, None]
511    else:
512        #
513        # Call the main PH routine without profiling.
514        #
515        ans = run_ph(options, args)
516
517    gc.enable()
518   
519    return ans
520
Note: See TracBrowser for help on using the repository browser.