source: coopr.pysp/trunk/coopr/pysp/ph_script.py @ 2059

Last change on this file since 2059 was 2059, checked in by jwatson, 10 years ago

Enabling garbage collection by default in PH.

  • Property svn:executable set to *
File size: 25.7 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20
21# garbage collection control.
22import gc
23
24# for profiling
25import cProfile
26import pstats
27
28# for serializing
29import pickle
30
31from coopr.pysp.convergence import *
32from coopr.pysp.scenariotree import *
33from coopr.pysp.ph import *
34from coopr.pysp.ef import *
35from coopr.opt.base import SolverFactory
36from coopr.opt.parallel import SolverManagerFactory
37
38#
39# Setup command-line options
40#
41
42parser = OptionParser()
43parser.add_option("--verbose",
44                  help="Generate verbose output for both initialization and execution.",
45                  action="store_true",
46                  dest="verbose",
47                  default=False)
48parser.add_option("--model-directory",
49                  help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
50                  action="store",
51                  dest="model_directory",
52                  type="string",
53                  default=".")
54parser.add_option("--instance-directory",
55                  help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
56                  action="store",
57                  dest="instance_directory",
58                  type="string",
59                  default=".")
60parser.add_option("--solver",
61                  help="The type of solver used to solve scenario sub-problems. Default is cplex.",
62                  action="store",
63                  dest="solver_type",
64                  type="string",
65                  default="cplex")
66parser.add_option("--solver-manager",
67                  help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
68                  action="store",
69                  dest="solver_manager_type",
70                  type="string",
71                  default="serial")
72parser.add_option("--scenario-solver-options",
73                  help="Solver options for all PH scenario sub-problems",
74                  action="append",
75                  dest="scenario_solver_options",
76                  type="string",
77                  default=[])
78parser.add_option("--scenario-mipgap",
79                  help="Specifies the mipgap for all PH scenario sub-problems",
80                  action="store",
81                  dest="scenario_mipgap",
82                  type="float",
83                  default=None)
84parser.add_option("--ef-solver-options",
85                  help="Solver options for the extension form problem",
86                  action="append",
87                  dest="ef_solver_options",
88                  type="string",
89                  default=[])
90parser.add_option("--max-iterations",
91                  help="The maximal number of PH iterations. Default is 100.",
92                  action="store",
93                  dest="max_iterations",
94                  type="int",
95                  default=100)
96parser.add_option("--default-rho",
97                  help="The default (global) rho for all blended variables. Default is 1.",
98                  action="store",
99                  dest="default_rho",
100                  type="float",
101                  default=1.0)
102parser.add_option("--rho-cfgfile",
103                  help="The name of a configuration script to compute PH rho values. Default is None.",
104                  action="store",
105                  dest="rho_cfgfile",
106                  type="string",
107                  default=None)
108parser.add_option("--bounds-cfgfile",
109                  help="The name of a configuration script to set variable bound values. Default is None.",
110                  action="store",
111                  dest="bounds_cfgfile",
112                  default=None)
113parser.add_option("--enable-termdiff-convergence",
114                  help="Terminate PH based on the termdiff convergence metric. Default is True.",
115                  action="store_true",
116                  dest="enable_termdiff_convergence",
117                  default=True)
118parser.add_option("--enable-normalized-termdiff-convergence",
119                  help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
120                  action="store_true",
121                  dest="enable_normalized_termdiff_convergence",
122                  default=False)
123parser.add_option("--termdiff-threshold",
124                  help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
125                  action="store",
126                  dest="termdiff_threshold",
127                  type="float",
128                  default=0.01)
129parser.add_option("--enable-free-discrete-count-convergence",
130                  help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
131                  action="store_true",
132                  dest="enable_free_discrete_count_convergence",
133                  default=False)
134parser.add_option("--free-discrete-count-threshold",
135                  help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
136                  action="store",
137                  dest="free_discrete_count_threshold",
138                  type="float",
139                  default=20)
140parser.add_option("--enable-ww-extensions",
141                  help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
142                  action="store_true",
143                  dest="enable_ww_extensions",
144                  default=False)
145parser.add_option("--ww-extension-cfgfile",
146                  help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
147                  action="store",
148                  dest="ww_extension_cfgfile",
149                  type="string",
150                  default="")
151parser.add_option("--ww-extension-suffixfile",
152                  help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
153                  action="store",
154                  dest="ww_extension_suffixfile",
155                  type="string",
156                  default="")
157parser.add_option("--user-defined-extension",
158                  help="The name of a python module specifying a user-defined PH extension plugin.",
159                  action="store",
160                  dest="user_defined_extension",
161                  type="string",
162                  default=None)
163parser.add_option("--write-ef",
164                  help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
165                  action="store_true",
166                  dest="write_ef",
167                  default=False)
168parser.add_option("--solve-ef",
169                  help="Following write of the extensive form model, solve it.",
170                  action="store_true",
171                  dest="solve_ef",
172                  default=False)
173parser.add_option("--ef-output-file",
174                  help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
175                  action="store",
176                  dest="ef_output_file",
177                  type="string",
178                  default="efout.lp")
179parser.add_option("--suppress-continuous-variable-output",
180                  help="Eliminate PH-related output involving continuous variables.",
181                  action="store_true",
182                  dest="suppress_continuous_variable_output",
183                  default=False)
184parser.add_option("--keep-solver-files",
185                  help="Retain temporary input and output files for scenario sub-problem solves",
186                  action="store_true",
187                  dest="keep_solver_files",
188                  default=False)
189parser.add_option("--output-solver-logs",
190                  help="Output solver logs during scenario sub-problem solves",
191                  action="store_true",
192                  dest="output_solver_logs",
193                  default=False)
194parser.add_option("--output-ef-solver-log",
195                  help="Output solver log during the extensive form solve",
196                  action="store_true",
197                  dest="output_ef_solver_log",
198                  default=False)
199parser.add_option("--output-solver-results",
200                  help="Output solutions obtained after each scenario sub-problem solve",
201                  action="store_true",
202                  dest="output_solver_results",
203                  default=False)
204parser.add_option("--output-times",
205                  help="Output timing statistics for various PH components",
206                  action="store_true",
207                  dest="output_times",
208                  default=False)
209parser.add_option("--disable-warmstarts",
210                  help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1",
211                  action="store_true",
212                  dest="disable_warmstarts",
213                  default=False)
214parser.add_option("--drop-proximal-terms",
215                  help="Eliminate proximal terms (i.e., the quadratic penalty terms) from the weighted PH objective",
216                  action="store_true",
217                  dest="drop_proximal_terms",
218                  default=False)
219parser.add_option("--retain-quadratic-binary-terms",
220                  help="Do not linearize PH objective terms involving binary decision variables",
221                  action="store_true",
222                  dest="retain_quadratic_binary_terms",
223                  default=False)
224parser.add_option("--linearize-nonbinary-penalty-terms",
225                  help="Approximate the PH quadratic term for non-binary variables with a piece-wise linear function, using the supplied number of equal-length pieces from each bound to the average",
226                  action="store",
227                  dest="linearize_nonbinary_penalty_terms",
228                  type="int",
229                  default=0)
230parser.add_option("--breakpoint-strategy",
231                  help="Specify the strategy to distribute breakpoints on the [lb, ub] interval of each variable when linearizing. 0 indicates uniform distribution. 1 indicates breakpoints at the node min and max, uniformly in-between. 2 indicates more aggressive concentration of breakpoints near the observed node min/max.",
232                  action="store",
233                  dest="breakpoint_strategy",
234                  type="int",
235                  default=0)
236parser.add_option("--checkpoint-interval",
237                  help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
238                  action="store",
239                  dest="checkpoint_interval",
240                  type="int",
241                  default=0)
242parser.add_option("--restore-from-checkpoint",
243                  help="The name of the checkpoint file from which PH should be initialized. Default is "", indicating no checkpoint restoration",
244                  action="store",
245                  dest="restore_from_checkpoint",
246                  type="string",
247                  default="")
248parser.add_option("--profile",
249                  help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
250                  action="store",
251                  dest="profile",
252                  type="int",
253                  default=0)
254parser.add_option("--enable-gc",
255                  help="Enable the python garbage collecter.",
256                  action="store_true",
257                  dest="enable_gc",
258                  default=True)
259
260
261parser.usage="runph [options]"
262
263#
264# The main PH initialization / runner routine.
265#
266
267def run_ph(options, args):
268
269   start_time = time.time()
270
271   ph = None
272
273   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
274   if len(options.restore_from_checkpoint) > 0:
275
276      # we need to load the reference model, as pickle doesn't save contents of .py files!
277      try:
278         reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
279         if options.verbose is True:
280            print "Scenario reference model filename="+reference_model_filename
281         model_import = pyutilib.misc.import_file(reference_model_filename)
282         if "model" not in dir(model_import):
283            print ""
284            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
285            return
286
287         if model_import.model is None:
288            print ""
289            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
290            return
291 
292         reference_model = model_import.model
293      except IOError:
294         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
295         return     
296
297      # import the saved state
298     
299      try:
300         checkpoint_file = open(options.restore_from_checkpoint,"r")
301         ph = pickle.load(checkpoint_file)
302         checkpoint_file.close()
303         
304#         print "PH=",ph
305#         ph._scenario_tree.pprint()         
306      except IOError, msg:
307         raise RuntimeError, msg
308
309      # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
310
311      raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
312     
313   else:
314      #
315      # create and populate the reference model/instance pair.
316      #
317
318      reference_model = None
319      reference_instance = None
320
321      try:
322         reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
323         if options.verbose is True:
324            print "Scenario reference model filename="+reference_model_filename
325         model_import = pyutilib.misc.import_file(reference_model_filename)
326         if "model" not in dir(model_import):
327            print ""
328            print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
329            return
330
331         if model_import.model is None:
332            print ""
333            print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
334            return
335 
336         reference_model = model_import.model
337      except IOError:
338         print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
339         return
340
341      try:
342         reference_instance_filename = options.instance_directory+os.sep+"ReferenceModel.dat"
343         if options.verbose is True:
344            print "Scenario reference instance filename="+reference_instance_filename
345         reference_instance = reference_model.create(reference_instance_filename)
346      except IOError:
347         print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
348         return     
349
350      #
351      # create and populate the scenario tree model
352      #
353
354      scenario_tree_model = None
355      scenario_tree_instance = None
356
357      try:
358         scenario_tree_model_filename = options.model_directory+os.sep+"ScenarioStructure.py"
359         if options.verbose is True:
360            print "Scenario tree model filename="+scenario_tree_model_filename
361         scenario_tree_import = pyutilib.misc.import_file(scenario_tree_model_filename)
362         scenario_tree_model = scenario_tree_import.model
363      except IOError:
364         print "***ERROR: Failed to load scenario tree reference model from file="+scenario_tree_model_filename
365         return   
366
367      try:
368         scenario_tree_instance_filename = options.instance_directory+os.sep+"ScenarioStructure.dat"
369         if options.verbose is True:
370            print "Scenario tree instance filename="+scenario_tree_instance_filename
371         scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
372      except IOError:
373         print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
374         return
375   
376      #
377      # construct the scenario tree
378      #
379      scenario_tree = ScenarioTree(model=reference_instance,
380                                   nodes=scenario_tree_instance.Nodes,
381                                   nodechildren=scenario_tree_instance.Children,
382                                   nodestages=scenario_tree_instance.NodeStage,
383                                   nodeprobabilities=scenario_tree_instance.ConditionalProbability,
384                                   stages=scenario_tree_instance.Stages,
385                                   stagevariables=scenario_tree_instance.StageVariables,
386                                   stagecostvariables=scenario_tree_instance.StageCostVariable,
387                                   scenarios=scenario_tree_instance.Scenarios,
388                                   scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
389                                   scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
390
391      #
392      # print the input tree for validation/information purposes.
393      #
394      if options.verbose is True:
395         scenario_tree.pprint()
396
397      #
398      # validate the tree prior to doing anything serious
399      #
400      print ""
401      if scenario_tree.validate() is False:
402         print "***ERROR: Scenario tree is invalid****"
403         return
404      else:
405         if options.verbose is True:
406            print "Scenario tree is valid!"
407      print ""
408
409      #
410      # if any of the ww extension configuration options are specified without the
411      # ww extension itself being enabled, halt and warn the user - this has led
412      # to confusion in the past, and will save user support time.
413      #
414      if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
415         print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
416         return
417
418      if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
419         print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"         
420         return
421
422      #
423      # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
424      #
425      if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
426         print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
427         return         
428
429      #
430      # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
431      # order only matters if both are specified.
432      #
433      if options.enable_ww_extensions is True:
434
435         from coopr.pysp import wwphextension
436
437         plugin = ExtensionPoint(IPHExtension)
438         if len(options.ww_extension_cfgfile) > 0:
439            plugin.service()._configuration_filename = options.ww_extension_cfgfile
440         if len(options.ww_extension_suffixfile) > 0:
441            plugin.service()._suffix_filename = options.ww_extension_suffixfile
442
443      if options.user_defined_extension is not None:
444         print "Trying to import user-defined PH extension module="+options.user_defined_extension
445         # JPW removed the exception handling logic, as the module importer
446         # can raise a broad array of exceptions.
447         __import__(options.user_defined_extension)
448         print "Module successfully loaded"
449
450      #
451      # construct the convergence "computer" class.
452      #
453      converger = None
454      # go with the non-defaults first, and then with the default.
455      if options.enable_free_discrete_count_convergence is True:
456         converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
457      elif options.enable_normalized_termdiff_convergence is True:
458         converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
459      else:
460         converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
461
462     
463      #
464      # construct and initialize PH
465      #
466      ph = ProgressiveHedging(max_iterations=options.max_iterations, \
467                              rho=options.default_rho, \
468                              rho_setter=options.rho_cfgfile, \
469                              bounds_setter=options.bounds_cfgfile, \
470                              solver=options.solver_type, \
471                              solver_manager=options.solver_manager_type, \
472                              scenario_solver_options=options.scenario_solver_options, \
473                              scenario_mipgap=options.scenario_mipgap, \
474                              keep_solver_files=options.keep_solver_files, \
475                              output_solver_log=options.output_solver_logs, \
476                              output_solver_results=options.output_solver_results, \
477                              verbose=options.verbose, \
478                              output_times=options.output_times, \
479                              disable_warmstarts=options.disable_warmstarts,
480                              drop_proximal_terms=options.drop_proximal_terms,
481                              retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
482                              linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
483                              breakpoint_strategy=options.breakpoint_strategy, \
484                              checkpoint_interval=options.checkpoint_interval)
485   
486      ph.initialize(scenario_data_directory_name=options.instance_directory, \
487                    model=reference_model, \
488                    model_instance=reference_instance, \
489                    scenario_tree=scenario_tree, \
490                    converger=converger)
491
492      if options.suppress_continuous_variable_output is True:
493         ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
494
495   #
496   # at this point, we have a PH object by some means.
497   #
498
499   #
500   # kick off the solve
501   #
502   ph.solve()
503
504   print ""
505   print "DONE..."
506
507   end_time = time.time()
508
509   print ""
510   print "Total execution time=%8.2f seconds" %(end_time - start_time)
511   print ""
512   if options.output_times is True:
513      ph.print_time_stats()
514
515   #
516   # write the extensive form, accounting for any fixed variables.
517   #
518   if (options.write_ef is True) or (options.solve_ef is True):
519      print ""
520      print "Writing EF for remainder problem"
521      print ""
522      write_ef(ph._scenario_tree, ph._instances, options.ef_output_file)
523
524   #
525   # solve the extensive form.
526   #
527   if options.solve_ef is True:
528      print ""
529      print "Solving extensive form written to file="+options.ef_output_file
530      print ""
531
532      ef_solver = SolverFactory(options.solver_type)
533      if ef_solver is None:
534         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
535      if len(options.ef_solver_options) > 0:
536         print "Initializing ef solver with options="+str(options.ef_solver_options)         
537         ef_solver.set_options("".join(options.ef_solver_options))
538
539      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
540      if ef_solver is None:
541         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
542
543      print "Queuing extensive form solve"
544      ef_action_handle = ef_solver_manager.queue(options.ef_output_file, opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
545      print "Waiting for extensive form solve"
546      ef_results = ef_solver_manager.wait_for(ef_action_handle)
547      print "Extensive form solve results:"
548      ef_results.write(num=1)
549
550def run(args=None):
551
552    #
553    # Top-level command that executes the extensive form writer.
554    # This is segregated from run_ef_writer to enable profiling.
555    #
556
557    #
558    # Parse command-line options.
559    #
560    try:
561       (options, args) = parser.parse_args(args=args)
562    except SystemExit:
563       # the parser throws a system exit if "-h" is specified - catch
564       # it to exit gracefully.
565       return
566
567    # for a one-pass execution, garbage collection doesn't make
568    # much sense - so it is disabled by default. Because: It drops
569    # the run-time by a factor of 3-4 on bigger instances.
570    print "OPTIONS.ENABLE_GC=",options.enable_gc
571    if options.enable_gc is False:
572       gc.disable()
573    else:
574       gc.enable()
575
576    if options.profile > 0:
577        #
578        # Call the main PH routine with profiling.
579        #
580        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
581        tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
582        p = pstats.Stats(tfile).strip_dirs()
583        p.sort_stats('time', 'cum')
584        p = p.print_stats(options.profile)
585        p.print_callers(options.profile)
586        p.print_callees(options.profile)
587        p = p.sort_stats('cum','calls')
588        p.print_stats(options.profile)
589        p.print_callers(options.profile)
590        p.print_callees(options.profile)
591        p = p.sort_stats('calls')
592        p.print_stats(options.profile)
593        p.print_callers(options.profile)
594        p.print_callees(options.profile)
595        pyutilib.services.TempfileManager.clear_tempfiles()
596        ans = [tmp, None]
597    else:
598        #
599        # Call the main PH routine without profiling.
600        #
601        ans = run_ph(options, args)
602
603    gc.enable()
604   
605    return ans
606
Note: See TracBrowser for help on using the repository browser.