source: trunk/coopr/pysp/ph_script.py @ 1757

Last change on this file since 1757 was 1757, checked in by jwatson, 11 years ago

Added preliminary checkpointing of PH state via Python's pickle'ing mechanism. Simultaneously slick and scary. This commit only deals with the writing of the state - restoration is in a subsequent commit.

  • Property svn:executable set to *
File size: 18.5 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib
17import textwrap
18import traceback
19
20# garbage collection control.
21import gc
22
23# for profiling
24import cProfile
25import pstats
26
27from coopr.pysp.convergence import *
28from coopr.pysp.scenariotree import *
29from coopr.pysp.ph import *
30from coopr.pysp.ef import *
31from coopr.opt.base import SolverFactory
32from coopr.opt.parallel import SolverManagerFactory
33
34#
35# Setup command-line options
36#
37
38parser = OptionParser()
39parser.add_option("--verbose",
40                  help="Generate verbose output for both initialization and execution.",
41                  action="store_true",
42                  dest="verbose",
43                  default=False)
44parser.add_option("--model-directory",
45                  help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
46                  action="store",
47                  dest="model_directory",
48                  type="string",
49                  default=".")
50parser.add_option("--instance-directory",
51                  help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
52                  action="store",
53                  dest="instance_directory",
54                  type="string",
55                  default=".")
56parser.add_option("--solver",
57                  help="The type of solver used to solve scenario sub-problems. Default is cplex.",
58                  action="store",
59                  dest="solver_type",
60                  type="string",
61                  default="cplex")
62parser.add_option("--solver-manager",
63                  help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
64                  action="store",
65                  dest="solver_manager_type",
66                  type="string",
67                  default="serial")
68parser.add_option("--max-iterations",
69                  help="The maximal number of PH iterations. Default is 100.",
70                  action="store",
71                  dest="max_iterations",
72                  type="int",
73                  default=100)
74parser.add_option("--default-rho",
75                  help="The default (global) rho for all blended variables. Default is 1.",
76                  action="store",
77                  dest="default_rho",
78                  type="float",
79                  default=1.0)
80parser.add_option("--rho-cfgfile",
81                  help="The name of a configuration script to compute PH rho values. Default is None.",
82                  action="store",
83                  dest="rho_cfgfile",
84                  type="string",
85                  default=None)
86parser.add_option("--enable-termdiff-convergence",
87                  help="Terminate PH based on the termdiff convergence metric. Default is True.",
88                  action="store_true",
89                  dest="enable_termdiff_convergence",
90                  default=True)
91parser.add_option("--enable-normalized-termdiff-convergence",
92                  help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
93                  action="store_true",
94                  dest="enable_normalized_termdiff_convergence",
95                  default=False)
96parser.add_option("--termdiff-threshold",
97                  help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
98                  action="store",
99                  dest="termdiff_threshold",
100                  type="float",
101                  default=0.01)
102parser.add_option("--enable-free-discrete-count-convergence",
103                  help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
104                  action="store_true",
105                  dest="enable_free_discrete_count_convergence",
106                  default=False)
107parser.add_option("--free-discrete-count-threshold",
108                  help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
109                  action="store",
110                  dest="free_discrete_count_threshold",
111                  type="float",
112                  default=20)
113parser.add_option("--enable-ww-extensions",
114                  help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
115                  action="store_true",
116                  dest="enable_ww_extensions",
117                  default=False)
118parser.add_option("--ww-extension-cfgfile",
119                  help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
120                  action="store",
121                  dest="ww_extension_cfgfile",
122                  type="string",
123                  default="wwph.cfg")
124parser.add_option("--ww-extension-suffixfile",
125                  help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
126                  action="store",
127                  dest="ww_extension_suffixfile",
128                  type="string",
129                  default="wwph.suffixes")
130parser.add_option("--write-ef",
131                  help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
132                  action="store_true",
133                  dest="write_ef",
134                  default=False)
135parser.add_option("--solve-ef",
136                  help="Following write of the extensive form model, solve it.",
137                  action="store_true",
138                  dest="solve_ef",
139                  default=False)
140parser.add_option("--ef-output-file",
141                  help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
142                  action="store",
143                  dest="ef_output_file",
144                  type="string",
145                  default="efout.lp")
146parser.add_option("--suppress-continuous-variable-output",
147                  help="Eliminate PH-related output involving continuous variables.",
148                  action="store_true",
149                  dest="suppress_continuous_variable_output",
150                  default=False)
151parser.add_option("--keep-solver-files",
152                  help="Retain temporary input and output files for scenario sub-problem solves",
153                  action="store_true",
154                  dest="keep_solver_files",
155                  default=False)
156parser.add_option("--output-solver-logs",
157                  help="Output solver logs during scenario sub-problem solves",
158                  action="store_true",
159                  dest="output_solver_logs",
160                  default=False)
161parser.add_option("--output-ef-solver-log",
162                  help="Output solver log during the extensive form solve",
163                  action="store_true",
164                  dest="output_ef_solver_log",
165                  default=False)
166parser.add_option("--output-solver-results",
167                  help="Output solutions obtained after each scenario sub-problem solve",
168                  action="store_true",
169                  dest="output_solver_results",
170                  default=False)
171parser.add_option("--output-times",
172                  help="Output timing statistics for various PH components",
173                  action="store_true",
174                  dest="output_times",
175                  default=False)
176parser.add_option("--disable-warmstarts",
177                  help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1",
178                  action="store_true",
179                  dest="disable_warmstarts",
180                  default=False)
181parser.add_option("--retain-quadratic-binary-terms",
182                  help="Do not linearize PH objective terms involving binary decision variables",
183                  action="store_true",
184                  dest="retain_quadratic_binary_terms",
185                  default=False)
186parser.add_option("--checkpoint-interval",
187                  help="The number of iterations between writing of a checkpoint file. Default is 0, indicating never.",
188                  action="store",
189                  dest="checkpoint_interval",
190                  type="int",
191                  default=0)
192parser.add_option("--profile",
193                  help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
194                  action="store",
195                  dest="profile",
196                  type="int",
197                  default=0)
198parser.add_option("--enable-gc",
199                  help="Enable the python garbage collecter.",
200                  action="store_true",
201                  dest="enable_gc",
202                  default=False)
203
204
205parser.usage="runph [options]"
206
207#
208# The main PH initialization / runner routine.
209#
210
211def run_ph(options, args):
212
213   start_time = time.time()
214
215   #
216   # create and populate the reference model/instance pair.
217   #
218
219   reference_model = None
220   reference_instance = None
221
222   try:
223      reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
224      if options.verbose is True:
225         print "Scenario reference model filename="+reference_model_filename
226      model_import = pyutilib.import_file(reference_model_filename)
227      if "model" not in dir(model_import):
228         print ""
229         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
230         return
231
232      if model_import.model is None:
233         print ""
234         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
235         return
236 
237      reference_model = model_import.model
238   except IOError:
239      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
240      return
241
242   try:
243      reference_instance_filename = options.instance_directory+os.sep+"ReferenceModel.dat"
244      if options.verbose is True:
245         print "Scenario reference instance filename="+reference_instance_filename
246      reference_instance = reference_model.create(reference_instance_filename)
247   except IOError:
248      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
249      return     
250
251   #
252   # create and populate the scenario tree model
253   #
254
255   scenario_tree_model = None
256   scenario_tree_instance = None
257
258   try:
259      scenario_tree_model_filename = options.model_directory+os.sep+"ScenarioStructure.py"
260      if options.verbose is True:
261         print "Scenario tree model filename="+scenario_tree_model_filename
262      scenario_tree_import = pyutilib.import_file(scenario_tree_model_filename)
263      scenario_tree_model = scenario_tree_import.model
264   except IOError:
265      print "***ERROR: Failed to load scenario tree reference model from file="+scenario_tree_model_filename
266      return   
267
268   try:
269      scenario_tree_instance_filename = options.instance_directory+os.sep+"ScenarioStructure.dat"
270      if options.verbose is True:
271         print "Scenario tree instance filename="+scenario_tree_instance_filename
272      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
273   except IOError:
274      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
275      return
276   
277   #
278   # construct the scenario tree
279   #
280   scenario_tree = ScenarioTree(model=reference_instance,
281                                nodes=scenario_tree_instance.Nodes,
282                                nodechildren=scenario_tree_instance.Children,
283                                nodestages=scenario_tree_instance.NodeStage,
284                                nodeprobabilities=scenario_tree_instance.ConditionalProbability,
285                                stages=scenario_tree_instance.Stages,
286                                stagevariables=scenario_tree_instance.StageVariables,
287                                stagecostvariables=scenario_tree_instance.StageCostVariable,
288                                scenarios=scenario_tree_instance.Scenarios,
289                                scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
290                                scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
291
292   #
293   # print the input tree for validation/information purposes.
294   #
295   if options.verbose is True:
296      scenario_tree.pprint()
297
298   #
299   # validate the tree prior to doing anything serious
300   #
301   print ""
302   if scenario_tree.validate() is False:
303      print "***ERROR: Scenario tree is invalid****"
304      return
305   else:
306      if options.verbose is True:
307         print "Scenario tree is valid!"
308   print ""
309
310   #
311   # deal with any plugins (we currently only deal with one!)
312   #
313   if options.enable_ww_extensions is True:
314
315      from coopr.pysp import wwphextension
316
317      plugin = ExtensionPoint(IPHExtension)
318      plugin.service()._configuration_filename = options.ww_extension_cfgfile
319      plugin.service()._suffix_filename = options.ww_extension_suffixfile
320
321   #
322   # construct the convergence "computer" class.
323   #
324   converger = None
325   # go with the non-defaults first, and then with the default.
326   if options.enable_free_discrete_count_convergence is True:
327      converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
328   elif options.enable_normalized_termdiff_convergence is True:
329      converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
330   else:
331      converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
332
333     
334   #
335   # construct and initialize PH
336   #
337   ph = ProgressiveHedging(max_iterations=options.max_iterations, \
338                           rho=options.default_rho, \
339                           rho_setter=options.rho_cfgfile, \
340                           solver=options.solver_type, \
341                           solver_manager=options.solver_manager_type, \
342                           keep_solver_files=options.keep_solver_files, \
343                           output_solver_log=options.output_solver_logs, \
344                           output_solver_results=options.output_solver_results, \
345                           verbose=options.verbose, \
346                           output_times=options.output_times, \
347                           disable_warmstarts=options.disable_warmstarts,
348                           retain_quadratic_binary_terms=options.retain_quadratic_binary_terms,
349                           checkpoint_interval=options.checkpoint_interval)
350   
351   ph.initialize(scenario_data_directory_name=options.instance_directory, \
352                 model=reference_model, \
353                 model_instance=reference_instance, \
354                 scenario_tree=scenario_tree, \
355                 converger=converger)
356
357   if options.suppress_continuous_variable_output is True:
358      ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
359
360   #
361   # kick off the solve
362   #
363   ph.solve()
364
365   print ""
366   print "DONE..."
367
368   end_time = time.time()
369
370   print ""
371   print "Total execution time=%8.2f seconds" %(end_time - start_time)
372   print ""
373   if options.output_times is True:
374      ph.print_time_stats()
375
376   #
377   # write the extensive form, accounting for any fixed variables.
378   #
379   if (options.write_ef is True) or (options.solve_ef is True):
380      print ""
381      print "Writing EF for remainder problem"
382      print ""
383      write_ef(ph._scenario_tree, ph._instances, options.ef_output_file)
384
385   #
386   # solve the extensive form.
387   #
388   if options.solve_ef is True:
389      print ""
390      print "Solving extensive form written to file="+options.ef_output_file
391      print ""
392
393      ef_solver = SolverFactory(options.solver_type)
394      if ef_solver is None:
395         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
396
397      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
398      if ef_solver is None:
399         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
400
401      print "Queuing extensive form solve"
402      ef_action_handle = ef_solver_manager.queue(options.ef_output_file, opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
403      print "Waiting for extensive form solve"
404      ef_results = ef_solver_manager.wait_for(ef_action_handle)
405      print "Extensive form solve results:"
406      print ef_results
407     
408
409def run(args=None):
410
411    #
412    # Top-level command that executes the extensive form writer.
413    # This is segregated from run_ef_writer to enable profiling.
414    #
415
416    #
417    # Parse command-line options.
418    #
419    try:
420       (options, args) = parser.parse_args(args=args)
421    except SystemExit:
422       # the parser throws a system exit if "-h" is specified - catch
423       # it to exit gracefully.
424       return
425
426    # for a one-pass execution, garbage collection doesn't make
427    # much sense - so it is disabled by default. Because: It drops
428    # the run-time by a factor of 3-4 on bigger instances.
429    if options.enable_gc is False:
430       gc.disable()       
431
432    if options.profile > 0:
433        #
434        # Call the main PH routine with profiling.
435        #
436        tfile = pyutilib.TempfileManager.create_tempfile(suffix=".profile")
437        tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
438        p = pstats.Stats(tfile).strip_dirs()
439        p.sort_stats('time', 'cum')
440        options.profile = eval(options.profile)
441        p = p.print_stats(options.profile)
442        p.print_callers(options.profile)
443        p.print_callees(options.profile)
444        p = p.sort_stats('cum','calls')
445        p.print_stats(options.profile)
446        p.print_callers(options.profile)
447        p.print_callees(options.profile)
448        p = p.sort_stats('calls')
449        p.print_stats(options.profile)
450        p.print_callers(options.profile)
451        p.print_callees(options.profile)
452        pyutilib.TempfileManager.clear_tempfiles()
453        ans = [tmp, None]
454    else:
455        #
456        # Call the main PH routine without profiling.
457        #
458        ans = run_ph(options, args)
459
460    gc.enable()
461   
462    return ans
463
Note: See TracBrowser for help on using the repository browser.