source: trunk/coopr/pysp/ph_script.py @ 1743

Last change on this file since 1743 was 1743, checked in by jwatson, 11 years ago

Initial commit of binary quadratic term linearization in PH - works for minimization, next commit will address maximization (minor twist).

I also added an option for disabling the linearization; default is "on", as there is little reason to believe linearization won't decrease - or at least not increase - run-time.

  • Property svn:executable set to *
File size: 18.1 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib
17import textwrap
18import traceback
19
20# garbage collection control.
21import gc
22
23# for profiling
24import cProfile
25import pstats
26
27from coopr.pysp.convergence import *
28from coopr.pysp.scenariotree import *
29from coopr.pysp.ph import *
30from coopr.pysp.ef import *
31from coopr.opt.base import SolverFactory
32from coopr.opt.parallel import SolverManagerFactory
33
34#
35# Setup command-line options
36#
37
38parser = OptionParser()
39parser.add_option("--verbose",
40                  help="Generate verbose output for both initialization and execution.",
41                  action="store_true",
42                  dest="verbose",
43                  default=False)
44parser.add_option("--model-directory",
45                  help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
46                  action="store",
47                  dest="model_directory",
48                  type="string",
49                  default=".")
50parser.add_option("--instance-directory",
51                  help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
52                  action="store",
53                  dest="instance_directory",
54                  type="string",
55                  default=".")
56parser.add_option("--solver",
57                  help="The type of solver used to solve scenario sub-problems. Default is cplex.",
58                  action="store",
59                  dest="solver_type",
60                  type="string",
61                  default="cplex")
62parser.add_option("--solver-manager",
63                  help="The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.",
64                  action="store",
65                  dest="solver_manager_type",
66                  type="string",
67                  default="serial")
68parser.add_option("--max-iterations",
69                  help="The maximal number of PH iterations. Default is 100.",
70                  action="store",
71                  dest="max_iterations",
72                  type="int",
73                  default=100)
74parser.add_option("--default-rho",
75                  help="The default (global) rho for all blended variables. Default is 1.",
76                  action="store",
77                  dest="default_rho",
78                  type="float",
79                  default=1.0)
80parser.add_option("--rho-cfgfile",
81                  help="The name of a configuration script to compute PH rho values. Default is None.",
82                  action="store",
83                  dest="rho_cfgfile",
84                  type="string",
85                  default=None)
86parser.add_option("--enable-termdiff-convergence",
87                  help="Terminate PH based on the termdiff convergence metric. Default is True.",
88                  action="store_true",
89                  dest="enable_termdiff_convergence",
90                  default=True)
91parser.add_option("--enable-normalized-termdiff-convergence",
92                  help="Terminate PH based on the normalized termdiff convergence metric. Default is True.",
93                  action="store_true",
94                  dest="enable_normalized_termdiff_convergence",
95                  default=False)
96parser.add_option("--termdiff-threshold",
97                  help="The convergence threshold used in the term-diff and normalized term-diff convergence criteria. Default is 0.01.",
98                  action="store",
99                  dest="termdiff_threshold",
100                  type="float",
101                  default=0.01)
102parser.add_option("--enable-free-discrete-count-convergence",
103                  help="Terminate PH based on the free discrete variable count convergence metric. Default is False.",
104                  action="store_true",
105                  dest="enable_free_discrete_count_convergence",
106                  default=False)
107parser.add_option("--free-discrete-count-threshold",
108                  help="The convergence threshold used in the criterion based on when the free discrete variable count convergence criterion. Default is 20.",
109                  action="store",
110                  dest="free_discrete_count_threshold",
111                  type="float",
112                  default=20)
113parser.add_option("--enable-ww-extensions",
114                  help="Enable the Watson-Woodruff PH extensions plugin. Default is False.",
115                  action="store_true",
116                  dest="enable_ww_extensions",
117                  default=False)
118parser.add_option("--ww-extension-cfgfile",
119                  help="The name of a configuration file for the Watson-Woodruff PH extensions plugin. Default is wwph.cfg.",
120                  action="store",
121                  dest="ww_extension_cfgfile",
122                  type="string",
123                  default="wwph.cfg")
124parser.add_option("--ww-extension-suffixfile",
125                  help="The name of a variable suffix file for the Watson-Woodruff PH extensions plugin. Default is wwph.suffixes.",
126                  action="store",
127                  dest="ww_extension_suffixfile",
128                  type="string",
129                  default="wwph.suffixes")
130parser.add_option("--write-ef",
131                  help="Upon termination, write the extensive form of the model - accounting for all fixed variables.",
132                  action="store_true",
133                  dest="write_ef",
134                  default=False)
135parser.add_option("--solve-ef",
136                  help="Following write of the extensive form model, solve it.",
137                  action="store_true",
138                  dest="solve_ef",
139                  default=False)
140parser.add_option("--ef-output-file",
141                  help="The name of the extensive form output file (currently only LP format is supported), if writing of the extensive form is enabled. Default is efout.lp.",
142                  action="store",
143                  dest="ef_output_file",
144                  type="string",
145                  default="efout.lp")
146parser.add_option("--suppress-continuous-variable-output",
147                  help="Eliminate PH-related output involving continuous variables.",
148                  action="store_true",
149                  dest="suppress_continuous_variable_output",
150                  default=False)
151parser.add_option("--keep-solver-files",
152                  help="Retain temporary input and output files for scenario sub-problem solves",
153                  action="store_true",
154                  dest="keep_solver_files",
155                  default=False)
156parser.add_option("--output-solver-logs",
157                  help="Output solver logs during scenario sub-problem solves",
158                  action="store_true",
159                  dest="output_solver_logs",
160                  default=False)
161parser.add_option("--output-ef-solver-log",
162                  help="Output solver log during the extensive form solve",
163                  action="store_true",
164                  dest="output_ef_solver_log",
165                  default=False)
166parser.add_option("--output-solver-results",
167                  help="Output solutions obtained after each scenario sub-problem solve",
168                  action="store_true",
169                  dest="output_solver_results",
170                  default=False)
171parser.add_option("--output-times",
172                  help="Output timing statistics for various PH components",
173                  action="store_true",
174                  dest="output_times",
175                  default=False)
176parser.add_option("--disable-warmstarts",
177                  help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1",
178                  action="store_true",
179                  dest="disable_warmstarts",
180                  default=False)
181parser.add_option("--retain-quadratic-binary-terms",
182                  help="Do not linearize PH objective terms involving binary decision variables",
183                  action="store_true",
184                  dest="retain_quadratic_binary_terms",
185                  default=False)
186parser.add_option("--profile",
187                  help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
188                  action="store",
189                  dest="profile",
190                  type="int",
191                  default=0)
192parser.add_option("--enable-gc",
193                  help="Enable the python garbage collecter.",
194                  action="store_true",
195                  dest="enable_gc",
196                  default=False)
197
198
199parser.usage="runph [options]"
200
201#
202# The main PH initialization / runner routine.
203#
204
205def run_ph(options, args):
206
207   start_time = time.time()
208
209   #
210   # create and populate the reference model/instance pair.
211   #
212
213   reference_model = None
214   reference_instance = None
215
216   try:
217      reference_model_filename = options.model_directory+os.sep+"ReferenceModel.py"
218      if options.verbose is True:
219         print "Scenario reference model filename="+reference_model_filename
220      model_import = pyutilib.import_file(reference_model_filename)
221      if "model" not in dir(model_import):
222         print ""
223         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
224         return
225
226      if model_import.model is None:
227         print ""
228         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
229         return
230 
231      reference_model = model_import.model
232   except IOError:
233      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
234      return
235
236   try:
237      reference_instance_filename = options.instance_directory+os.sep+"ReferenceModel.dat"
238      if options.verbose is True:
239         print "Scenario reference instance filename="+reference_instance_filename
240      reference_instance = reference_model.create(reference_instance_filename)
241   except IOError:
242      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
243      return     
244
245   #
246   # create and populate the scenario tree model
247   #
248
249   scenario_tree_model = None
250   scenario_tree_instance = None
251
252   try:
253      scenario_tree_model_filename = options.model_directory+os.sep+"ScenarioStructure.py"
254      if options.verbose is True:
255         print "Scenario tree model filename="+scenario_tree_model_filename
256      scenario_tree_import = pyutilib.import_file(scenario_tree_model_filename)
257      scenario_tree_model = scenario_tree_import.model
258   except IOError:
259      print "***ERROR: Failed to load scenario tree reference model from file="+scenario_tree_model_filename
260      return   
261
262   try:
263      scenario_tree_instance_filename = options.instance_directory+os.sep+"ScenarioStructure.dat"
264      if options.verbose is True:
265         print "Scenario tree instance filename="+scenario_tree_instance_filename
266      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
267   except IOError:
268      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
269      return
270   
271   #
272   # construct the scenario tree
273   #
274   scenario_tree = ScenarioTree(model=reference_instance,
275                                nodes=scenario_tree_instance.Nodes,
276                                nodechildren=scenario_tree_instance.Children,
277                                nodestages=scenario_tree_instance.NodeStage,
278                                nodeprobabilities=scenario_tree_instance.ConditionalProbability,
279                                stages=scenario_tree_instance.Stages,
280                                stagevariables=scenario_tree_instance.StageVariables,
281                                stagecostvariables=scenario_tree_instance.StageCostVariable,
282                                scenarios=scenario_tree_instance.Scenarios,
283                                scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
284                                scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
285
286   #
287   # print the input tree for validation/information purposes.
288   #
289   if options.verbose is True:
290      scenario_tree.pprint()
291
292   #
293   # validate the tree prior to doing anything serious
294   #
295   print ""
296   if scenario_tree.validate() is False:
297      print "***ERROR: Scenario tree is invalid****"
298      return
299   else:
300      if options.verbose is True:
301         print "Scenario tree is valid!"
302   print ""
303
304   #
305   # deal with any plugins (we currently only deal with one!)
306   #
307   if options.enable_ww_extensions is True:
308
309      from coopr.pysp import wwphextension
310
311      plugin = ExtensionPoint(IPHExtension)
312      plugin.service()._configuration_filename = options.ww_extension_cfgfile
313      plugin.service()._suffix_filename = options.ww_extension_suffixfile
314
315   #
316   # construct the convergence "computer" class.
317   #
318   converger = None
319   # go with the non-defaults first, and then with the default.
320   if options.enable_free_discrete_count_convergence is True:
321      converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
322   elif options.enable_normalized_termdiff_convergence is True:
323      converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
324   else:
325      converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
326
327     
328   #
329   # construct and initialize PH
330   #
331   ph = ProgressiveHedging(max_iterations=options.max_iterations, \
332                           rho=options.default_rho, \
333                           rho_setter=options.rho_cfgfile, \
334                           solver=options.solver_type, \
335                           solver_manager=options.solver_manager_type, \
336                           keep_solver_files=options.keep_solver_files, \
337                           output_solver_log=options.output_solver_logs, \
338                           output_solver_results=options.output_solver_results, \
339                           verbose=options.verbose, \
340                           output_times=options.output_times, \
341                           disable_warmstarts=options.disable_warmstarts,
342                           retain_quadratic_binary_terms=options.retain_quadratic_binary_terms)
343   
344   ph.initialize(scenario_data_directory_name=options.instance_directory, \
345                 model=reference_model, \
346                 model_instance=reference_instance, \
347                 scenario_tree=scenario_tree, \
348                 converger=converger)
349
350   if options.suppress_continuous_variable_output is True:
351      ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
352
353   #
354   # kick off the solve
355   #
356   ph.solve()
357
358   print ""
359   print "DONE..."
360
361   end_time = time.time()
362
363   print ""
364   print "Total execution time=%8.2f seconds" %(end_time - start_time)
365   print ""
366   if options.output_times is True:
367      ph.print_time_stats()
368
369   #
370   # write the extensive form, accounting for any fixed variables.
371   #
372   if (options.write_ef is True) or (options.solve_ef is True):
373      print ""
374      print "Writing EF for remainder problem"
375      print ""
376      write_ef(ph._scenario_tree, ph._instances, options.ef_output_file)
377
378   #
379   # solve the extensive form.
380   #
381   if options.solve_ef is True:
382      print ""
383      print "Solving extensive form written to file="+options.ef_output_file
384      print ""
385
386      ef_solver = SolverFactory(options.solver_type)
387      if ef_solver is None:
388         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
389
390      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
391      if ef_solver is None:
392         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
393
394      print "Queuing extensive form solve"
395      ef_action_handle = ef_solver_manager.queue(options.ef_output_file, opt=ef_solver, warmstart=False, tee=options.output_ef_solver_log)
396      print "Waiting for extensive form solve"
397      ef_results = ef_solver_manager.wait_for(ef_action_handle)
398      print "Extensive form solve results:"
399      print ef_results
400     
401
402def run(args=None):
403
404    #
405    # Top-level command that executes the extensive form writer.
406    # This is segregated from run_ef_writer to enable profiling.
407    #
408
409    #
410    # Parse command-line options.
411    #
412    try:
413       (options, args) = parser.parse_args(args=args)
414    except SystemExit:
415       # the parser throws a system exit if "-h" is specified - catch
416       # it to exit gracefully.
417       return
418
419    # for a one-pass execution, garbage collection doesn't make
420    # much sense - so it is disabled by default. Because: It drops
421    # the run-time by a factor of 3-4 on bigger instances.
422    if options.enable_gc is False:
423       gc.disable()       
424
425    if options.profile > 0:
426        #
427        # Call the main PH routine with profiling.
428        #
429        tfile = pyutilib.TempfileManager.create_tempfile(suffix=".profile")
430        tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
431        p = pstats.Stats(tfile).strip_dirs()
432        p.sort_stats('time', 'cum')
433        options.profile = eval(options.profile)
434        p = p.print_stats(options.profile)
435        p.print_callers(options.profile)
436        p.print_callees(options.profile)
437        p = p.sort_stats('cum','calls')
438        p.print_stats(options.profile)
439        p.print_callers(options.profile)
440        p.print_callees(options.profile)
441        p = p.sort_stats('calls')
442        p.print_stats(options.profile)
443        p.print_callers(options.profile)
444        p.print_callees(options.profile)
445        pyutilib.TempfileManager.clear_tempfiles()
446        ans = [tmp, None]
447    else:
448        #
449        # Call the main PH routine without profiling.
450        #
451        ans = run_ph(options, args)
452
453    gc.enable()
454   
455    return ans
456
Note: See TracBrowser for help on using the repository browser.