source: coopr.pysp/trunk/coopr/pysp/phserver.py @ 2406

Last change on this file since 2406 was 2406, checked in by jwatson, 9 years ago

More tweaks to the PH solver manager/servers.

  • Property svn:executable set to *
File size: 20.2 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2010 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  Fr more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20import time
21
22# Coopr
23from coopr.opt.base import SolverFactory
24from coopr.pysp.scenariotree import *
25from phutils import *
26from phobjective import *
27
28# Pyro
29import Pyro.core
30import Pyro.naming
31
32# garbage collection control.
33import gc
34
35# for profiling
36import cProfile
37import pstats
38
39import pickle
40
41# disable multi-threading. for a solver server, this
42# would truly be a bad idea, as you (1) likely have limited
43# solver licenses and (2) likely want to dedicate all compute
44# resources on this node to a single solve.
45Pyro.config.PYRO_MULTITHREADED=0
46
47#
48# the class responsible for executing all solution server requests.
49#
50class PHSolverServer(object):
51
52   def __init__(self, scenario_instances, solver, scenario_tree):
53
54      # the obvious stuff!
55      self._instances = scenario_instances
56      self._solver = solver
57      self._scenario_tree = scenario_tree
58
59      # the objective functions are modified throughout the course of PH iterations.
60      # save the original, as a baseline to modify in subsequent iterations. reserve
61      # the original objectives, for subsequent modification.
62      self._original_objective_expression = {}
63      for instance_name, instance in self._instances.items():
64         objective_name = instance.active_components(Objective).keys()[0]
65         expr = instance.active_components(Objective)[objective_name]._data[None].expr
66         if isinstance(expr, Expression) is False:
67            expr = _IdentityExpression(expr)
68         self._original_objective_expression[instance_name] = expr
69
70      # the server is in one of two modes - solve the baseline instances, or
71      # those augmented with the PH penalty terms. default is standard.
72      # NOTE: This is currently a global flag for all scenarios handled
73      #       by this server - easy enough to extend if we want.
74      self._solving_standard_objective = True
75
76   def enable_standard_objective(self):
77
78      print "RECEIVED REQUEST TO ENABLE STANDARD OBJECTIVE FOR ALL SCENARIOS"
79
80      self._solving_standard_objective = True
81
82   def enable_ph_objective(self):
83
84      print "RECEIVED REQUEST TO ENABLE PH OBJECTIVE FOR ALL SCENARIOS"
85
86      self._solving_standard_objective = False
87
88   def solve(self, scenario_name):
89
90      print "RECEIVED REQUEST TO SOLVE SCENARIO INSTANCE="+scenario_name
91      if self._solving_standard_objective is True:
92         print "OBJECTIVE=STANDARD"
93      else:
94         print "OBJECTIVE=PH"
95         
96      if scenario_name not in self._instances:
97         print "***ERROR: Requested instance to solve not in PH solver server instance collection!"
98         return None
99      scenario_instance = self._instances[scenario_name]
100
101      # IMPT: You have to re-presolve, as the simple presolver collects the linear terms together. If you
102      # don't do this, you won't see any chance in the output files as you vary the problem parameters!
103      # ditto for instance fixing!
104      scenario_instance.preprocess()
105
106      # form the desired objective, depending on the solve mode.
107      if self._solving_standard_objective is True:
108         form_standard_objective(scenario_name, scenario_instance, self._original_objective_expression[scenario_name], self._scenario_tree)         
109      else:
110         # TBD - command-line drive the various options (integer tolerance, breakpoint strategies, etc.)
111         form_ph_objective(scenario_name, scenario_instance, \
112                           self._original_objective_expression[scenario_name], self._scenario_tree, \
113                           False, False, False, 0, 0.00001)
114                           
115      results = self._solver.solve(scenario_instance)
116
117      print "Successfully solved scenario instance="+scenario_name
118#      print "RESULTS:"
119#      results.write(num=1)
120      encoded_results = pickle.dumps(results)
121     
122      return encoded_results
123
124   def update_weights_and_averages(self, scenario_name, new_weights, new_averages):
125
126      print "RECEIVED REQUEST TO UPDATE WEIGHTS AND AVERAGES FOR SCENARIO=",scenario_name
127
128      if scenario_name not in self._instances:
129         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
130         return None
131      scenario_instance = self._instances[scenario_name]
132     
133      for weight_update in new_weights:
134         weight_update.pprint()
135         
136         weight_index = weight_update._index
137
138         target_weight_parameter = getattr(scenario_instance, weight_update.name)
139
140         for index in weight_index:
141            target_weight_parameter[index] = weight_update[index]
142
143      for average_update in new_averages:
144         average_update.pprint()
145         
146         average_index = average_update._index
147
148         target_average_parameter = getattr(scenario_instance, average_update.name)
149
150         for index in average_index:
151            target_average_parameter[index] = average_update[index]           
152
153   def update_rhos(self, scenario_name, new_rhos):
154
155      print "RECEIVED REQUEST TO UPDATE RHOS FOR SCENARIO=",scenario_name
156
157      if scenario_name not in self._instances:
158         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
159         return None
160      scenario_instance = self._instances[scenario_name]     
161
162      for rho_update in new_rhos:
163         rho_update.pprint()
164         
165         rho_index = rho_update._index
166
167         target_rho_parameter = getattr(scenario_instance, rho_update.name)
168
169         for index in rho_index:
170            target_rho_parameter[index] = rho_update[index]
171
172   def update_tree_node_statistics(self, scenario_name, new_node_minimums, new_node_maximums):
173
174      print "RECEIVED REQUEST TO UPDATE TREE NODE STATISTICS SCENARIO=",scenario_name
175
176      print "TREE NODE MINS=",new_node_minimums
177      print "TREE NODE MAXS=",new_node_maximums     
178
179      for tree_node_name, tree_node_minimums in new_node_minimums.items():
180
181         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
182         tree_node._minimums = tree_node_minimums
183
184      for tree_node_name, tree_node_maximums in new_node_maximums.items():
185
186         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
187         tree_node._maximums = tree_node_maximums         
188     
189           
190#
191# utility method to construct an option parser for ph arguments, to be
192# supplied as an argument to the runph method.
193#
194
195def construct_options_parser(usage_string):
196
197   parser = OptionParser()
198   parser.add_option("--verbose",
199                     help="Generate verbose output for both initialization and execution. Default is False.",
200                     action="store_true",
201                     dest="verbose",
202                     default=False)
203   parser.add_option("--scenario",
204                     help="Specify a scenario that this server is responsible for solving",
205                     action="append",
206                     dest="scenarios",
207                     default=[])
208   parser.add_option("--all-scenarios",
209                     help="Indicate that the server is responsible for solving all scenarios",
210                     action="store_true",
211                     dest="all_scenarios",
212                     default=False)
213   parser.add_option("--report-solutions",
214                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
215                     action="store_true",
216                     dest="report_solutions",
217                     default=False)
218   parser.add_option("--report-weights",
219                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
220                     action="store_true",
221                     dest="report_weights",
222                     default=False)
223   parser.add_option("--model-directory",
224                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
225                     action="store",
226                     dest="model_directory",
227                     type="string",
228                     default=".")
229   parser.add_option("--instance-directory",
230                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
231                     action="store",
232                     dest="instance_directory",
233                     type="string",
234                     default=".")
235   parser.add_option("--solver",
236                     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
237                     action="store",
238                     dest="solver_type",
239                     type="string",
240                     default="cplex")
241   parser.add_option("--scenario-solver-options",
242                     help="Solver options for all PH scenario sub-problems",
243                     action="append",
244                     dest="scenario_solver_options",
245                     type="string",
246                     default=[])
247   parser.add_option("--scenario-mipgap",
248                     help="Specifies the mipgap for all PH scenario sub-problems",
249                     action="store",
250                     dest="scenario_mipgap",
251                     type="float",
252                     default=None)
253   parser.add_option("--keep-solver-files",
254                     help="Retain temporary input and output files for scenario sub-problem solves",
255                     action="store_true",
256                     dest="keep_solver_files",
257                     default=False)
258   parser.add_option("--output-solver-logs",
259                     help="Output solver logs during scenario sub-problem solves",
260                     action="store_true",
261                     dest="output_solver_logs",
262                     default=False)
263   parser.add_option("--output-solver-results",
264                     help="Output solutions obtained after each scenario sub-problem solve",
265                     action="store_true",
266                     dest="output_solver_results",
267                     default=False)
268   parser.add_option("--output-times",
269                     help="Output timing statistics for various PH components",
270                     action="store_true",
271                     dest="output_times",
272                     default=False)
273   parser.add_option("--disable-warmstarts",
274                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
275                     action="store_true",
276                     dest="disable_warmstarts",
277                     default=False)
278   parser.add_option("--profile",
279                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
280                     action="store",
281                     dest="profile",
282                     type="int",
283                     default=0)
284   parser.add_option("--disable-gc",
285                     help="Disable the python garbage collecter. Default is False.",
286                     action="store_true",
287                     dest="disable_gc",
288                     default=False)
289
290   parser.usage=usage_string
291
292   return parser
293
294#
295# Create the reference model / instance and scenario tree instance for PH.
296#
297
298def load_reference_and_scenario_models(options):
299
300   #
301   # create and populate the reference model/instance pair.
302   #
303
304   reference_model = None
305   reference_instance = None
306
307   try:
308      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
309      if options.verbose is True:
310         print "Scenario reference model filename="+reference_model_filename
311      model_import = pyutilib.misc.import_file(reference_model_filename)
312      if "model" not in dir(model_import):
313         print ""
314         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
315         return None, None, None, None
316
317      if model_import.model is None:
318         print ""
319         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
320         return None, None, None, None
321 
322      reference_model = model_import.model
323   except IOError:
324      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
325      return None, None, None, None
326
327   try:
328      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
329      if options.verbose is True:
330         print "Scenario reference instance filename="+reference_instance_filename
331      reference_instance = reference_model.create(reference_instance_filename)
332   except IOError:
333      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
334      return None, None, None, None
335
336   #
337   # create and populate the scenario tree model
338   #
339
340   from coopr.pysp.util.scenariomodels import scenario_tree_model
341   scenario_tree_instance = None
342
343   try:
344      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
345      if options.verbose is True:
346         print "Scenario tree instance filename="+scenario_tree_instance_filename
347      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
348   except IOError:
349      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
350      return None, None, None
351   
352   #
353   # construct the scenario tree
354   #
355   scenario_tree = ScenarioTree(scenarioinstance=reference_instance,
356                                scenariotreeinstance=scenario_tree_instance)
357
358   return reference_model, reference_instance, scenario_tree, scenario_tree_instance
359
360#
361# Execute the PH solver server daemon.
362#
363
364def run_server(options, scenario_instances, solver, scenario_tree):
365
366   start_time = time.time()
367
368   Pyro.core.initServer(banner=0)   
369
370   locator = Pyro.naming.NameServerLocator()
371   ns = None
372   try:
373      ns = locator.getNS()
374   except Pyro.errors.NamingError:
375      raise RuntimeError, "PH solver server failed to locate Pyro name server"     
376
377   solver_daemon = Pyro.core.Daemon()
378   solver_daemon.useNameServer(ns)
379
380   daemon_object = PHSolverServer(scenario_instances, solver, scenario_tree)
381   delegator_object = Pyro.core.ObjBase()
382   delegator_object.delegateTo(daemon_object)
383
384   # register an entry in the nameserver for
385   # each scenario processed by this daemon.
386   for scenario_name, scenario_instance in scenario_instances.items():
387      try:
388         solver_daemon.connect(delegator_object, scenario_name)
389      except Pyro.errors.NamingError:
390         raise RuntimeError, "Entry in name server already exists for scenario="+scenario_name
391
392   while True:
393      solver_daemon.handleRequests()
394
395   solver_daemon.disconnect()
396
397   end_time = time.time()
398
399#
400# The main daemon initialization / runner routine.
401#
402
403def exec_server(options):
404
405   # we need the base model (not so much the reference instance, but that
406   # can't hurt too much - until proven otherwise, that is) to construct
407   # the scenarios that this server is responsible for.
408   reference_model, reference_instance, scenario_tree, scenario_tree_instance = load_reference_and_scenario_models(options)
409   if reference_model is None or reference_instance is None or scenario_tree is None:
410      print "***Unable to launch PH solver server"
411      return
412
413   # construct the set of scenario instances based on the command-line.
414   scenario_instances = {}
415   if (options.all_scenarios is True) and (len(options.scenarios) > 0):
416      print "***WARNING: Both individual scenarios and all-scenarios were specified on the command-line; proceeding using all scenarios"
417
418   if (len(options.scenarios) == 0) and (options.all_scenarios is False):
419      print "***Unable to launch PH solver server - no scenario(s) specified!"
420      return
421
422   scenarios_to_construct = []
423   if options.all_scenarios is True:
424      scenarios_to_construct.extend(scenario_tree._scenario_map.keys())
425   else:
426      scenarios_to_construct.extend(options.scenarios)
427
428   for scenario_name in scenarios_to_construct:
429      print "Creating instance for scenario="+scenario_name
430      if scenario_tree.contains_scenario(scenario_name) is False:
431         print "***Unable to launch PH solver server - unknown scenario specified with name="+scenario_name
432         return
433
434      # create the baseline scenario instance
435      scenario_instance = construct_scenario_instance(scenario_tree,
436                                                      options.instance_directory,
437                                                      scenario_name,
438                                                      reference_model,
439                                                      options.verbose)
440      if scenario_instance is None:
441         print "***Unable to launch PH solver server - failed to create instance for scenario="+scenario_name
442         return
443
444      # augment the instance with PH-specific parameters (weights, rhos, etc).
445      # TBD: The default rho of 1.0 is kind of bogus. Need to somehow propagate
446      #      this value and the linearization parameter as a command-line argument.
447      new_penalty_variable_names = create_ph_parameters(scenario_instance, scenario_tree, 1.0, False)
448
449      scenario_instances[scenario_name] = scenario_instance
450
451   # the solver instance is persistent, applicable to all instances here.
452   if options.verbose is True:
453      print "Constructing solver type="+options.solver_type         
454   solver = SolverFactory(options.solver_type)
455   if solver == None:
456      raise ValueError, "Unknown solver type=" + options.solver_type + " specified"
457   if options.keep_solver_files is True:
458      solver.keepFiles = True
459   if len(options.scenario_solver_options) > 0:
460      if options.verbose is True:
461         print "Initializing scenario sub-problem solver with options="+str(options.scenario_solver_options)
462      solver.set_options("".join(options.scenario_solver_options))
463   if options.output_times is True:
464      solver._report_timing = True
465
466   # spawn the daemon.
467   run_server(options, scenario_instances, solver, scenario_tree)
468
469def run(args=None):
470
471    #
472    # Top-level command that executes the ph solver server daemon.
473    # This is segregated from phsolverserver to faciliate profiling.
474    #
475
476    #
477    # Parse command-line options.
478    #
479    try:
480       options_parser = construct_options_parser("phsolverserver [options]")
481       (options, args) = options_parser.parse_args(args=args)
482    except SystemExit:
483       # the parser throws a system exit if "-h" is specified - catch
484       # it to exit gracefully.
485       return
486
487    # for a one-pass execution, garbage collection doesn't make
488    # much sense - so it is disabled by default. Because: It drops
489    # the run-time by a factor of 3-4 on bigger instances.
490    if options.disable_gc is True:
491       gc.disable()
492    else:
493       gc.enable()
494
495    if options.profile > 0:
496        #
497        # Call the main PH routine with profiling.
498        #
499        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
500        tmp = cProfile.runctx('exec_ph(options)',globals(),locals(),tfile)
501        p = pstats.Stats(tfile).strip_dirs()
502        p.sort_stats('time', 'cum')
503        p = p.print_stats(options.profile)
504        p.print_callers(options.profile)
505        p.print_callees(options.profile)
506        p = p.sort_stats('cum','calls')
507        p.print_stats(options.profile)
508        p.print_callers(options.profile)
509        p.print_callees(options.profile)
510        p = p.sort_stats('calls')
511        p.print_stats(options.profile)
512        p.print_callers(options.profile)
513        p.print_callees(options.profile)
514        pyutilib.services.TempfileManager.clear_tempfiles()
515        ans = [tmp, None]
516    else:
517        #
518        # Call the main PH routine without profiling.
519        #
520        ans = exec_server(options)
521
522    gc.enable()
523   
524    return ans
525
Note: See TracBrowser for help on using the repository browser.