source: coopr.pysp/trunk/coopr/pysp/phserver.py @ 2410

Last change on this file since 2410 was 2410, checked in by jwatson, 9 years ago

Fixed the PH solver server to automatically deregister a delegator object if it already exists in the name server.

Fixed various bugs in the PH solver server relating to quadratic expression output and weight/rho updates. It now replicates CPLEX results across the server network.

  • Property svn:executable set to *
File size: 20.5 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2010 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  Fr more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20import time
21
22# Coopr
23from coopr.opt.base import SolverFactory
24from coopr.pysp.scenariotree import *
25from phutils import *
26from phobjective import *
27
28# Pyro
29import Pyro.core
30import Pyro.naming
31from Pyro.errors import PyroError,NamingError
32
33# garbage collection control.
34import gc
35
36# for profiling
37import cProfile
38import pstats
39
40import pickle
41
42# disable multi-threading. for a solver server, this
43# would truly be a bad idea, as you (1) likely have limited
44# solver licenses and (2) likely want to dedicate all compute
45# resources on this node to a single solve.
46Pyro.config.PYRO_MULTITHREADED=0
47
48#
49# the class responsible for executing all solution server requests.
50#
51class PHSolverServer(object):
52
53   def __init__(self, scenario_instances, solver, scenario_tree):
54
55      # the obvious stuff!
56      self._instances = scenario_instances # a map from scenario name to pyomo model instance
57      self._solver = solver
58      self._scenario_tree = scenario_tree
59
60      # the objective functions are modified throughout the course of PH iterations.
61      # save the original, as a baseline to modify in subsequent iterations. reserve
62      # the original objectives, for subsequent modification.
63      self._original_objective_expression = {}
64      for instance_name, instance in self._instances.items():
65         objective_name = instance.active_components(Objective).keys()[0]
66         expr = instance.active_components(Objective)[objective_name]._data[None].expr
67         if isinstance(expr, Expression) is False:
68            expr = _IdentityExpression(expr)
69         self._original_objective_expression[instance_name] = expr
70
71      # the server is in one of two modes - solve the baseline instances, or
72      # those augmented with the PH penalty terms. default is standard.
73      # NOTE: This is currently a global flag for all scenarios handled
74      #       by this server - easy enough to extend if we want.
75      self._solving_standard_objective = True
76
77   def enable_standard_objective(self):
78
79#      print "RECEIVED REQUEST TO ENABLE STANDARD OBJECTIVE FOR ALL SCENARIOS"
80
81      self._solving_standard_objective = True
82
83   def enable_ph_objective(self):
84
85#      print "RECEIVED REQUEST TO ENABLE PH OBJECTIVE FOR ALL SCENARIOS"
86
87      self._solving_standard_objective = False
88
89   def solve(self, scenario_name):
90
91#      print "RECEIVED REQUEST TO SOLVE SCENARIO INSTANCE="+scenario_name
92#      if self._solving_standard_objective is True:
93#         print "OBJECTIVE=STANDARD"
94#      else:
95#         print "OBJECTIVE=PH"
96         
97      if scenario_name not in self._instances:
98         print "***ERROR: Requested instance to solve not in PH solver server instance collection!"
99         return None
100      scenario_instance = self._instances[scenario_name]
101
102      # form the desired objective, depending on the solve mode.
103      if self._solving_standard_objective is True:
104         form_standard_objective(scenario_name, scenario_instance, self._original_objective_expression[scenario_name], self._scenario_tree)         
105      else:
106         # TBD - command-line drive the various options (integer tolerance, breakpoint strategies, etc.)
107         form_ph_objective(scenario_name, scenario_instance, \
108                           self._original_objective_expression[scenario_name], self._scenario_tree, \
109                           False, False, False, 0, 0.00001)
110
111      # IMPT: You have to re-presolve, as the simple presolver collects the linear terms together. If you
112      # don't do this, you won't see any chance in the output files as you vary the problem parameters!
113      # ditto for instance fixing!
114      scenario_instance.preprocess()
115
116      results = self._solver.solve(scenario_instance)
117
118      print "Successfully solved scenario instance="+scenario_name
119#      print "RESULTS:"
120#      results.write(num=1)
121      encoded_results = pickle.dumps(results)
122     
123      return encoded_results
124
125   def update_weights_and_averages(self, scenario_name, new_weights, new_averages):
126
127#      print "RECEIVED REQUEST TO UPDATE WEIGHTS AND AVERAGES FOR SCENARIO=",scenario_name
128
129      if scenario_name not in self._instances:
130         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
131         return None
132      scenario_instance = self._instances[scenario_name]
133     
134      for weight_update in new_weights:
135         
136         weight_index = weight_update._index
137
138         target_weight_parameter = getattr(scenario_instance, weight_update.name)
139
140         for index in weight_index:
141            target_weight_parameter[index] = weight_update[index]()
142
143      for average_update in new_averages:
144         
145         average_index = average_update._index
146
147         target_average_parameter = getattr(scenario_instance, average_update.name)
148
149         for index in average_index:
150            target_average_parameter[index] = average_update[index]()
151
152   def update_rhos(self, scenario_name, new_rhos):
153
154#      print "RECEIVED REQUEST TO UPDATE RHOS FOR SCENARIO=",scenario_name
155
156      if scenario_name not in self._instances:
157         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
158         return None
159      scenario_instance = self._instances[scenario_name]     
160
161      for rho_update in new_rhos:
162
163         rho_index = rho_update._index
164
165         target_rho_parameter = getattr(scenario_instance, rho_update.name)
166
167         for index in rho_index: 
168            target_rho_parameter[index] = rho_update[index]() # the value operator is crucial!
169
170   def update_tree_node_statistics(self, scenario_name, new_node_minimums, new_node_maximums):
171
172#      print "RECEIVED REQUEST TO UPDATE TREE NODE STATISTICS SCENARIO=",scenario_name
173
174      for tree_node_name, tree_node_minimums in new_node_minimums.items():
175
176         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
177         tree_node._minimums = tree_node_minimums
178
179      for tree_node_name, tree_node_maximums in new_node_maximums.items():
180
181         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
182         tree_node._maximums = tree_node_maximums         
183     
184           
185#
186# utility method to construct an option parser for ph arguments, to be
187# supplied as an argument to the runph method.
188#
189
190def construct_options_parser(usage_string):
191
192   parser = OptionParser()
193   parser.add_option("--verbose",
194                     help="Generate verbose output for both initialization and execution. Default is False.",
195                     action="store_true",
196                     dest="verbose",
197                     default=False)
198   parser.add_option("--scenario",
199                     help="Specify a scenario that this server is responsible for solving",
200                     action="append",
201                     dest="scenarios",
202                     default=[])
203   parser.add_option("--all-scenarios",
204                     help="Indicate that the server is responsible for solving all scenarios",
205                     action="store_true",
206                     dest="all_scenarios",
207                     default=False)
208   parser.add_option("--report-solutions",
209                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
210                     action="store_true",
211                     dest="report_solutions",
212                     default=False)
213   parser.add_option("--report-weights",
214                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
215                     action="store_true",
216                     dest="report_weights",
217                     default=False)
218   parser.add_option("--model-directory",
219                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
220                     action="store",
221                     dest="model_directory",
222                     type="string",
223                     default=".")
224   parser.add_option("--instance-directory",
225                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
226                     action="store",
227                     dest="instance_directory",
228                     type="string",
229                     default=".")
230   parser.add_option("--solver",
231                     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
232                     action="store",
233                     dest="solver_type",
234                     type="string",
235                     default="cplex")
236   parser.add_option("--scenario-solver-options",
237                     help="Solver options for all PH scenario sub-problems",
238                     action="append",
239                     dest="scenario_solver_options",
240                     type="string",
241                     default=[])
242   parser.add_option("--scenario-mipgap",
243                     help="Specifies the mipgap for all PH scenario sub-problems",
244                     action="store",
245                     dest="scenario_mipgap",
246                     type="float",
247                     default=None)
248   parser.add_option("--keep-solver-files",
249                     help="Retain temporary input and output files for scenario sub-problem solves",
250                     action="store_true",
251                     dest="keep_solver_files",
252                     default=False)
253   parser.add_option("--output-solver-logs",
254                     help="Output solver logs during scenario sub-problem solves",
255                     action="store_true",
256                     dest="output_solver_logs",
257                     default=False)
258   parser.add_option("--output-solver-results",
259                     help="Output solutions obtained after each scenario sub-problem solve",
260                     action="store_true",
261                     dest="output_solver_results",
262                     default=False)
263   parser.add_option("--output-times",
264                     help="Output timing statistics for various PH components",
265                     action="store_true",
266                     dest="output_times",
267                     default=False)
268   parser.add_option("--disable-warmstarts",
269                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
270                     action="store_true",
271                     dest="disable_warmstarts",
272                     default=False)
273   parser.add_option("--profile",
274                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
275                     action="store",
276                     dest="profile",
277                     type="int",
278                     default=0)
279   parser.add_option("--disable-gc",
280                     help="Disable the python garbage collecter. Default is False.",
281                     action="store_true",
282                     dest="disable_gc",
283                     default=False)
284
285   parser.usage=usage_string
286
287   return parser
288
289#
290# Create the reference model / instance and scenario tree instance for PH.
291#
292
293def load_reference_and_scenario_models(options):
294
295   #
296   # create and populate the reference model/instance pair.
297   #
298
299   reference_model = None
300   reference_instance = None
301
302   try:
303      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
304      if options.verbose is True:
305         print "Scenario reference model filename="+reference_model_filename
306      model_import = pyutilib.misc.import_file(reference_model_filename)
307      if "model" not in dir(model_import):
308         print ""
309         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
310         return None, None, None, None
311
312      if model_import.model is None:
313         print ""
314         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
315         return None, None, None, None
316 
317      reference_model = model_import.model
318   except IOError:
319      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
320      return None, None, None, None
321
322   try:
323      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
324      if options.verbose is True:
325         print "Scenario reference instance filename="+reference_instance_filename
326      reference_instance = reference_model.create(reference_instance_filename)
327   except IOError:
328      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
329      return None, None, None, None
330
331   #
332   # create and populate the scenario tree model
333   #
334
335   from coopr.pysp.util.scenariomodels import scenario_tree_model
336   scenario_tree_instance = None
337
338   try:
339      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
340      if options.verbose is True:
341         print "Scenario tree instance filename="+scenario_tree_instance_filename
342      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
343   except IOError:
344      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
345      return None, None, None
346   
347   #
348   # construct the scenario tree
349   #
350   scenario_tree = ScenarioTree(scenarioinstance=reference_instance,
351                                scenariotreeinstance=scenario_tree_instance)
352
353   return reference_model, reference_instance, scenario_tree, scenario_tree_instance
354
355#
356# Execute the PH solver server daemon.
357#
358
359def run_server(options, scenario_instances, solver, scenario_tree):
360
361   start_time = time.time()
362
363   Pyro.core.initServer(banner=0)   
364
365   locator = Pyro.naming.NameServerLocator()
366   ns = None
367   try:
368      ns = locator.getNS()
369   except Pyro.errors.NamingError:
370      raise RuntimeError, "PH solver server failed to locate Pyro name server"     
371
372   solver_daemon = Pyro.core.Daemon()
373   solver_daemon.useNameServer(ns)
374
375   daemon_object = PHSolverServer(scenario_instances, solver, scenario_tree)
376   delegator_object = Pyro.core.ObjBase()
377   delegator_object.delegateTo(daemon_object)
378
379   # register an entry in the nameserver for
380   # each scenario processed by this daemon.
381   for scenario_name, scenario_instance in scenario_instances.items():
382      # first, see if it's already registered - if no, cull the old entry.
383      # NOTE: This is a hack, as the cleanup / disconnect code doesn't
384      #       seem to be invoked when the daemon is killed.
385      try:
386         ns.unregister(scenario_name)
387      except NamingError:
388         pass
389
390      try:
391         solver_daemon.connect(delegator_object, scenario_name)
392      except Pyro.errors.NamingError:
393         raise RuntimeError, "Entry in name server already exists for scenario="+scenario_name
394
395   try:
396      solver_daemon.requestLoop()
397   except KeyboardInterrupt, SystemExit:
398      pass
399
400   solver_daemon.disconnect(delegator_object)
401   solver_daemon.shutdown()
402
403   end_time = time.time()
404
405#
406# The main daemon initialization / runner routine.
407#
408
409def exec_server(options):
410
411   # we need the base model (not so much the reference instance, but that
412   # can't hurt too much - until proven otherwise, that is) to construct
413   # the scenarios that this server is responsible for.
414   reference_model, reference_instance, scenario_tree, scenario_tree_instance = load_reference_and_scenario_models(options)
415   if reference_model is None or reference_instance is None or scenario_tree is None:
416      print "***Unable to launch PH solver server"
417      return
418
419   # construct the set of scenario instances based on the command-line.
420   scenario_instances = {}
421   if (options.all_scenarios is True) and (len(options.scenarios) > 0):
422      print "***WARNING: Both individual scenarios and all-scenarios were specified on the command-line; proceeding using all scenarios"
423
424   if (len(options.scenarios) == 0) and (options.all_scenarios is False):
425      print "***Unable to launch PH solver server - no scenario(s) specified!"
426      return
427
428   scenarios_to_construct = []
429   if options.all_scenarios is True:
430      scenarios_to_construct.extend(scenario_tree._scenario_map.keys())
431   else:
432      scenarios_to_construct.extend(options.scenarios)
433
434   for scenario_name in scenarios_to_construct:
435      print "Creating instance for scenario="+scenario_name
436      if scenario_tree.contains_scenario(scenario_name) is False:
437         print "***Unable to launch PH solver server - unknown scenario specified with name="+scenario_name
438         return
439
440      # create the baseline scenario instance
441      scenario_instance = construct_scenario_instance(scenario_tree,
442                                                      options.instance_directory,
443                                                      scenario_name,
444                                                      reference_model,
445                                                      options.verbose)
446      if scenario_instance is None:
447         print "***Unable to launch PH solver server - failed to create instance for scenario="+scenario_name
448         return
449
450      # augment the instance with PH-specific parameters (weights, rhos, etc).
451      # TBD: The default rho of 1.0 is kind of bogus. Need to somehow propagate
452      #      this value and the linearization parameter as a command-line argument.
453      new_penalty_variable_names = create_ph_parameters(scenario_instance, scenario_tree, 1.0, False)
454
455      scenario_instances[scenario_name] = scenario_instance
456
457   # the solver instance is persistent, applicable to all instances here.
458   if options.verbose is True:
459      print "Constructing solver type="+options.solver_type         
460   solver = SolverFactory(options.solver_type)
461   if solver == None:
462      raise ValueError, "Unknown solver type=" + options.solver_type + " specified"
463   if options.keep_solver_files is True:
464      solver.keepFiles = True
465   if len(options.scenario_solver_options) > 0:
466      if options.verbose is True:
467         print "Initializing scenario sub-problem solver with options="+str(options.scenario_solver_options)
468      solver.set_options("".join(options.scenario_solver_options))
469   if options.output_times is True:
470      solver._report_timing = True
471
472   # spawn the daemon.
473   run_server(options, scenario_instances, solver, scenario_tree)
474
475def run(args=None):
476
477    #
478    # Top-level command that executes the ph solver server daemon.
479    # This is segregated from phsolverserver to faciliate profiling.
480    #
481
482    #
483    # Parse command-line options.
484    #
485    try:
486       options_parser = construct_options_parser("phsolverserver [options]")
487       (options, args) = options_parser.parse_args(args=args)
488    except SystemExit:
489       # the parser throws a system exit if "-h" is specified - catch
490       # it to exit gracefully.
491       return
492
493    # for a one-pass execution, garbage collection doesn't make
494    # much sense - so it is disabled by default. Because: It drops
495    # the run-time by a factor of 3-4 on bigger instances.
496    if options.disable_gc is True:
497       gc.disable()
498    else:
499       gc.enable()
500
501    if options.profile > 0:
502        #
503        # Call the main PH routine with profiling.
504        #
505        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
506        tmp = cProfile.runctx('exec_ph(options)',globals(),locals(),tfile)
507        p = pstats.Stats(tfile).strip_dirs()
508        p.sort_stats('time', 'cum')
509        p = p.print_stats(options.profile)
510        p.print_callers(options.profile)
511        p.print_callees(options.profile)
512        p = p.sort_stats('cum','calls')
513        p.print_stats(options.profile)
514        p.print_callers(options.profile)
515        p.print_callees(options.profile)
516        p = p.sort_stats('calls')
517        p.print_stats(options.profile)
518        p.print_callers(options.profile)
519        p.print_callees(options.profile)
520        pyutilib.services.TempfileManager.clear_tempfiles()
521        ans = [tmp, None]
522    else:
523        #
524        # Call the main PH routine without profiling.
525        #
526        ans = exec_server(options)
527
528    gc.enable()
529   
530    return ans
531
Note: See TracBrowser for help on using the repository browser.