source: coopr.pysp/trunk/coopr/pysp/phserver.py @ 3482

Last change on this file since 3482 was 3482, checked in by khunter, 9 years ago

Dynamic solver list, minor import fixups.

Make the --help options show the list of available solvers, like pyomo --help.

Also some driveby option additions:

-k (--keep-solver-solutions)
-m (--model-directory)
-i (--instance-directory)

  • Property svn:executable set to *
File size: 20.7 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2010 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  Fr more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import gc         # garbage collection control.
13import os
14import pickle
15import pstats     # for profiling
16import sys
17import time
18import traceback
19
20from optparse import OptionParser
21
22# for profiling
23try:
24    import cProfile as profile
25except ImportError:
26    import profile
27
28# Coopr
29from coopr.opt.base import SolverFactory
30from coopr.pysp.scenariotree import *
31
32from pyutilib.misc import import_file
33from pyutilib.services import TempfileManager
34
35# Pyro
36import Pyro.core
37import Pyro.naming
38from Pyro.errors import PyroError, NamingError
39
40from phutils import *
41from phobjective import *
42
43# disable multi-threading. for a solver server, this
44# would truly be a bad idea, as you (1) likely have limited
45# solver licenses and (2) likely want to dedicate all compute
46# resources on this node to a single solve.
47Pyro.config.PYRO_MULTITHREADED=0
48
49#
50# the class responsible for executing all solution server requests.
51#
52class PHSolverServer(object):
53
54   def __init__(self, scenario_instances, solver, scenario_tree):
55
56      # the obvious stuff!
57      self._instances = scenario_instances # a map from scenario name to pyomo model instance
58      self._solver = solver
59      self._scenario_tree = scenario_tree
60
61      # the objective functions are modified throughout the course of PH iterations.
62      # save the original, as a baseline to modify in subsequent iterations. reserve
63      # the original objectives, for subsequent modification.
64      self._original_objective_expression = {}
65      for instance_name, instance in self._instances.items():
66         objective_name = instance.active_components(Objective).keys()[0]
67         expr = instance.active_components(Objective)[objective_name]._data[None].expr
68         if isinstance(expr, Expression) is False:
69            expr = _IdentityExpression(expr)
70         self._original_objective_expression[instance_name] = expr
71
72      # the server is in one of two modes - solve the baseline instances, or
73      # those augmented with the PH penalty terms. default is standard.
74      # NOTE: This is currently a global flag for all scenarios handled
75      #       by this server - easy enough to extend if we want.
76      self._solving_standard_objective = True
77
78   def enable_standard_objective(self):
79
80#      print "RECEIVED REQUEST TO ENABLE STANDARD OBJECTIVE FOR ALL SCENARIOS"
81
82      self._solving_standard_objective = True
83
84   def enable_ph_objective(self):
85
86#      print "RECEIVED REQUEST TO ENABLE PH OBJECTIVE FOR ALL SCENARIOS"
87
88      self._solving_standard_objective = False
89
90   def solve(self, scenario_name):
91
92#      print "RECEIVED REQUEST TO SOLVE SCENARIO INSTANCE="+scenario_name
93#      if self._solving_standard_objective is True:
94#         print "OBJECTIVE=STANDARD"
95#      else:
96#         print "OBJECTIVE=PH"
97
98      if scenario_name not in self._instances:
99         print "***ERROR: Requested instance to solve not in PH solver server instance collection!"
100         return None
101      scenario_instance = self._instances[scenario_name]
102
103      # form the desired objective, depending on the solve mode.
104      if self._solving_standard_objective is True:
105         form_standard_objective(scenario_name, scenario_instance, self._original_objective_expression[scenario_name], self._scenario_tree)
106      else:
107         # TBD - command-line drive the various options (integer tolerance, breakpoint strategies, etc.)
108         form_ph_objective(scenario_name, scenario_instance, \
109                           self._original_objective_expression[scenario_name], self._scenario_tree, \
110                           False, False, False, 0, 0.00001)
111
112      # IMPT: You have to re-presolve, as the simple presolver collects the linear terms together. If you
113      # don't do this, you won't see any chance in the output files as you vary the problem parameters!
114      # ditto for instance fixing!
115      scenario_instance.preprocess()
116
117      results = self._solver.solve(scenario_instance)
118
119      print "Successfully solved scenario instance="+scenario_name
120#      print "RESULTS:"
121#      results.write(num=1)
122      encoded_results = pickle.dumps(results)
123
124      return encoded_results
125
126   def update_weights_and_averages(self, scenario_name, new_weights, new_averages):
127
128#      print "RECEIVED REQUEST TO UPDATE WEIGHTS AND AVERAGES FOR SCENARIO=",scenario_name
129
130      if scenario_name not in self._instances:
131         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
132         return None
133      scenario_instance = self._instances[scenario_name]
134
135      for weight_update in new_weights:
136
137         weight_index = weight_update._index
138
139         target_weight_parameter = getattr(scenario_instance, weight_update.name)
140
141         for index in weight_index:
142            target_weight_parameter[index] = value(weight_update[index])
143
144      for average_update in new_averages:
145
146         average_index = average_update._index
147
148         target_average_parameter = getattr(scenario_instance, average_update.name)
149
150         for index in average_index:
151            target_average_parameter[index] = value(average_update[index])
152
153   def update_rhos(self, scenario_name, new_rhos):
154
155#      print "RECEIVED REQUEST TO UPDATE RHOS FOR SCENARIO=",scenario_name
156
157      if scenario_name not in self._instances:
158         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
159         return None
160      scenario_instance = self._instances[scenario_name]
161
162      for rho_update in new_rhos:
163
164         rho_index = rho_update._index
165
166         target_rho_parameter = getattr(scenario_instance, rho_update.name)
167
168         for index in rho_index:
169            target_rho_parameter[index] = value(rho_update[index])
170
171   def update_tree_node_statistics(self, scenario_name, new_node_minimums, new_node_maximums):
172
173#      print "RECEIVED REQUEST TO UPDATE TREE NODE STATISTICS SCENARIO=",scenario_name
174
175      for tree_node_name, tree_node_minimums in new_node_minimums.items():
176
177         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
178         tree_node._minimums = tree_node_minimums
179
180      for tree_node_name, tree_node_maximums in new_node_maximums.items():
181
182         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
183         tree_node._maximums = tree_node_maximums
184
185
186#
187# utility method to construct an option parser for ph arguments, to be
188# supplied as an argument to the runph method.
189#
190
191def construct_options_parser(usage_string):
192
193   solver_list = SolverFactory.services()
194   solver_list = sorted( filter(lambda x: '_' != x[0], solver_list) )
195   solver_help = \
196   "Specify the solver with which to solve scenario sub-problems.  The "      \
197   "following solver types are currently supported: %s; Default: cplex"
198   solver_help %= ', '.join( solver_list )
199
200   parser = OptionParser()
201   parser.add_option("--verbose",
202                     help="Generate verbose output for both initialization and execution. Default is False.",
203                     action="store_true",
204                     dest="verbose",
205                     default=False)
206   parser.add_option("--scenario",
207                     help="Specify a scenario that this server is responsible for solving",
208                     action="append",
209                     dest="scenarios",
210                     default=[])
211   parser.add_option("--all-scenarios",
212                     help="Indicate that the server is responsible for solving all scenarios",
213                     action="store_true",
214                     dest="all_scenarios",
215                     default=False)
216   parser.add_option("--report-solutions",
217                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
218                     action="store_true",
219                     dest="report_solutions",
220                     default=False)
221   parser.add_option("--report-weights",
222                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
223                     action="store_true",
224                     dest="report_weights",
225                     default=False)
226   parser.add_option('-m',"--model-directory",
227                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
228                     action="store",
229                     dest="model_directory",
230                     type="string",
231                     default=".")
232   parser.add_option('-i',"--instance-directory",
233                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
234                     action="store",
235                     dest="instance_directory",
236                     type="string",
237                     default=".")
238   parser.add_option("--solver",
239                     help=solver_help,
240                     action="store",
241                     dest="solver_type",
242                     type="string",
243                     default="cplex")
244   parser.add_option("--scenario-solver-options",
245                     help="Solver options for all PH scenario sub-problems",
246                     action="append",
247                     dest="scenario_solver_options",
248                     type="string",
249                     default=[])
250   parser.add_option("--scenario-mipgap",
251                     help="Specifies the mipgap for all PH scenario sub-problems",
252                     action="store",
253                     dest="scenario_mipgap",
254                     type="float",
255                     default=None)
256   parser.add_option('-k',"--keep-solver-files",
257                     help="Retain temporary input and output files for scenario sub-problem solves",
258                     action="store_true",
259                     dest="keep_solver_files",
260                     default=False)
261   parser.add_option("--output-solver-logs",
262                     help="Output solver logs during scenario sub-problem solves",
263                     action="store_true",
264                     dest="output_solver_logs",
265                     default=False)
266   parser.add_option("--output-solver-results",
267                     help="Output solutions obtained after each scenario sub-problem solve",
268                     action="store_true",
269                     dest="output_solver_results",
270                     default=False)
271   parser.add_option("--output-times",
272                     help="Output timing statistics for various PH components",
273                     action="store_true",
274                     dest="output_times",
275                     default=False)
276   parser.add_option("--disable-warmstarts",
277                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
278                     action="store_true",
279                     dest="disable_warmstarts",
280                     default=False)
281   parser.add_option("--profile",
282                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
283                     action="store",
284                     dest="profile",
285                     type="int",
286                     default=0)
287   parser.add_option("--disable-gc",
288                     help="Disable the python garbage collecter. Default is False.",
289                     action="store_true",
290                     dest="disable_gc",
291                     default=False)
292
293   parser.usage=usage_string
294
295   return parser
296
297#
298# Create the reference model / instance and scenario tree instance for PH.
299#
300
301def load_reference_and_scenario_models(options):
302
303   #
304   # create and populate the reference model/instance pair.
305   #
306
307   reference_model = None
308   reference_instance = None
309
310   try:
311      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
312      if options.verbose is True:
313         print "Scenario reference model filename="+reference_model_filename
314      model_import = import_file(reference_model_filename)
315      if "model" not in dir(model_import):
316         print ""
317         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
318         return None, None, None, None
319
320      if model_import.model is None:
321         print ""
322         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
323         return None, None, None, None
324
325      reference_model = model_import.model
326   except IOError:
327      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
328      return None, None, None, None
329
330   try:
331      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
332      if options.verbose is True:
333         print "Scenario reference instance filename="+reference_instance_filename
334      reference_instance = reference_model.create(reference_instance_filename)
335   except IOError:
336      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
337      return None, None, None, None
338
339   #
340   # create and populate the scenario tree model
341   #
342
343   from coopr.pysp.util.scenariomodels import scenario_tree_model
344   scenario_tree_instance = None
345
346   try:
347      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
348      if options.verbose is True:
349         print "Scenario tree instance filename="+scenario_tree_instance_filename
350      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
351   except IOError:
352      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
353      return None, None, None
354
355   #
356   # construct the scenario tree
357   #
358   scenario_tree = ScenarioTree(scenarioinstance=reference_instance,
359                                scenariotreeinstance=scenario_tree_instance)
360
361   return reference_model, reference_instance, scenario_tree, scenario_tree_instance
362
363#
364# Execute the PH solver server daemon.
365#
366
367def run_server(options, scenario_instances, solver, scenario_tree):
368
369   start_time = time.time()
370
371   Pyro.core.initServer(banner=0)
372
373   locator = Pyro.naming.NameServerLocator()
374   ns = None
375   try:
376      ns = locator.getNS()
377   except Pyro.errors.NamingError:
378      raise RuntimeError, "PH solver server failed to locate Pyro name server"
379
380   solver_daemon = Pyro.core.Daemon()
381   solver_daemon.useNameServer(ns)
382
383   daemon_object = PHSolverServer(scenario_instances, solver, scenario_tree)
384   delegator_object = Pyro.core.ObjBase()
385   delegator_object.delegateTo(daemon_object)
386
387   # register an entry in the nameserver for
388   # each scenario processed by this daemon.
389   for scenario_name, scenario_instance in scenario_instances.items():
390      # first, see if it's already registered - if no, cull the old entry.
391      # NOTE: This is a hack, as the cleanup / disconnect code doesn't
392      #       seem to be invoked when the daemon is killed.
393      try:
394         ns.unregister(scenario_name)
395      except NamingError:
396         pass
397
398      try:
399         solver_daemon.connect(delegator_object, scenario_name)
400      except Pyro.errors.NamingError:
401         raise RuntimeError, "Entry in name server already exists for scenario="+scenario_name
402
403   try:
404      solver_daemon.requestLoop()
405   except KeyboardInterrupt, SystemExit:
406      pass
407
408   solver_daemon.disconnect(delegator_object)
409   solver_daemon.shutdown()
410
411   end_time = time.time()
412
413#
414# The main daemon initialization / runner routine.
415#
416
417def exec_server(options):
418
419   # we need the base model (not so much the reference instance, but that
420   # can't hurt too much - until proven otherwise, that is) to construct
421   # the scenarios that this server is responsible for.
422   reference_model, reference_instance, scenario_tree, scenario_tree_instance = load_reference_and_scenario_models(options)
423   if reference_model is None or reference_instance is None or scenario_tree is None:
424      print "***Unable to launch PH solver server"
425      return
426
427   # construct the set of scenario instances based on the command-line.
428   scenario_instances = {}
429   if (options.all_scenarios is True) and (len(options.scenarios) > 0):
430      print "***WARNING: Both individual scenarios and all-scenarios were specified on the command-line; proceeding using all scenarios"
431
432   if (len(options.scenarios) == 0) and (options.all_scenarios is False):
433      print "***Unable to launch PH solver server - no scenario(s) specified!"
434      return
435
436   scenarios_to_construct = []
437   if options.all_scenarios is True:
438      scenarios_to_construct.extend(scenario_tree._scenario_map.keys())
439   else:
440      scenarios_to_construct.extend(options.scenarios)
441
442   for scenario_name in scenarios_to_construct:
443      print "Creating instance for scenario="+scenario_name
444      if scenario_tree.contains_scenario(scenario_name) is False:
445         print "***Unable to launch PH solver server - unknown scenario specified with name="+scenario_name
446         return
447
448      # create the baseline scenario instance
449      scenario_instance = construct_scenario_instance(scenario_tree,
450                                                      options.instance_directory,
451                                                      scenario_name,
452                                                      reference_model,
453                                                      options.verbose)
454      if scenario_instance is None:
455         print "***Unable to launch PH solver server - failed to create instance for scenario="+scenario_name
456         return
457
458      # augment the instance with PH-specific parameters (weights, rhos, etc).
459      # TBD: The default rho of 1.0 is kind of bogus. Need to somehow propagate
460      #      this value and the linearization parameter as a command-line argument.
461      new_penalty_variable_names = create_ph_parameters(scenario_instance, scenario_tree, 1.0, False)
462
463      scenario_instances[scenario_name] = scenario_instance
464
465   # the solver instance is persistent, applicable to all instances here.
466   if options.verbose is True:
467      print "Constructing solver type="+options.solver_type
468   solver = SolverFactory(options.solver_type)
469   if solver == None:
470      raise ValueError, "Unknown solver type=" + options.solver_type + " specified"
471   if options.keep_solver_files is True:
472      solver.keepFiles = True
473   if len(options.scenario_solver_options) > 0:
474      if options.verbose is True:
475         print "Initializing scenario sub-problem solver with options="+str(options.scenario_solver_options)
476      solver.set_options("".join(options.scenario_solver_options))
477   if options.output_times is True:
478      solver._report_timing = True
479
480   # spawn the daemon.
481   run_server(options, scenario_instances, solver, scenario_tree)
482
483def run(args=None):
484
485    #
486    # Top-level command that executes the ph solver server daemon.
487    # This is segregated from phsolverserver to faciliate profiling.
488    #
489
490    #
491    # Parse command-line options.
492    #
493    try:
494       options_parser = construct_options_parser("phsolverserver [options]")
495       (options, args) = options_parser.parse_args(args=args)
496    except SystemExit:
497       # the parser throws a system exit if "-h" is specified - catch
498       # it to exit gracefully.
499       return
500
501    # for a one-pass execution, garbage collection doesn't make
502    # much sense - so it is disabled by default. Because: It drops
503    # the run-time by a factor of 3-4 on bigger instances.
504    if options.disable_gc is True:
505       gc.disable()
506    else:
507       gc.enable()
508
509    if options.profile > 0:
510        #
511        # Call the main PH routine with profiling.
512        #
513        tfile = TempfileManager.create_tempfile(suffix=".profile")
514        tmp = profile.runctx('exec_ph(options)',globals(),locals(),tfile)
515        p = pstats.Stats(tfile).strip_dirs()
516        p.sort_stats('time', 'cum')
517        p = p.print_stats(options.profile)
518        p.print_callers(options.profile)
519        p.print_callees(options.profile)
520        p = p.sort_stats('cum','calls')
521        p.print_stats(options.profile)
522        p.print_callers(options.profile)
523        p.print_callees(options.profile)
524        p = p.sort_stats('calls')
525        p.print_stats(options.profile)
526        p.print_callers(options.profile)
527        p.print_callees(options.profile)
528        TempfileManager.clear_tempfiles()
529        ans = [tmp, None]
530    else:
531        #
532        # Call the main PH routine without profiling.
533        #
534        ans = exec_server(options)
535
536    gc.enable()
537
538    return ans
539
Note: See TracBrowser for help on using the repository browser.