source: coopr.pysp/trunk/coopr/pysp/phserver.py @ 3481

Last change on this file since 3481 was 3481, checked in by khunter, 9 years ago

NFC: EOL whitespace removal

  • Property svn:executable set to *
File size: 20.5 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2010 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  Fr more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser
15
16import pyutilib.services
17import pyutilib.misc
18import textwrap
19import traceback
20import time
21
22# Coopr
23from coopr.opt.base import SolverFactory
24from coopr.pysp.scenariotree import *
25from phutils import *
26from phobjective import *
27
28# Pyro
29import Pyro.core
30import Pyro.naming
31from Pyro.errors import PyroError,NamingError
32
33# garbage collection control.
34import gc
35
36# for profiling
37try:
38    import cProfile as profile
39except ImportError:
40    import profile
41import pstats
42
43import pickle
44
45# disable multi-threading. for a solver server, this
46# would truly be a bad idea, as you (1) likely have limited
47# solver licenses and (2) likely want to dedicate all compute
48# resources on this node to a single solve.
49Pyro.config.PYRO_MULTITHREADED=0
50
51#
52# the class responsible for executing all solution server requests.
53#
54class PHSolverServer(object):
55
56   def __init__(self, scenario_instances, solver, scenario_tree):
57
58      # the obvious stuff!
59      self._instances = scenario_instances # a map from scenario name to pyomo model instance
60      self._solver = solver
61      self._scenario_tree = scenario_tree
62
63      # the objective functions are modified throughout the course of PH iterations.
64      # save the original, as a baseline to modify in subsequent iterations. reserve
65      # the original objectives, for subsequent modification.
66      self._original_objective_expression = {}
67      for instance_name, instance in self._instances.items():
68         objective_name = instance.active_components(Objective).keys()[0]
69         expr = instance.active_components(Objective)[objective_name]._data[None].expr
70         if isinstance(expr, Expression) is False:
71            expr = _IdentityExpression(expr)
72         self._original_objective_expression[instance_name] = expr
73
74      # the server is in one of two modes - solve the baseline instances, or
75      # those augmented with the PH penalty terms. default is standard.
76      # NOTE: This is currently a global flag for all scenarios handled
77      #       by this server - easy enough to extend if we want.
78      self._solving_standard_objective = True
79
80   def enable_standard_objective(self):
81
82#      print "RECEIVED REQUEST TO ENABLE STANDARD OBJECTIVE FOR ALL SCENARIOS"
83
84      self._solving_standard_objective = True
85
86   def enable_ph_objective(self):
87
88#      print "RECEIVED REQUEST TO ENABLE PH OBJECTIVE FOR ALL SCENARIOS"
89
90      self._solving_standard_objective = False
91
92   def solve(self, scenario_name):
93
94#      print "RECEIVED REQUEST TO SOLVE SCENARIO INSTANCE="+scenario_name
95#      if self._solving_standard_objective is True:
96#         print "OBJECTIVE=STANDARD"
97#      else:
98#         print "OBJECTIVE=PH"
99
100      if scenario_name not in self._instances:
101         print "***ERROR: Requested instance to solve not in PH solver server instance collection!"
102         return None
103      scenario_instance = self._instances[scenario_name]
104
105      # form the desired objective, depending on the solve mode.
106      if self._solving_standard_objective is True:
107         form_standard_objective(scenario_name, scenario_instance, self._original_objective_expression[scenario_name], self._scenario_tree)
108      else:
109         # TBD - command-line drive the various options (integer tolerance, breakpoint strategies, etc.)
110         form_ph_objective(scenario_name, scenario_instance, \
111                           self._original_objective_expression[scenario_name], self._scenario_tree, \
112                           False, False, False, 0, 0.00001)
113
114      # IMPT: You have to re-presolve, as the simple presolver collects the linear terms together. If you
115      # don't do this, you won't see any chance in the output files as you vary the problem parameters!
116      # ditto for instance fixing!
117      scenario_instance.preprocess()
118
119      results = self._solver.solve(scenario_instance)
120
121      print "Successfully solved scenario instance="+scenario_name
122#      print "RESULTS:"
123#      results.write(num=1)
124      encoded_results = pickle.dumps(results)
125
126      return encoded_results
127
128   def update_weights_and_averages(self, scenario_name, new_weights, new_averages):
129
130#      print "RECEIVED REQUEST TO UPDATE WEIGHTS AND AVERAGES FOR SCENARIO=",scenario_name
131
132      if scenario_name not in self._instances:
133         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
134         return None
135      scenario_instance = self._instances[scenario_name]
136
137      for weight_update in new_weights:
138
139         weight_index = weight_update._index
140
141         target_weight_parameter = getattr(scenario_instance, weight_update.name)
142
143         for index in weight_index:
144            target_weight_parameter[index] = value(weight_update[index])
145
146      for average_update in new_averages:
147
148         average_index = average_update._index
149
150         target_average_parameter = getattr(scenario_instance, average_update.name)
151
152         for index in average_index:
153            target_average_parameter[index] = value(average_update[index])
154
155   def update_rhos(self, scenario_name, new_rhos):
156
157#      print "RECEIVED REQUEST TO UPDATE RHOS FOR SCENARIO=",scenario_name
158
159      if scenario_name not in self._instances:
160         print "***ERROR: Received request to update weights for instance not in PH solver server instance collection!"
161         return None
162      scenario_instance = self._instances[scenario_name]
163
164      for rho_update in new_rhos:
165
166         rho_index = rho_update._index
167
168         target_rho_parameter = getattr(scenario_instance, rho_update.name)
169
170         for index in rho_index:
171            target_rho_parameter[index] = value(rho_update[index])
172
173   def update_tree_node_statistics(self, scenario_name, new_node_minimums, new_node_maximums):
174
175#      print "RECEIVED REQUEST TO UPDATE TREE NODE STATISTICS SCENARIO=",scenario_name
176
177      for tree_node_name, tree_node_minimums in new_node_minimums.items():
178
179         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
180         tree_node._minimums = tree_node_minimums
181
182      for tree_node_name, tree_node_maximums in new_node_maximums.items():
183
184         tree_node = self._scenario_tree._tree_node_map[tree_node_name]
185         tree_node._maximums = tree_node_maximums
186
187
188#
189# utility method to construct an option parser for ph arguments, to be
190# supplied as an argument to the runph method.
191#
192
193def construct_options_parser(usage_string):
194
195   parser = OptionParser()
196   parser.add_option("--verbose",
197                     help="Generate verbose output for both initialization and execution. Default is False.",
198                     action="store_true",
199                     dest="verbose",
200                     default=False)
201   parser.add_option("--scenario",
202                     help="Specify a scenario that this server is responsible for solving",
203                     action="append",
204                     dest="scenarios",
205                     default=[])
206   parser.add_option("--all-scenarios",
207                     help="Indicate that the server is responsible for solving all scenarios",
208                     action="store_true",
209                     dest="all_scenarios",
210                     default=False)
211   parser.add_option("--report-solutions",
212                     help="Always report PH solutions after each iteration. Enabled if --verbose is enabled. Default is False.",
213                     action="store_true",
214                     dest="report_solutions",
215                     default=False)
216   parser.add_option("--report-weights",
217                     help="Always report PH weights prior to each iteration. Enabled if --verbose is enabled. Default is False.",
218                     action="store_true",
219                     dest="report_weights",
220                     default=False)
221   parser.add_option("--model-directory",
222                     help="The directory in which all model (reference and scenario) definitions are stored. Default is \".\".",
223                     action="store",
224                     dest="model_directory",
225                     type="string",
226                     default=".")
227   parser.add_option("--instance-directory",
228                     help="The directory in which all instance (reference and scenario) definitions are stored. Default is \".\".",
229                     action="store",
230                     dest="instance_directory",
231                     type="string",
232                     default=".")
233   parser.add_option("--solver",
234                     help="The type of solver used to solve scenario sub-problems. Default is cplex.",
235                     action="store",
236                     dest="solver_type",
237                     type="string",
238                     default="cplex")
239   parser.add_option("--scenario-solver-options",
240                     help="Solver options for all PH scenario sub-problems",
241                     action="append",
242                     dest="scenario_solver_options",
243                     type="string",
244                     default=[])
245   parser.add_option("--scenario-mipgap",
246                     help="Specifies the mipgap for all PH scenario sub-problems",
247                     action="store",
248                     dest="scenario_mipgap",
249                     type="float",
250                     default=None)
251   parser.add_option("--keep-solver-files",
252                     help="Retain temporary input and output files for scenario sub-problem solves",
253                     action="store_true",
254                     dest="keep_solver_files",
255                     default=False)
256   parser.add_option("--output-solver-logs",
257                     help="Output solver logs during scenario sub-problem solves",
258                     action="store_true",
259                     dest="output_solver_logs",
260                     default=False)
261   parser.add_option("--output-solver-results",
262                     help="Output solutions obtained after each scenario sub-problem solve",
263                     action="store_true",
264                     dest="output_solver_results",
265                     default=False)
266   parser.add_option("--output-times",
267                     help="Output timing statistics for various PH components",
268                     action="store_true",
269                     dest="output_times",
270                     default=False)
271   parser.add_option("--disable-warmstarts",
272                     help="Disable warm-start of scenario sub-problem solves in PH iterations >= 1. Default is False.",
273                     action="store_true",
274                     dest="disable_warmstarts",
275                     default=False)
276   parser.add_option("--profile",
277                     help="Enable profiling of Python code.  The value of this option is the number of functions that are summarized.",
278                     action="store",
279                     dest="profile",
280                     type="int",
281                     default=0)
282   parser.add_option("--disable-gc",
283                     help="Disable the python garbage collecter. Default is False.",
284                     action="store_true",
285                     dest="disable_gc",
286                     default=False)
287
288   parser.usage=usage_string
289
290   return parser
291
292#
293# Create the reference model / instance and scenario tree instance for PH.
294#
295
296def load_reference_and_scenario_models(options):
297
298   #
299   # create and populate the reference model/instance pair.
300   #
301
302   reference_model = None
303   reference_instance = None
304
305   try:
306      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
307      if options.verbose is True:
308         print "Scenario reference model filename="+reference_model_filename
309      model_import = pyutilib.misc.import_file(reference_model_filename)
310      if "model" not in dir(model_import):
311         print ""
312         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
313         return None, None, None, None
314
315      if model_import.model is None:
316         print ""
317         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
318         return None, None, None, None
319
320      reference_model = model_import.model
321   except IOError:
322      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
323      return None, None, None, None
324
325   try:
326      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
327      if options.verbose is True:
328         print "Scenario reference instance filename="+reference_instance_filename
329      reference_instance = reference_model.create(reference_instance_filename)
330   except IOError:
331      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
332      return None, None, None, None
333
334   #
335   # create and populate the scenario tree model
336   #
337
338   from coopr.pysp.util.scenariomodels import scenario_tree_model
339   scenario_tree_instance = None
340
341   try:
342      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
343      if options.verbose is True:
344         print "Scenario tree instance filename="+scenario_tree_instance_filename
345      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
346   except IOError:
347      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
348      return None, None, None
349
350   #
351   # construct the scenario tree
352   #
353   scenario_tree = ScenarioTree(scenarioinstance=reference_instance,
354                                scenariotreeinstance=scenario_tree_instance)
355
356   return reference_model, reference_instance, scenario_tree, scenario_tree_instance
357
358#
359# Execute the PH solver server daemon.
360#
361
362def run_server(options, scenario_instances, solver, scenario_tree):
363
364   start_time = time.time()
365
366   Pyro.core.initServer(banner=0)
367
368   locator = Pyro.naming.NameServerLocator()
369   ns = None
370   try:
371      ns = locator.getNS()
372   except Pyro.errors.NamingError:
373      raise RuntimeError, "PH solver server failed to locate Pyro name server"
374
375   solver_daemon = Pyro.core.Daemon()
376   solver_daemon.useNameServer(ns)
377
378   daemon_object = PHSolverServer(scenario_instances, solver, scenario_tree)
379   delegator_object = Pyro.core.ObjBase()
380   delegator_object.delegateTo(daemon_object)
381
382   # register an entry in the nameserver for
383   # each scenario processed by this daemon.
384   for scenario_name, scenario_instance in scenario_instances.items():
385      # first, see if it's already registered - if no, cull the old entry.
386      # NOTE: This is a hack, as the cleanup / disconnect code doesn't
387      #       seem to be invoked when the daemon is killed.
388      try:
389         ns.unregister(scenario_name)
390      except NamingError:
391         pass
392
393      try:
394         solver_daemon.connect(delegator_object, scenario_name)
395      except Pyro.errors.NamingError:
396         raise RuntimeError, "Entry in name server already exists for scenario="+scenario_name
397
398   try:
399      solver_daemon.requestLoop()
400   except KeyboardInterrupt, SystemExit:
401      pass
402
403   solver_daemon.disconnect(delegator_object)
404   solver_daemon.shutdown()
405
406   end_time = time.time()
407
408#
409# The main daemon initialization / runner routine.
410#
411
412def exec_server(options):
413
414   # we need the base model (not so much the reference instance, but that
415   # can't hurt too much - until proven otherwise, that is) to construct
416   # the scenarios that this server is responsible for.
417   reference_model, reference_instance, scenario_tree, scenario_tree_instance = load_reference_and_scenario_models(options)
418   if reference_model is None or reference_instance is None or scenario_tree is None:
419      print "***Unable to launch PH solver server"
420      return
421
422   # construct the set of scenario instances based on the command-line.
423   scenario_instances = {}
424   if (options.all_scenarios is True) and (len(options.scenarios) > 0):
425      print "***WARNING: Both individual scenarios and all-scenarios were specified on the command-line; proceeding using all scenarios"
426
427   if (len(options.scenarios) == 0) and (options.all_scenarios is False):
428      print "***Unable to launch PH solver server - no scenario(s) specified!"
429      return
430
431   scenarios_to_construct = []
432   if options.all_scenarios is True:
433      scenarios_to_construct.extend(scenario_tree._scenario_map.keys())
434   else:
435      scenarios_to_construct.extend(options.scenarios)
436
437   for scenario_name in scenarios_to_construct:
438      print "Creating instance for scenario="+scenario_name
439      if scenario_tree.contains_scenario(scenario_name) is False:
440         print "***Unable to launch PH solver server - unknown scenario specified with name="+scenario_name
441         return
442
443      # create the baseline scenario instance
444      scenario_instance = construct_scenario_instance(scenario_tree,
445                                                      options.instance_directory,
446                                                      scenario_name,
447                                                      reference_model,
448                                                      options.verbose)
449      if scenario_instance is None:
450         print "***Unable to launch PH solver server - failed to create instance for scenario="+scenario_name
451         return
452
453      # augment the instance with PH-specific parameters (weights, rhos, etc).
454      # TBD: The default rho of 1.0 is kind of bogus. Need to somehow propagate
455      #      this value and the linearization parameter as a command-line argument.
456      new_penalty_variable_names = create_ph_parameters(scenario_instance, scenario_tree, 1.0, False)
457
458      scenario_instances[scenario_name] = scenario_instance
459
460   # the solver instance is persistent, applicable to all instances here.
461   if options.verbose is True:
462      print "Constructing solver type="+options.solver_type
463   solver = SolverFactory(options.solver_type)
464   if solver == None:
465      raise ValueError, "Unknown solver type=" + options.solver_type + " specified"
466   if options.keep_solver_files is True:
467      solver.keepFiles = True
468   if len(options.scenario_solver_options) > 0:
469      if options.verbose is True:
470         print "Initializing scenario sub-problem solver with options="+str(options.scenario_solver_options)
471      solver.set_options("".join(options.scenario_solver_options))
472   if options.output_times is True:
473      solver._report_timing = True
474
475   # spawn the daemon.
476   run_server(options, scenario_instances, solver, scenario_tree)
477
478def run(args=None):
479
480    #
481    # Top-level command that executes the ph solver server daemon.
482    # This is segregated from phsolverserver to faciliate profiling.
483    #
484
485    #
486    # Parse command-line options.
487    #
488    try:
489       options_parser = construct_options_parser("phsolverserver [options]")
490       (options, args) = options_parser.parse_args(args=args)
491    except SystemExit:
492       # the parser throws a system exit if "-h" is specified - catch
493       # it to exit gracefully.
494       return
495
496    # for a one-pass execution, garbage collection doesn't make
497    # much sense - so it is disabled by default. Because: It drops
498    # the run-time by a factor of 3-4 on bigger instances.
499    if options.disable_gc is True:
500       gc.disable()
501    else:
502       gc.enable()
503
504    if options.profile > 0:
505        #
506        # Call the main PH routine with profiling.
507        #
508        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
509        tmp = profile.runctx('exec_ph(options)',globals(),locals(),tfile)
510        p = pstats.Stats(tfile).strip_dirs()
511        p.sort_stats('time', 'cum')
512        p = p.print_stats(options.profile)
513        p.print_callers(options.profile)
514        p.print_callees(options.profile)
515        p = p.sort_stats('cum','calls')
516        p.print_stats(options.profile)
517        p.print_callers(options.profile)
518        p.print_callees(options.profile)
519        p = p.sort_stats('calls')
520        p.print_stats(options.profile)
521        p.print_callers(options.profile)
522        p.print_callees(options.profile)
523        pyutilib.services.TempfileManager.clear_tempfiles()
524        ans = [tmp, None]
525    else:
526        #
527        # Call the main PH routine without profiling.
528        #
529        ans = exec_server(options)
530
531    gc.enable()
532
533    return ans
534
Note: See TracBrowser for help on using the repository browser.