source: coopr.pysp/stable/coopr/pysp/ef_writer_script.py @ 3185

Last change on this file since 3185 was 3185, checked in by wehart, 10 years ago

Merged revisions 3115-3184 via svnmerge from
https://software.sandia.gov/svn/public/coopr/coopr.pysp/trunk

........

r3120 | jwatson | 2010-10-18 21:11:21 -0600 (Mon, 18 Oct 2010) | 3 lines


Adding PySP options for linearizing expressions.

........

r3134 | jwatson | 2010-10-21 15:45:49 -0600 (Thu, 21 Oct 2010) | 3 lines


Bug fix associated with linear expressions.

........

r3137 | khunter | 2010-10-22 10:19:44 -0600 (Fri, 22 Oct 2010) | 2 lines


Add name as contributor.

........

r3138 | jwatson | 2010-10-22 14:11:43 -0600 (Fri, 22 Oct 2010) | 3 lines


Added "--simplify-expressions" option to runph, in order to eliminate the memory and time costs associated with simplifying expressions (e.g., in formulation of the PH objective) for which simpification is very unlikely to help. By default, expression simplification is disabled. This may cause issues with certain writers, e.g., NL - which is why I have retained the option.

........

r3139 | jwatson | 2010-10-22 15:20:49 -0600 (Fri, 22 Oct 2010) | 3 lines


When forming the PH linear terms for all variables and the proximal terms for binary variables, use value() to immediately extract the fixed value of the underlying parameters. This speeds up the associated expression tree construction time, which was non-trivial (5% of the total run-time, for example). Because we re-form the expressions each iteration, this is OK.

........

r3143 | jwatson | 2010-10-22 22:07:07 -0600 (Fri, 22 Oct 2010) | 3 lines


Modifying more PH parameters to be of the "nochecking" variety. Also slightly improved the performance of the update_variable_statistics PH function.

........

r3145 | jwatson | 2010-10-22 22:21:06 -0600 (Fri, 22 Oct 2010) | 3 lines


Update of PySP baseline output files, due to small numerical differences (4th significant digit) associated with some code mods I recently performed in the course of optimization.

........

r3146 | jwatson | 2010-10-23 13:27:38 -0600 (Sat, 23 Oct 2010) | 3 lines


Speed improvements in weight update.

........

r3147 | jwatson | 2010-10-23 13:38:52 -0600 (Sat, 23 Oct 2010) | 3 lines


More speed improvements to PH weight computation procedure.

........

r3152 | jwatson | 2010-10-24 13:20:03 -0600 (Sun, 24 Oct 2010) | 3 lines


Further code optimizations to PH weight and statistic update routines.

........

r3155 | jwatson | 2010-10-24 22:15:29 -0600 (Sun, 24 Oct 2010) | 3 lines


Added --linearize-expression and associated processing to runef PySP script.

........

r3156 | jwatson | 2010-10-24 22:19:44 -0600 (Sun, 24 Oct 2010) | 3 lines


Correcting shift from "integer" to "general" variable blocks when writing the LP file for an extensive form.

........

r3165 | jwatson | 2010-10-26 18:46:24 -0600 (Tue, 26 Oct 2010) | 3 lines


Update of PySP CHANGELOG for 2.4 release.

........

r3168 | jwatson | 2010-10-26 20:12:23 -0600 (Tue, 26 Oct 2010) | 3 lines


Completing implementation of optional processing of expression simplification in PySP.

........

  • Property svn:executable set to *
File size: 12.2 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2009 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import sys
13import os
14from optparse import OptionParser, OptionGroup
15
16import pyutilib.services
17import textwrap
18import traceback
19try:
20    import cProfile as profile
21except ImportError:
22    import profile
23import pstats
24import gc
25
26from coopr.pysp.ef import *
27
28from coopr.opt.base import SolverFactory
29from coopr.opt.parallel import SolverManagerFactory
30
31from pyutilib.component.core import ExtensionPoint
32from coopr.pysp.solutionwriter import ISolutionWriterExtension
33
34#
35# utility method to construct an option parser for ef writer arguments
36#
37
38def construct_ef_writer_options_parser(usage_string):
39
40   parser = OptionParser()
41   parser.usage=usage_string
42
43   inputOpts  = OptionGroup( parser, 'Input Options' )
44   efOpts     = OptionGroup( parser, 'EF Options' )
45   solverOpts = OptionGroup( parser, 'Solver Options' )
46   outputOpts = OptionGroup( parser, 'Output Options' )
47   otherOpts  = OptionGroup( parser, 'Other Options' )
48   parser.add_option_group( inputOpts )
49   parser.add_option_group( efOpts )
50   parser.add_option_group( solverOpts )
51   parser.add_option_group( outputOpts )
52   parser.add_option_group( otherOpts )
53
54   inputOpts.add_option('-i','--instance-directory',
55     help='The directory in which all instance (reference and scenario) definitions are stored. Default is ".".',
56     action='store',
57     dest='instance_directory',
58     type='string',
59     default='.')
60   inputOpts.add_option('-m','--model-directory',
61     help='The directory in which all model (reference and scenario) definitions are stored. Default is ".".',
62     action='store',
63     dest='model_directory',
64     type='string',
65     default='.')
66
67   efOpts.add_option('--cvar-weight',
68     help='The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.',
69     action='store',
70     dest='cvar_weight',
71     type='float',
72     default=1.0)
73   efOpts.add_option('--generate-weighted-cvar',
74     help='Add a weighted CVaR term to the primary objective',
75     action='store_true',
76     dest='generate_weighted_cvar',
77     default=False)
78   efOpts.add_option('--risk-alpha',
79     help='The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.',
80     action='store',
81     dest='risk_alpha',
82     type='float',
83     default=0.95)
84   efOpts.add_option("--linearize-expressions",
85     help="EXPERIMENTAL: An option intended for use on linear or mixed-integer models " \
86          "in which expression trees in a model (constraints or objectives) are compacted " \
87          "into a more memory-efficient and concise form. The trees themselves are eliminated. ",
88     action="store_true",
89     dest="linearize_expressions",
90     default=False)
91
92   solverOpts.add_option('--mipgap',
93     help='Specifies the mipgap for the EF solve.',
94     action='store',
95     dest='mipgap',
96     type='float',
97     default=None)
98   solverOpts.add_option('--solve',
99     help='Following write of the extensive form model, solve it.',
100     action='store_true',
101     dest='solve_ef',
102     default=False)
103   solverOpts.add_option('--solver',
104     help='The type of solver used to solve scenario sub-problems. Default is cplex.',
105     action='store',
106     dest='solver_type',
107     type='string',
108     default='cplex')
109   solverOpts.add_option('--solver-manager',
110     help='The type of solver manager used to coordinate scenario sub-problem solves. Default is serial.',
111     action='store',
112     dest='solver_manager_type',
113     type='string',
114     default='serial')
115   solverOpts.add_option('--solver-options',
116     help='Solver options for the extension form problem.',
117     action='append',
118     dest='solver_options',
119     type='string',
120     default=[])
121
122   outputOpts.add_option('--output-file',
123     help='Specify the name of the extensive form output file',
124     action='store',
125     dest='output_file',
126     type='string',
127     default='efout.lp')
128   outputOpts.add_option('--output-solver-log',
129     help='Output solver log during the extensive form solve.',
130     action='store_true',
131     dest='output_solver_log',
132     default=False)
133   outputOpts.add_option('--solution-writer',
134     help='The plugin invoked to write the scenario tree solution. Defaults to the empty list.',
135     action='append',
136     dest='solution_writer',
137     type='string',
138     default = [])
139   outputOpts.add_option('--verbose',
140     help='Generate verbose output, beyond the usual status output. Default is False.',
141     action='store_true',
142     dest='verbose',
143     default=False)
144
145   otherOpts.add_option('--disable-gc',
146     help='Disable the python garbage collecter. Default is False.',
147     action='store_true',
148     dest='disable_gc',
149     default=False)
150   otherOpts.add_option('--keep-solver-files',
151     help='Retain temporary input and output files for solve.',
152     action='store_true',
153     dest='keep_solver_files',
154     default=False)
155   otherOpts.add_option('--profile',
156     help='Enable profiling of Python code.  The value of this option is the number of functions that are summarized.',
157     action='store',
158     dest='profile',
159     default=0)
160   otherOpts.add_option('--traceback',
161     help='When an exception is thrown, show the entire call stack. Ignored if profiling is enabled. Default is False.',
162     action='store_true',
163     dest='traceback',
164     default=False)
165
166   return parser
167
168
169def run_ef_writer(options, args):
170
171   # if the user enabled the addition of the weighted cvar term to the objective,
172   # then validate the associated parameters.
173   generate_weighted_cvar = False
174   cvar_weight = None
175   risk_alpha = None
176
177   if options.generate_weighted_cvar is True:
178
179      generate_weighted_cvar = True
180      cvar_weight = options.cvar_weight
181      risk_alpha = options.risk_alpha
182
183   # validate the solution writer plugin exists, to avoid a lot of wasted work.
184   for solution_writer_name in options.solution_writer:
185      print "Trying to import solution writer="+solution_writer_name
186      __import__(solution_writer_name)
187      print "Module successfully loaded"
188
189   scenario_tree, binding_instance, scenario_instances = write_ef_from_scratch(os.path.expanduser(options.model_directory),
190                                                                               os.path.expanduser(options.instance_directory),
191                                                                               os.path.expanduser(options.output_file),
192                                                                               options.verbose,
193                                                                               options.linearize_expressions,
194                                                                               generate_weighted_cvar, cvar_weight, risk_alpha)
195
196   if (scenario_tree is None) or (binding_instance is None) or (scenario_instances is None):
197      raise RuntimeError, "Failed to write extensive form."
198
199   if options.solve_ef is True:
200
201      ef_solver = SolverFactory(options.solver_type)
202      if ef_solver is None:
203         raise ValueError, "Failed to create solver of type="+options.solver_type+" for use in extensive form solve"
204      if len(options.solver_options) > 0:
205         print "Initializing ef solver with options="+str(options.solver_options)
206         ef_solver.set_options("".join(options.solver_options))
207      if options.mipgap is not None:
208         if (options.mipgap < 0.0) or (options.mipgap > 1.0):
209            raise ValueError, "Value of the mipgap parameter for the EF solve must be on the unit interval; value specified=" + `options.mipgap`
210         else:
211            ef_solver.mipgap = options.mipgap
212      if options.keep_solver_files is True:
213         ef_solver.keepFiles = True
214
215      ef_solver_manager = SolverManagerFactory(options.solver_manager_type)
216      if ef_solver is None:
217         raise ValueError, "Failed to create solver manager of type="+options.solver_type+" for use in extensive form solve"
218
219      # at this point you have a specific solver - communicate solver capabilities
220      # to the writer via the instance.
221      binding_instance.has_capability = ef_solver.has_capability
222      for scenario_name, scenario_instance in scenario_instances.items():
223         scenario_instance.has_capability = ef_solver.has_capability
224
225      print "Queuing extensive form solve"
226      ef_action_handle = ef_solver_manager.queue(os.path.expanduser(options.output_file), opt=ef_solver, tee=options.output_solver_log)
227      print "Waiting for extensive form solve"
228      ef_results = ef_solver_manager.wait_for(ef_action_handle)
229      load_ef_solution(ef_results, binding_instance, scenario_instances)
230      scenario_tree.snapshotSolutionFromInstances(scenario_instances)
231
232      # handle output of solution from the scenario tree.
233      print ""
234      print "Extensive form solution:"
235      scenario_tree.pprintSolution()
236      print ""
237      print "Extensive form costs:"
238      scenario_tree.pprintCosts(scenario_instances)
239
240      solution_writer_plugins = ExtensionPoint(ISolutionWriterExtension)
241      for plugin in solution_writer_plugins:
242         plugin.write(scenario_tree, "ef")
243
244def run(args=None):
245
246    #
247    # Top-level command that executes the extensive form writer.
248    # This is segregated from run_ef_writer to enable profiling.
249    #
250
251    #
252    # Parse command-line options.
253    #
254    try:
255       options_parser = construct_ef_writer_options_parser("runef [options]")
256       (options, args) = options_parser.parse_args(args=args)
257    except SystemExit:
258       # the parser throws a system exit if "-h" is specified - catch
259       # it to exit gracefully.
260       return
261
262    if options.disable_gc is True:
263       gc.disable()
264    else:
265       gc.enable()
266
267    # if an exception is triggered and traceback is enabled, 'ans' won't
268    # have a value and the return statement from this function will flag
269    # an error, masking the stack trace that you really want to see.
270    ans = None
271
272    if options.profile > 0:
273        #
274        # Call the main ef writer with profiling.
275        #
276        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
277        tmp = profile.runctx('run_ef_writer(options,args)',globals(),locals(),tfile)
278        p = pstats.Stats(tfile).strip_dirs()
279        p.sort_stats('time', 'cum')
280        options.profile = eval(options.profile)
281        p = p.print_stats(options.profile)
282        p.print_callers(options.profile)
283        p.print_callees(options.profile)
284        p = p.sort_stats('cum','calls')
285        p.print_stats(options.profile)
286        p.print_callers(options.profile)
287        p.print_callees(options.profile)
288        p = p.sort_stats('calls')
289        p.print_stats(options.profile)
290        p.print_callers(options.profile)
291        p.print_callees(options.profile)
292        pyutilib.services.TempfileManager.clear_tempfiles()
293        ans = [tmp, None]
294    else:
295        #
296        # Call the main EF writer without profiling.
297        #
298        if options.traceback is True:
299           ans = run_ef_writer(options, args)
300        else:
301           errmsg = None
302           try:
303              ans = run_ef_writer(options, args)
304           except ValueError, err:
305              errmsg = 'VALUE ERROR: %s' % err
306           except TypeError, err:
307              errmsg = 'TYPE ERROR: %s' % err
308           except NameError, err:
309              errmsg = 'NAME ERROR: %s' % err
310           except IOError, err:
311              errmsg = 'I/O ERROR: %s' % err
312           except RuntimeError, err:
313              errmsg = 'RUN-TIME ERROR: %s' % err
314           except pyutilib.common.ApplicationError, err:
315              errmsg = 'APPLICATION ERROR: %s' % err
316           except Exception, err:
317              errmsg = 'UNKNOWN ERROR: %s' % err
318              traceback.print_exc()
319
320           if errmsg is not None:
321              print >>sys.stderr, errmsg
322
323    gc.enable()
324
325    return ans
326
Note: See TracBrowser for help on using the repository browser.