source: coopr.pyomo/trunk/coopr/pyomo/scripting/util.py @ 3679

Last change on this file since 3679 was 3679, checked in by wehart, 9 years ago

A major rework of command-line options that are supported for the
pyomo command. This now uses the argparse package, which is supported in
future Python releases.

File size: 22.3 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11import gc
12try:
13    import pyutilib.logging as logging
14except ImportError:
15    import logging
16import os
17import sys
18import textwrap
19import traceback
20import types
21
22try:
23    import cProfile as profile
24except ImportError:
25    import profile
26try:
27    import pstats
28    pstats_available=True
29except ImportError:
30    pstats_available=False
31
32from coopr.pyomo import *
33from coopr.opt import ProblemFormat
34from coopr.opt.base import SolverFactory
35from coopr.opt.parallel import SolverManagerFactory
36import pyutilib.misc
37from pyutilib.component.core import ExtensionPoint, Plugin, implements
38from pyutilib.services import TempfileManager
39
40from coopr.pyomo.expr.linear_repn import linearize_model_expressions
41
42filter_excepthook=False
43modelapi = {    'pyomo_create_model':IPyomoScriptCreateModel,
44                'pyomo_create_modeldata':IPyomoScriptCreateModelData,
45                'pyomo_print_model':IPyomoScriptPrintModel,
46                'pyomo_modify_instance':IPyomoScriptModifyInstance,
47                'pyomo_print_instance':IPyomoScriptPrintInstance,
48                'pyomo_save_instance':IPyomoScriptSaveInstance,
49                'pyomo_print_results':IPyomoScriptPrintResults,
50                'pyomo_save_results':IPyomoScriptSaveResults,
51                'pyomo_postprocess':IPyomoScriptPostprocess}
52
53
54logger = logging.getLogger('coopr.pyomo')
55
56#
57# Print information about modeling components supported by Pyomo
58#
59def print_components(options):
60    print ""
61    print "----------------------------------------------------------------"
62    print "Pyomo Model Components:"
63    print "----------------------------------------------------------------"
64    components = pyomo.model_components()
65    index = pyutilib.misc.sort_index(components)
66    for i in index:
67        print ""
68        print " "+components[i][0]
69        for line in textwrap.wrap(components[i][1], 59):
70            print "    "+line
71    print ""
72    print "----------------------------------------------------------------"
73    print "Pyomo Virtual Sets:"
74    print "----------------------------------------------------------------"
75    pyomo_sets = pyomo.predefined_sets()
76    index = pyutilib.misc.sort_index(pyomo_sets)
77    for i in index:
78        print ""
79        print " "+pyomo_sets[i][0]
80        print "    "+pyomo_sets[i][1]
81
82#
83# Print information about the solvers that are available
84#
85def print_solver_help(options):
86    print "The following solvers are are currently supported:"
87    solver_list = SolverFactory.services()
88    solver_list = sorted( filter(lambda x: '_' != x[0], solver_list) )
89    n = max(map(len, solver_list))
90    wrapper = textwrap.TextWrapper(subsequent_indent=' '*(n+9))
91    for s in solver_list:
92        format = '    %-'+str(n)+'s  %s'
93        print wrapper.fill(format % (s , SolverFactory.doc(s)))
94    print 'The default solver is glpk.'
95    print ""
96    wrapper = textwrap.TextWrapper(replace_whitespace=False)
97    print wrapper.fill('Subsolver options can be specified by with the solver name followed by colon and then the subsolver.  For example, the following specifies that the asl solver will be used:')
98    print '   --asl:PICO'
99    print wrapper.fill('This indicates that the asl solver will launch the PICO executable to perform optimization. Currently, no other solver supports this syntax.')
100
101
102#
103# Setup Pyomo execution environment
104#
105def setup_environment(options):
106    #
107    # Disable garbage collection
108    #
109    if options.disable_gc:
110        gc.disable()
111    #
112    # Setup management for temporary files
113    #
114    if not options.tempdir is None:
115        if not os.path.exists(options.tempdir):
116            msg =  'Directory for temporary files does not exist: %s'
117            raise ValueError, msg % options.tempdir
118        TempfileManager.tempdir = options.tempdir
119
120    #
121    # Configure exception management
122    #
123    def pyomo_excepthook(etype,value,tb):
124        """
125        This exception hook gets called when debugging is on. Otherwise,
126        run_command in this module is called.
127        """
128        global filter_excepthook
129        if len(options.model_file) > 0:
130            name = "model " + options.model_file
131        else:
132            name = "model"
133
134
135        if filter_excepthook:
136            action = "loading"
137        else:
138            action = "running"
139
140        msg = "Unexpected exception while %s %s\n" % (action, name)
141
142        #
143        # This handles the case where the error is propagated by a KeyError.
144        # KeyError likes to pass raw strings that don't handle newlines
145        # (they translate "\n" to "\\n"), as well as tacking on single
146        # quotes at either end of the error message. This undoes all that.
147        #
148        if etype == KeyError:
149            valueStr = str(value).replace("\\n","\n")[1:-1]
150        else:
151            valueStr = str(value)
152
153        logger.error(msg+valueStr)
154       
155        tb_list = traceback.extract_tb(tb,None)
156        i = 0
157        if not logger.isEnabledFor(logging.DEBUG) and filter_excepthook:
158            while i < len(tb_list):
159                #print "Y",model,tb_list[i][0]
160                if options.model_file in tb_list[i][0]:
161                    break
162                i += 1
163            if i == len(tb_list):
164                i = 0
165        print "\nTraceback (most recent call last):"
166        for item in tb_list[i:]:
167            print "  File \""+item[0]+"\", line "+str(item[1])+", in "+item[2]
168            if item[3] is not None:
169                print "    "+item[3]
170        sys.exit(1)
171    sys.excepthook = pyomo_excepthook
172    return True
173
174
175#
176# Execute preprocessing files
177#
178def apply_preprocessing(options, parser):
179    global filter_excepthook
180    #
181    #
182    # Setup solver and model
183    #
184    #
185    if len(options.model_file) == 0:
186        parser.print_help()
187        return False
188    #
189    if not options.preprocess is None:
190        for file in options.preprocess:
191            preprocess = pyutilib.misc.import_file(file)
192    #
193    for ep in ExtensionPoint(IPyomoScriptPreprocess):
194        ep.apply( options=options )
195    #
196    filter_excepthook=True
197    options.usermodel = pyutilib.misc.import_file(options.model_file)
198    filter_excepthook=False
199
200    usermodel_dir = dir(options.usermodel)
201    options._usermodel_plugins = []
202    for key in modelapi:
203        if key in usermodel_dir:
204            class TMP(Plugin):
205                implements(modelapi[key])
206                def __init__(self):
207                    Plugin.__init__(self)
208                    self.fn = getattr(options.usermodel, key)
209                def apply(self,**kwds):
210                    return self.fn(**kwds)
211            options._usermodel_plugins.append( TMP() )
212
213    if 'pyomo_preprocess' in usermodel_dir:
214        if options.model_name in usermodel_dir:
215            msg = "Preprocessing function 'pyomo_preprocess' defined in file" \
216                  " '%s', but model is already constructed!"
217            raise SystemExit, msg % options.model_file
218        getattr(options.usermodel, 'pyomo_preprocess')( options=options )
219
220    return True
221
222#
223# Create instance of Pyomo model
224#
225# TODO: extend this API to make it more generic:
226#       create_model(options, model_file=None, data_files=None, modeldata=None)
227#
228def create_model(options):
229    #
230    # Verify that files exist
231    #
232    for file in [options.model_file]+options.data_files:
233        if not os.path.exists(file):
234            raise IOError, "File "+file+" does not exist!"
235    #
236    # Create Model
237    #
238    ep = ExtensionPoint(IPyomoScriptCreateModel)
239    model_name = 'model'
240    if options.model_name is not None: model_name = options.model_name
241
242    if model_name in dir(options.usermodel):
243        if len(ep) > 0:
244            msg = "Model construction function 'create_model' defined in "    \
245                  "file '%s', but model is already constructed!"
246            raise SystemExit, msg % options.model_file
247        model = getattr(options.usermodel, model_name)
248
249        if model is None:
250            msg = "'%s' object is 'None' in module %s"
251            raise SystemExit, msg % (model_name, options.model_file)
252            sys.exit(0)
253
254    else:
255       if len(ep) == 0:
256           msg = "Neither '%s' nor 'pyomo_create_model' are available in "    \
257                 'module %s'
258           raise SystemExit, msg % ( model_name, options.model_file )
259       elif len(ep) > 1:
260           msg = 'Multiple model construction plugins have been registered!'
261           raise SystemExit, msg
262       else:
263            model_options = options.model_options
264            if model_options is None:
265                model_options = []
266            model = ep.service().apply( options=pyutilib.misc.Container(*model_options) )
267    #
268    for ep in ExtensionPoint(IPyomoScriptPrintModel):
269        ep.apply( options=options, model=model )
270
271    #
272    # Disable canonical repn for ASL solvers, and if the user has specified as such (in which case, we assume they know what they are doing!).
273    #
274    # Likely we need to change the framework so that canonical repn
275    # is not assumed to be required by all solvers?
276    #
277    if not options.solver is None and options.solver.startswith('asl'):
278        model.skip_canonical_repn = True
279    elif options.skip_canonical_repn is True:
280        model.skip_canonical_repn = True
281
282    #
283    # Create Problem Instance
284    #
285    ep = ExtensionPoint(IPyomoScriptCreateModelData)
286    if len(ep) > 1:
287        msg = 'Multiple model data construction plugins have been registered!'
288        raise SystemExit, msg
289
290    if len(ep) == 1:
291        modeldata = ep.service().apply( options=options, model=model )
292    else:
293        modeldata = ModelData()
294
295    if len(options.data_files) > 1:
296        #
297        # Load a list of *.dat files
298        #
299        for file in options.data_files:
300            suffix = (file).split(".")[-1]
301            if suffix != "dat":
302                msg = 'When specifiying multiple data files, they must all '  \
303                      'be *.dat files.  File specified: %s'
304                raise SystemExit, msg % str( file )
305
306            modeldata.add(file)
307
308        modeldata.read(model)
309
310        if not options.profile_memory is None:
311           instance = model.create(modeldata, profile_memory=options.profile_memory)
312        else:
313           instance = model.create(modeldata)
314
315    elif len(options.data_files) == 1:
316       #
317       # Load a *.dat file or process a *.py data file
318       #
319       suffix = (options.data_files[0]).split(".")[-1]
320       if suffix == "dat":
321          if not options.profile_memory is None:
322             instance = model.create(options.data_files[0], profile_memory=options.profile_memory)
323          else:
324             instance = model.create(options.data_files[0])
325       elif suffix == "py":
326          userdata = pyutilib.misc.import_file(options.data_files[0])
327          if "modeldata" in dir(userdata):
328                if len(ep) == 1:
329                    msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
330                          " 'modeldata' object that is provided in the model"
331                    raise SystemExit, msg
332
333                if userdata.modeldata is None:
334                    msg = "'modeldata' object is 'None' in module %s"
335                    raise SystemExit, msg % str( options.data_files[0] )
336
337                modeldata=userdata.modeldata
338
339          else:
340                if len(ep) == 0:
341                    msg = "Neither 'modeldata' nor 'pyomo_create_model_data"  \
342                          'is defined in module %s'
343                    raise SystemExit, msg % str( options.data_files[0] )
344
345          modeldata.read(model)
346          if not options.profile_memory is None:
347             instance = model.create(modeldata, profile_memory=options.profile_memory)
348          else:
349             instance = model.create(modeldata)
350       else:
351          raise ValueError, "Unknown data file type: "+options.data_files[0]
352    else:
353       if not options.profile_memory is None:
354          instance = model.create(modeldata, profile_memory=options.profile_memory)
355       else:
356          instance = model.create(modeldata)
357
358    if options.linearize_expressions is True:
359       linearize_model_expressions(instance)
360
361    #
362    ep = ExtensionPoint(IPyomoScriptModifyInstance)
363    for ep in ExtensionPoint(IPyomoScriptModifyInstance):
364        ep.apply( options=options, model=model, instance=instance )
365    #
366    if logger.isEnabledFor(logging.DEBUG):
367       print "MODEL INSTANCE"
368       instance.pprint()
369       print ""
370
371    for ep in ExtensionPoint(IPyomoScriptPrintInstance):
372        ep.apply( options=options, instance=instance )
373
374    fname=None
375    symbol_map=None
376    if not options.save_model is None:
377        if options.save_model == True:
378            if options.format in (ProblemFormat.cpxlp, ProblemFormat.lpxlp):
379                fname = (options.data_files[0])[:-3]+'lp'
380            else:
381                fname = (options.data_files[0])[:-3]+str(options.format)
382            format=options.format
383        else:
384            fname = options.save_model
385            format=None
386        (fname, symbol_map) = instance.write(filename=fname, format=format)
387        if not os.path.exists(fname):
388            print "ERROR: file "+fname+" has not been created!"
389        else:
390            print "Model written to file '"+str(fname)+"'"
391    for ep in ExtensionPoint(IPyomoScriptSaveInstance):
392        ep.apply( options=options, instance=instance )
393
394    return pyutilib.misc.Container(
395        model=model, instance=instance, symbol_map=symbol_map, filename=fname )
396
397#
398# Perform optimization with concrete instance
399#
400def apply_optimizer(options, instance):
401    #
402    # Create Solver and Perform Optimization
403    #
404    solver = options.solver
405    if solver is None:
406       raise ValueError, "Problem constructing solver:  no solver specified"
407
408    subsolver=None
409    if not solver is None and ':' in solver:
410        solver, subsolver = solver.split(':')
411    opt = SolverFactory( solver )
412    if opt is None:
413        solver_list = SolverFactory.services()
414        solver_list = sorted( filter(lambda x: '_' != x[0], solver_list) )
415        raise ValueError, "Problem constructing solver `"+str(solver)+"' (choose from: %s)" % ", ".join(solver_list)
416
417    # let the model know of our chosen solver's capabilities
418    instance.has_capability = opt.has_capability
419
420    opt.keepFiles=options.keepfiles or options.log
421    if options.timelimit == 0:
422       options.timelimit=None
423
424    if options.solver_mipgap is not None:
425       opt.mipgap = options.solver_mipgap
426
427    if not options.solver_suffixes is None:
428        opt.suffixes = options.solver_suffixes
429
430    if not subsolver is None:
431        subsolver=' solver='+subsolver
432    else:
433        subsolver=''
434
435    if not options.solver_options is None:
436        opt.set_options(" ".join(options.solver_options)+subsolver)
437
438    if options.smanager_type is None:
439        solver_mngr = SolverManagerFactory( 'serial' )
440    else:
441        solver_mngr = SolverManagerFactory( options.smanager_type )
442
443    if solver_mngr is None:
444        msg = "Problem constructing solver manager '%s'"
445        raise ValueError, msg % str( options.smanager_type )
446
447    results = solver_mngr.solve( instance, opt=opt, tee=options.tee, timelimit=options.timelimit )
448
449    if results == None:
450       raise ValueError, "opt.solve returned None"
451
452    return results, opt
453
454#
455# Process results
456#
457def process_results(options, instance, results, opt):
458    #
459    if options.log:
460       print ""
461       print "=========================================================="
462       print "Solver Logfile:",opt.log_file
463       print "=========================================================="
464       print ""
465       INPUT = open(opt.log_file, "r")
466       for line in INPUT:
467         print line,
468       INPUT.close()
469    #
470    if options.save_results:
471        results.write(filename=options.save_results)
472    #
473    ep = ExtensionPoint(IPyomoScriptPrintResults)
474    if len(ep) == 0:
475        try:
476            instance.load(results)
477        except Exception, e:
478            print "Problem loading solver results"
479            raise
480        print ""
481        results.write(num=1)
482    #
483    if options.summary:
484       print ""
485       print "=========================================================="
486       print "Solution Summary"
487       print "=========================================================="
488       if len(results.solution(0).variable) > 0:
489          print ""
490          display(instance)
491       else:
492          print "No solutions reported by solver."
493    #
494    for ep in ExtensionPoint(IPyomoScriptPrintResults):
495        ep.apply( options=options, instance=instance, results=results )
496    #
497    for ep in ExtensionPoint(IPyomoScriptSaveResults):
498        ep.apply( options=options, instance=instance, results=results )
499    #
500    return True
501
502
503def apply_postprocessing(options, instance, results):
504    for file in options.postprocess:
505        postprocess = pyutilib.misc.import_file(file)
506        if "postprocess" in dir(postprocess):
507            postprocess.postprocess(instance,results)
508    for ep in ExtensionPoint(IPyomoScriptPostprocess):
509        ep.apply( options=options, instance=instance, results=results )
510    #
511    # Deactivate and delete plugins
512    #
513    for plugin in options._usermodel_plugins:
514        plugin.deactivate()
515    options._usermodel_plugins = []
516
517
518#
519# Execute a function that processes command-line arguments and then
520# calls a command-line driver.  This
521# is segregated from the driver to enable profiling.
522#
523def run_command(command, parser, args=None, name='unknown'):
524    #
525    #
526    # Parse command-line options
527    #
528    #
529    try:
530       _options = parser.parse_args(args=args)
531       # Replace the parser options object with a pyutilib.misc.Options object
532       options = pyutilib.misc.Options()
533       for key in dir(_options):
534            if key[0] != '_':
535                val = getattr(_options, key)
536                if not isinstance(val, types.MethodType):
537                    options[key] = val
538    except SystemExit:
539       # the parser throws a system exit if "-h" is specified - catch
540       # it to exit gracefully.
541       return
542    #
543    # Configure the logger
544    #
545    logging.getLogger('coopr.pyomo').setLevel(logging.ERROR)
546    logging.getLogger('coopr').setLevel(logging.ERROR)
547    logging.getLogger('pyutilib').setLevel(logging.ERROR)
548    #
549    if options.warning:
550        logging.getLogger('coopr.pyomo').setLevel(logging.WARNING)
551        logging.getLogger('coopr').setLevel(logging.WARNING)
552        logging.getLogger('pyutilib').setLevel(logging.WARNING)
553    if options.info:
554        logging.getLogger('coopr.pyomo').setLevel(logging.INFO)
555        logging.getLogger('coopr').setLevel(logging.INFO)
556        logging.getLogger('pyutilib').setLevel(logging.INFO)
557    if options.verbose > 0:
558        if options.verbose >= 1:
559            logger.setLevel(logging.DEBUG)
560        if options.verbose >= 2:
561            logging.getLogger('coopr').setLevel(logging.DEBUG)
562        if options.verbose >= 3:
563            logging.getLogger('pyutilib').setLevel(logging.DEBUG)
564    if options.debug:
565        logging.getLogger('coopr.pyomo').setLevel(logging.DEBUG)
566        logging.getLogger('coopr').setLevel(logging.DEBUG)
567        logging.getLogger('pyutilib').setLevel(logging.DEBUG)
568    #
569    # Setup I/O redirect to a logfile
570    #
571    logfile = getattr(options, 'logfile', None)
572    if not logfile is None:
573        pyutilib.misc.setup_redirect(logfile)
574    #
575    # Call the main Pyomo runner with profiling
576    #
577    if options.profile > 0:
578        if not pstats_available:
579            if not logfile is None:
580                pyutilib.misc.reset_redirect()
581            msg = "Cannot use the 'profile' option.  The Python 'pstats' "    \
582                  'package cannot be imported!'
583            raise ValueError, msg
584        tfile = TempfileManager.create_tempfile(suffix=".profile")
585        tmp = profile.runctx(
586          command.__name__ + '(options=options,parser=parser)', command.__globals__, locals(), tfile
587        )
588        p = pstats.Stats(tfile).strip_dirs()
589        p.sort_stats('time', 'cum')
590        p = p.print_stats(options.profile)
591        p.print_callers(options.profile)
592        p.print_callees(options.profile)
593        p = p.sort_stats('cum','calls')
594        p.print_stats(options.profile)
595        p.print_callers(options.profile)
596        p.print_callees(options.profile)
597        p = p.sort_stats('calls')
598        p.print_stats(options.profile)
599        p.print_callers(options.profile)
600        p.print_callees(options.profile)
601        TempfileManager.clear_tempfiles()
602        ans = [tmp, None]
603    else:
604        #
605        # Call the main Pyomo runner without profiling
606        #
607        try:
608            ans = command(options=options, parser=parser)
609        except SystemExit, err:
610            if __debug__:
611                if options.debug:
612                    sys.exit(0)
613            print 'Exiting %s: %s' % (name, str(err))
614            ans = None
615        except Exception, err:
616            # If debugging is enabled, pass the exception up the chain
617            # (to pyomo_excepthook)
618            if __debug__:
619                if options.debug:
620                    if not logfile is None:
621                        pyutilib.misc.reset_redirect()
622                    raise
623
624            if len(options.model_file) > 0:
625                name = "model " + options.model_file
626            else:
627                name = "model"
628               
629            global filter_excepthook
630            if filter_excepthook:
631                action = "loading"
632            else:
633                action = "running"
634
635            msg = "Unexpected exception while %s %s\n" % (action, name)
636            #
637            # This handles the case where the error is propagated by a KeyError.
638            # KeyError likes to pass raw strings that don't handle newlines
639            # (they translate "\n" to "\\n"), as well as tacking on single
640            # quotes at either end of the error message. This undoes all that.
641            #
642            errStr = str(err)
643            if type(err) == KeyError:
644                errStr = str(err).replace(r"\n","\n")[1:-1]
645
646            logging.getLogger('coopr.pyomo').error(msg+errStr)
647            ans = None
648
649    if not logfile is None:
650        pyutilib.misc.reset_redirect()
651
652    if options.disable_gc:
653        gc.enable()
654    return ans
Note: See TracBrowser for help on using the repository browser.