Changeset 3681


Ignore:
Timestamp:
Feb 20, 2011 8:29:41 PM (9 years ago)
Author:
wehart
Message:

Migrating the profiling logic into scripting/util.py.
This allows it to be applied uniformly in the conversion scripts:
pyomo2nl and pyomo2lp.

Location:
coopr.pyomo/trunk/coopr/pyomo
Files:
2 added
4 edited

Legend:

Unmodified
Added
Removed
  • coopr.pyomo/trunk/coopr/pyomo/scripting/convert.py

    r3680 r3681  
    4040def convert(options=Options(), parser=None):
    4141    global format
    42     options.verbose = False
     42    if options.save_model is None:
     43        if format == ProblemFormat.cpxlp:
     44            options.save_model = 'unknown.lp'
     45        else:
     46            options.save_model = 'unknown.'+str(format)
    4347    options.format = format
     48    #
    4449    if options.help_components:
    4550        util.print_components(options)
    4651        return Container()
     52    #
    4753    if not util.setup_environment(options):
    48         return                                               #pragma:nocover
     54        return Container()
     55    #
    4956    if not util.apply_preprocessing(options, parser):
    50         return
    51     model = util.create_model(options)
    52     return model
     57        return Container()
     58    #
     59    model_data = util.create_model(options)
     60    #
     61    util.finalize(options, model=model_data.model)
     62    #
     63    return Container(model=model_data.model)
    5364
    5465def pyomo2lp(args=None):
  • coopr.pyomo/trunk/coopr/pyomo/scripting/pyomo.py

    r3679 r3681  
    1414from coopr.opt.parallel import SolverManagerFactory
    1515from pyutilib.misc import Options, Container
    16 try:
    17     import IPython
    18     IPython_available=True
    19     from IPython.Shell import IPShellEmbed
    20     ipshell = IPShellEmbed([''],
    21                 banner = '\n# Dropping into Python interpreter',
    22                 exit_msg = '\n# Leaving Interpreter, back to Pyomo\n')
    23 except ImportError:
    24     IPython_available=False
    2516
    2617try:
     
    2819   from pympler.muppy import summary
    2920   from pympler.muppy import tracker
    30    from pympler.asizeof import *   
     21   from pympler.asizeof import *
    3122   pympler_available = True
    3223except ImportError:
     
    3425
    3526import util
    36 import gc
    3727
    3828
     
    4838        dest='preprocess',
    4939        default=[])
    50     group.add_argument('--instance-only',
    51         help='Generate a model instance, and then return',
    52         action='store_true',
    53         dest='only_instance',
    54         default=False)
    5540    group.add_argument('--model-name',
    5641        help='The name of the model object that is created in the specified ' \
     
    8469        dest='save_model',
    8570        default=None)
     71    return group
    8672
    8773
     
    114100        dest='debug',
    115101        default=False)
     102    return group
    116103
    117104
     
    168155        type=int,
    169156        default=0)
     157    return group
     158
    170159
    171160def add_postsolve_group(parser):
     
    205194        dest='summary',
    206195        default=False)
     196    return group
     197
    207198
    208199def add_misc_group(parser):
     
    255246        dest='tempdir',
    256247        default=None)
     248    return group
    257249   
    258250
     
    266258                usage = '%(prog)s [options] <model.py> [<model.dat>]'
    267259                )
    268     add_model_group(parser)
     260    group = add_model_group(parser)
     261    group.add_argument('--instance-only',
     262        help='Generate a model instance, and then return',
     263        action='store_true',
     264        dest='only_instance',
     265        default=False)
    269266    add_solver_group(parser)
    270267    add_postsolve_group(parser)
     
    277274
    278275def run_pyomo(options=Options(), parser=None):
    279 
    280     if (pympler_available is True) and (options.profile_memory >= 1):
    281        memory_tracker = tracker.SummaryTracker()
    282 
     276    if options.help_solvers:
     277        util.print_solver_help(options)
     278        return Container()
     279    #
    283280    if options.help_components:
    284281        util.print_components(options)
    285282        return Container()
    286     if options.help_solvers:
    287         util.print_solver_help(options)
    288         return Container()
     283    #
    289284    if not util.setup_environment(options):
    290285        return Container()                                   #pragma:nocover
     286    #
    291287    if not util.apply_preprocessing(options, parser):
    292288        return Container()                                   #pragma:nocover
    293     if (pympler_available is True) and (options.profile_memory >= 1):
    294        objects_before_instance_creation = muppy.get_objects()
    295        summary_before_instance_creation = summary.summarize(objects_before_instance_creation)
    296        print "Initial set of objects:"
    297        summary.print_(summary_before_instance_creation, limit=50)                   
    298        
     289    #
    299290    model_data = util.create_model(options)
    300    
    301     if (pympler_available is True) and (options.profile_memory >= 1):
    302        objects_after_instance_creation = muppy.get_objects()
    303        summary_after_instance_creation = summary.summarize(objects_after_instance_creation)
    304        print "Objects created during Pyomo instance creation:"       
    305        memory_tracker.print_diff(summary1=summary_before_instance_creation, summary2=summary_after_instance_creation)
    306    
    307     if options.save_model or options.only_instance:
     291    if options.save_model and options.only_instance:
    308292        return Container(instance=model_data.instance)
    309    
     293    #
    310294    results, opt = util.apply_optimizer(options, model_data.instance)
    311 
    312     if (pympler_available is True) and (options.profile_memory >= 1):
    313        objects_after_optimization = muppy.get_objects()
    314        summary_after_optimization = summary.summarize(objects_after_optimization)
    315        print "Objects created during optimization:"
    316        memory_tracker.print_diff(summary1=summary_after_instance_creation, summary2=summary_after_optimization)
    317    
     295    #
    318296    if not util.process_results(options, model_data.instance, results, opt):
    319297        return Container(instance=model_data.instance, results=results) #pragma:nocover
    320 
    321     if (pympler_available is True) and (options.profile_memory >= 1):
    322        objects_after_results_processing = muppy.get_objects()
    323        summary_after_results_processing = summary.summarize(objects_after_results_processing)
    324        diff_summary = summary.get_diff(summary_after_optimization, summary_after_results_processing)
    325        print "Objects created during results processing:"
    326        memory_tracker.print_diff(summary1=summary_after_optimization, summary2=summary_after_results_processing)
    327    
     298    #
    328299    util.apply_postprocessing(options, model_data.instance, results)
    329 
    330     if (pympler_available is True) and (options.profile_memory >= 1):
    331        final_objects = muppy.get_objects()
    332        summary_final = summary.summarize(final_objects)
    333        final_summary = summary.summarize(final_objects)
    334        print "Final set of objects:"
    335        summary.print_(final_summary, limit=50)
    336    
    337     model=model_data.model
    338     instance=model_data.instance
    339     if options.interactive:
    340         if IPython_available:
    341             ipshell()
    342         else:
    343             import code
    344             shell = code.InteractiveConsole(locals())
    345             print '\n# Dropping into Python interpreter'
    346             shell.interact()
    347             print '\n# Leaving Interpreter, back to Pyomo\n'
    348            
     300    #
     301    util.finalize(options, model=model_data.model, instance=model_data.instance, results=results)
     302    #       
    349303    return Container(instance=model_data.instance, results=results)
    350304
  • coopr.pyomo/trunk/coopr/pyomo/scripting/util.py

    r3679 r3681  
    2929except ImportError:
    3030    pstats_available=False
     31
     32try:
     33    import IPython
     34    IPython_available=True
     35    from IPython.Shell import IPShellEmbed
     36    ipshell = IPShellEmbed([''],
     37                banner = '\n# Dropping into Python interpreter',
     38                exit_msg = '\n# Leaving Interpreter, back to Pyomo\n')
     39except ImportError:
     40    IPython_available=False
     41
     42from pyutilib.misc import Options
     43try:
     44   from pympler.muppy import muppy
     45   from pympler.muppy import summary
     46   from pympler.muppy import tracker
     47   from pympler.asizeof import *
     48   pympler_available = True
     49except ImportError:
     50   pympler_available = False
     51memory_tracker = None
     52memory_data = Options()
    3153
    3254from coopr.pyomo import *
     
    105127def setup_environment(options):
    106128    #
     129    # Setup memory tracker
     130    #
     131    if (pympler_available is True) and (options.profile_memory >= 1):
     132        global memory_tracker
     133        memory_tracker = tracker.SummaryTracker()
     134    #
    107135    # Disable garbage collection
    108136    #
     
    223251# Create instance of Pyomo model
    224252#
    225 # TODO: extend this API to make it more generic:
    226 #       create_model(options, model_file=None, data_files=None, modeldata=None)
    227 #
    228253def create_model(options):
     254    if (pympler_available is True) and (options.profile_memory >= 1):
     255        global memory_data
     256        objects_before_instance_creation = muppy.get_objects()
     257        memory_data.summary_before_instance_creation = summary.summarize(objects_before_instance_creation)
     258        print "Initial set of objects:"
     259        summary.print_(memory_data.summary_before_instance_creation,limit=50)
    229260    #
    230261    # Verify that files exist
     
    374405    fname=None
    375406    symbol_map=None
     407    if options.save_model is None and options.debug:
     408        options.save_model = 'unknown.lp'
    376409    if not options.save_model is None:
    377410        if options.save_model == True:
     
    392425        ep.apply( options=options, instance=instance )
    393426
    394     return pyutilib.misc.Container(
    395         model=model, instance=instance, symbol_map=symbol_map, filename=fname )
     427    if (pympler_available is True) and (options.profile_memory >= 1):
     428        objects_after_instance_creation = muppy.get_objects()
     429        memory_data.summary_after_instance_creation = summary.summarize(objects_after_instance_creation)
     430        print "\nObjects created during Pyomo instance creation:"
     431        memory_tracker.print_diff(
     432                    summary1=memory_data.summary_before_instance_creation,
     433                    summary2=memory_data.summary_after_instance_creation)
     434
     435    return pyutilib.misc.Container(
     436                    model=model, instance=instance,
     437                    symbol_map=symbol_map, filename=fname )
    396438
    397439#
     
    449491    if results == None:
    450492       raise ValueError, "opt.solve returned None"
     493
     494    if (pympler_available is True) and (options.profile_memory >= 1):
     495        global memory_data
     496        objects_after_optimization = muppy.get_objects()
     497        memory_data.summary_after_optimization = summary.summarize(objects_after_optimization)
     498        print "\nObjects created during optimization:"
     499        memory_tracker.print_diff(
     500                    summary1=memory_data.summary_after_instance_creation,
     501                    summary2=memory_data.summary_after_optimization)
    451502
    452503    return results, opt
     
    498549        ep.apply( options=options, instance=instance, results=results )
    499550    #
     551    if (pympler_available is True) and (options.profile_memory >= 1):
     552        global memory_data
     553        objects_after_results_processing = muppy.get_objects()
     554        memory_data.summary_after_results_processing = summary.summarize(objects_after_results_processing)
     555        #diff_summary = summary.get_diff( memory_data.summary_after_optimization, memory_data.summary_after_results_processing)
     556        print "\nObjects created during results processing:"
     557        memory_tracker.print_diff(
     558                    summary1=memory_data.summary_after_optimization,
     559                    summary2=memory_data.summary_after_results_processing)
     560
    500561    return True
    501562
     
    514575        plugin.deactivate()
    515576    options._usermodel_plugins = []
     577    #
     578    if (pympler_available is True) and (options.profile_memory >= 1):
     579        final_objects = muppy.get_objects()
     580        summary_final = summary.summarize(final_objects)
     581        final_summary = summary.summarize(final_objects)
     582        print "\nFinal set of objects:"
     583        summary.print_(final_summary, limit=50)
     584    #
     585    return True
     586
     587
     588def finalize(options, model=None, instance=None, results=None):
     589    model=model
     590    instance=instance
     591    results=results
     592    #
     593    if options.interactive:
     594        if IPython_available:
     595            ipshell()
     596        else:
     597            import code
     598            shell = code.InteractiveConsole(locals())
     599            print '\n# Dropping into Python interpreter'
     600            shell.interact()
     601            print '\n# Leaving Interpreter, back to Pyomo\n'
    516602
    517603
  • coopr.pyomo/trunk/coopr/pyomo/tests/examples/test_convert.py

    r3680 r3681  
    2828
    2929
    30 #class Test(unittest.TestCase):
    31 class Test(object):
     30class Test(unittest.TestCase):
    3231
    3332    def convert(self, cmd, type, **kwds):
     
    5655    def test1a(self):
    5756        """Simple execution of 'convert2nl'"""
    58         self.convert('pmedian.py pmedian.dat', type='lp', file=currdir+'test1.out')
    59         self.failUnlessFileEqualsBaseline(currdir+'pmedian.lp', currdir+'_pmedian.lp')
    60         os.remove(currdir+'test1.out')
     57        self.convert('pmedian.py pmedian.dat', type='nl', file=currdir+'test1a.out')
     58        self.failUnlessFileEqualsBaseline(currdir+'unknown.nl', currdir+'_pmedian.nl')
     59        os.remove(currdir+'test1a.out')
    6160
    62     def test1c(self):
    63         """Simple execution of 'pyomo'"""
    64         self.run_pyomo('pmedian.py pmedian.dat', file=currdir+'test1.out')
    65         self.failUnlessFileEqualsBaseline(currdir+"test1.out", currdir+"test1.txt")
     61    def test1b(self):
     62        """Simple execution of 'pyomo2nl'"""
     63        self.run_convert2nl('pmedian.py pmedian.dat', file=currdir+'test1b.out')
     64        self.failUnlessFileEqualsBaseline(currdir+'unknown.nl', currdir+'_pmedian.nl')
     65        os.remove(currdir+'test1b.out')
    6666
    67     def test2(self):
    68         """Run pyomo with bad --model-name option value"""
    69         output=self.pyomo('--model-name=dummy pmedian.py pmedian.dat')
    70         self.failUnlessEqual(output.strip(),"Exiting pyomo: Neither 'dummy' nor 'pyomo_create_model' are available in module pmedian.py")
     67    def test2a(self):
     68        """Simple execution of 'convert2lp'"""
     69        self.convert('pmedian.py pmedian.dat', type='lp', file=currdir+'test2a.out')
     70        self.failUnlessFileEqualsBaseline(currdir+'unknown.lp', currdir+'_pmedian.lp')
     71        os.remove(currdir+'test2a.out')
    7172
    72     def test3(self):
    73         """Run pyomo with model that does not define model object"""
    74         output=self.pyomo('pmedian1.py pmedian.dat')
    75         self.failUnlessEqual(output.strip(),"Exiting pyomo: Neither 'model' nor 'pyomo_create_model' are available in module pmedian1.py")
     73    def test2b(self):
     74        """Simple execution of 'pyomo2lp'"""
     75        self.run_convert2lp('pmedian.py pmedian.dat', file=currdir+'test2b.out')
     76        self.failUnlessFileEqualsBaseline(currdir+'unknown.lp', currdir+'_pmedian.lp')
     77        os.remove(currdir+'test2b.out')
    7678
    77     def test4(self):
    78         """Run pyomo with good --model-name option value"""
    79         self.pyomo('--model-name=MODEL pmedian1.py pmedian.dat', file=currdir+'test4.out')
    80         self.failUnlessFileEqualsBaseline(currdir+"test4.out", currdir+"test1.txt")
    81 
    82     def test5(self):
    83         """Run pyomo with create_model function"""
    84         self.pyomo('pmedian2.py pmedian.dat', file=currdir+'test5.out')
    85         self.failUnlessFileEqualsBaseline(currdir+"test5.out", currdir+"test5.txt")
    86 
    87     def test6(self):
    88         """Run pyomo with help-components option"""
    89         self.pyomo('--help-components', file=currdir+'test6.out')
    90         self.failUnlessFileEqualsBaseline(currdir+"test6.out", currdir+"test6.txt", filter=filter_fn)
    91 
    92     def Xtest7(self):
    93         """Run pyomo with help option"""
    94         self.pyomo('--help', file=currdir+'test7.out')
    95         self.failUnlessFileEqualsBaseline(currdir+"test7.out", currdir+"test7.txt")
    96 
    97     def test8(self):
    98         """Run pyomo with --instance-only option"""
    99         output = self.pyomo('--instance-only pmedian.py pmedian.dat', file=currdir+'test8.out')
    100         self.failUnlessEqual(type(output.instance), coopr.pyomo.AbstractModel)
    101         os.remove(currdir+'test8.out')
    102 
    103     def test9(self):
    104         """Run pyomo with --disable-gc option"""
    105         output = self.pyomo('--disable-gc pmedian.py pmedian.dat', file=currdir+'test9.out')
    106         self.failUnlessEqual(type(output.instance), coopr.pyomo.AbstractModel)
    107         os.remove(currdir+'test9.out')
    108 
    109     def test10(self):
    110         """Run pyomo with --verbose option"""
    111         def filter(line):
    112             #print "HERE",line
    113             return line.startswith("Writing") | line.startswith('DEBUG:') | line.startswith('INFO:')
    114         self.pyomo('-v pmedian.py pmedian.dat', file=currdir+'test10.out')
    115         self.failUnlessFileEqualsBaseline(currdir+"test10.out", currdir+"test10.txt", filter)
    116 
    117     def Xtest11(self):
    118         """Run pyomo with --debug=generate option"""
    119         self.pyomo('--debug=generate pmedian.py pmedian.dat', file=currdir+'test11.out')
    120         self.failUnlessFileEqualsBaseline(currdir+"test11.out", currdir+"test11.txt")
    121 
    122     def test12(self):
    123         """Run pyomo with --logfile option"""
    124         def filter(line):
    125             #print "HERE",line
    126             return line.startswith("Writing")
    127         self.pyomo('--logfile=%s pmedian.py pmedian.dat' % (currdir+'test12.log'), file=currdir+'test12.out')
    128         self.failUnlessFileEqualsBaseline(currdir+"test12.log", currdir+"test12.txt", filter)
    129         os.remove(currdir+'test12.out')
    130 
    131     def test13(self):
    132         """Simple execution of 'pyomo' with implicit rules"""
    133         self.pyomo('pmedian3.py pmedian.dat', file=currdir+'test1a.out')
    134         self.failUnlessFileEqualsBaseline(currdir+"test1a.out", currdir+"test1.txt")
    135 
    136 
    137 Test = unittest.skipIf(pyutilib.services.registered_executable("glpsol") is None, "The 'glpsol' executable is not available")(Test)
    13879
    13980if __name__ == "__main__":
Note: See TracChangeset for help on using the changeset viewer.