Changeset 5829


Ignore:
Timestamp:
May 29, 2012 4:37:18 PM (7 years ago)
Author:
jwatson
Message:

Re-working behavior of profile_memory option on pyomo script and PyomoModel? load() methods. Reporting should be more sane/useful.

Location:
coopr.pyomo/trunk/coopr/pyomo
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • coopr.pyomo/trunk/coopr/pyomo/base/PyomoModel.py

    r5817 r5829  
    1818import traceback
    1919import weakref
     20import gc
     21
    2022try:
    2123    from collections import OrderedDict
     
    4547
    4648try:
    47     from pympler import tracker
     49    from pympler import muppy
     50    from pympler import summary
    4851    pympler_available = True
    4952except ImportError:
     
    361364        Load the model with data from a file or a Solution object
    362365        """
     366
    363367        if arg is None or type(arg) is str:
    364368            self._load_model_data(ModelData(filename=arg,model=self), namespaces, profile_memory=profile_memory)
     
    447451        #
    448452        suspend_gc = PauseGC()
     453
    449454        #
    450455        # Unlike the standard method in the pympler summary module, the tracker
     
    452457        #
    453458        profile_memory = kwds.get('profile_memory', 0)
     459
    454460        if (pympler_available is True) and (profile_memory >= 2):
    455             memory_tracker = tracker.SummaryTracker()
     461            mem_used = muppy.get_size(muppy.get_objects())
     462            print ""
     463            print "      Total memory = %d bytes prior to model construction" % mem_used
     464
    456465        #
    457466        # Do some error checking
     
    465474        #
    466475        for component_name in self.components:
    467             if (pympler_available is True) and (profile_memory >= 2):
    468                 memory_tracker.create_summary()
    469             #
     476
    470477            if self.components[component_name].type() is Model:
    471478                continue
     
    524531                raise
    525532
    526             #print component_name, data # XXX
    527 
    528533            if (pympler_available is True) and (profile_memory >= 2):
    529                 print "Objects created during construction of component "+component_name+":"
    530                 memory_tracker.print_diff()
     534                mem_used = muppy.get_size(muppy.get_objects())
     535                print "      Total memory = %d bytes following construction of component=%s" % (mem_used, component_name)
     536
     537        if (pympler_available is True) and (profile_memory >= 2):
     538            gc.collect()
     539            print ""
     540            print "      Summary of objects following instance construction"
     541            post_construction_summary = summary.summarize(muppy.get_objects())
     542            summary.print_(post_construction_summary, limit=100)
     543
     544            print ""
    531545
    532546    def _load_solution( self, soln, symbol_map,
     
    756770    #   
    757771    if preprocess is True:
     772
     773        instance.preprocess()
     774
    758775        if (pympler_available is True) and (profile_memory >= 2):
    759             memory_tracker = tracker.SummaryTracker()
    760             memory_tracker.create_summary()
    761         #
    762         instance.preprocess()
    763         #
    764         if (pympler_available is True) and (profile_memory >= 2):
    765             print "Objects created during instance preprocessing:"
    766             memory_tracker.print_diff()
    767     #
     776            mem_used = muppy.get_size(muppy.get_objects())
     777            print "      Total memory = %d bytes following instance preprocessing" % mem_used
     778            print ""
     779
    768780    if not name is None:
    769781        instance.name=name
    770     #
     782
    771783    return Bunch(instance=instance)
    772784
  • coopr.pyomo/trunk/coopr/pyomo/scripting/util.py

    r5817 r5829  
    4848try:
    4949    from pympler import muppy
    50     from pympler.muppy import summary
    51     from pympler import tracker
    5250    from pympler.asizeof import *
    5351    pympler_available = True
    5452except ImportError:
    5553    pympler_available = False
    56 memory_tracker = None
    5754memory_data = Options()
    5855
     
    150147        sys.stdout.write('[%8.2f] Setting up Pyomo environment\n' % 0.0)
    151148        sys.stdout.flush()
    152     #
    153     # Setup memory tracker
    154     #
    155     if (pympler_available is True) and (data.options.profile_memory >= 1):
    156         global memory_tracker
    157         memory_tracker = tracker.SummaryTracker()
     149
    158150    #
    159151    # Disable garbage collection
     
    308300    if (pympler_available is True) and (data.options.profile_memory >= 1):
    309301        global memory_data
    310         objects_before_instance_creation = muppy.get_objects()
    311         memory_data.summary_before_instance_creation = summary.summarize(objects_before_instance_creation)
    312         print "Initial set of objects"
    313         mem_used = muppy.get_size(objects_before_instance_creation)
     302        mem_used = muppy.get_size(muppy.get_objects())
    314303        data.options.max_memory = mem_used
    315         print "   Total memory used: ",mem_used
    316         if data.options.profile_memory > 1:
    317             summary.print_(memory_data.summary_before_instance_creation,limit=50)
     304        print "   Total memory = %d bytes prior to model construction" % mem_used
     305
    318306    #
    319307    # Create Model
     
    392380
    393381        if not data.options.profile_memory is None:
    394             instance = model.create(modeldata, namespaces=data.options.namespaces, profile_memory=data.options.profile_memory-1)
     382            instance = model.create(modeldata, namespaces=data.options.namespaces, profile_memory=data.options.profile_memory)
    395383        else:
    396384            instance = model.create(modeldata, namespaces=data.options.namespaces)
     
    403391        if suffix == "dat":
    404392            if not data.options.profile_memory is None:
    405                 instance = model.create(data.options.data_files[0], namespaces=data.options.namespaces, profile_memory=data.options.profile_memory-1)
     393                instance = model.create(data.options.data_files[0], namespaces=data.options.namespaces, profile_memory=data.options.profile_memory)
    406394            else:
    407395                instance = model.create(data.options.data_files[0], namespaces=data.options.namespaces)
     
    428416            modeldata.read(model)
    429417            if not data.options.profile_memory is None:
    430                 instance = model.create(modeldata, namespaces=data.options.namespaces, profile_memory=data.options.profile_memory-1)
     418                instance = model.create(modeldata, namespaces=data.options.namespaces, profile_memory=data.options.profile_memory)
    431419            else:
    432420                instance = model.create(modeldata, namespaces=data.options.namespaces)
     
    435423    else:
    436424        if not data.options.profile_memory is None:
    437             instance = model.create(modeldata, namespaces=data.options.namespaces, profile_memory=data.options.profile_memory-1)
     425            instance = model.create(modeldata, namespaces=data.options.namespaces, profile_memory=data.options.profile_memory)
    438426        else:
    439427            instance = model.create(modeldata, namespaces=data.options.namespaces)
     
    479467
    480468    if (pympler_available is True) and (data.options.profile_memory >= 1):
    481         objects_after_instance_creation = muppy.get_objects()
    482         memory_data.summary_after_instance_creation = summary.summarize(objects_after_instance_creation)
    483         print "\nObjects created during Pyomo instance creation"
    484         mem_used = muppy.get_size(objects_after_instance_creation)
     469        mem_used = muppy.get_size(muppy.get_objects())
    485470        if mem_used > data.options.max_memory:
    486471            data.options.max_memory = mem_used
    487         print "   Total memory used: ",mem_used
    488         if data.options.profile_memory > 1:
    489             memory_tracker.print_diff(
    490                     summary1=memory_data.summary_before_instance_creation,
    491                     summary2=memory_data.summary_after_instance_creation)
     472        print "   Total memory = %d bytes following Pyomo instance creation" % mem_used
    492473
    493474    return pyutilib.misc.Options(
     
    560541    if (pympler_available is True) and (data.options.profile_memory >= 1):
    561542        global memory_data
    562         objects_after_optimization = muppy.get_objects()
    563         memory_data.summary_after_optimization = summary.summarize(objects_after_optimization)
    564         print "\nObjects created during optimization"
    565         mem_used = muppy.get_size(objects_after_optimization)
     543        mem_used = muppy.get_size(muppy.get_objects())
    566544        if mem_used > data.options.max_memory:
    567545            data.options.max_memory = mem_used
    568         print "   Total memory used: ",mem_used
    569         if data.options.profile_memory > 1:
    570             memory_tracker.print_diff(
    571                     summary1=memory_data.summary_after_instance_creation,
    572                     summary2=memory_data.summary_after_optimization)
     546        print "   Total memory = %d bytes following optimization" % mem_used
    573547
    574548    return pyutilib.misc.Options(results=results, opt=opt)
     
    731705    if (pympler_available is True) and (data.options.profile_memory >= 1):
    732706        global memory_data
    733         objects_after_results_processing = muppy.get_objects()
    734         memory_data.summary_after_results_processing = summary.summarize(objects_after_results_processing)
    735         #diff_summary = summary.get_diff( memory_data.summary_after_optimization, memory_data.summary_after_results_processing)
    736         print "\nObjects created during results processing"
    737         mem_used = muppy.get_size(objects_after_results_processing)
     707        mem_used = muppy.get_size(muppy.get_objects())
    738708        if mem_used > data.options.max_memory:
    739709            data.options.max_memory = mem_used
    740         print "   Total memory used: ",mem_used
    741         if data.options.profile_memory > 1:
    742             memory_tracker.print_diff(
    743                     summary1=memory_data.summary_after_optimization,
    744                     summary2=memory_data.summary_after_results_processing)
    745 
     710        print "   Total memory = %d bytes following results processing" % mem_used
    746711
    747712@coopr_api(namespace='pyomo.script')
     
    771736        plugin.deactivate()
    772737    data.options._usermodel_plugins = []
    773     #
     738
    774739    if (pympler_available is True) and (data.options.profile_memory >= 1):
    775         final_objects = muppy.get_objects()
    776         summary_final = summary.summarize(final_objects)
    777         final_summary = summary.summarize(final_objects)
    778         print "\nFinal set of objects"
    779         mem_used = muppy.get_size(final_objects)
     740        mem_used = muppy.get_size(muppy.get_objects())
    780741        if mem_used > data.options.max_memory:
    781742            data.options.max_memory = mem_used
    782         print "   Total memory used: ",mem_used
    783         if data.options.profile_memory > 1:
    784             summary.print_(final_summary, limit=50)
    785 
     743        print "   Total memory = %d bytes upon termination" % mem_used
    786744
    787745@coopr_api(namespace='pyomo.script')
     
    805763        sys.stdout.write('[%8.2f] Pyomo Finished\n' % (time.time()-start_time))
    806764        if (pympler_available is True) and (data.options.profile_memory >= 1):
    807             sys.stdout.write('Maximum Memory Used: %d\n' % data.options.max_memory)
     765            sys.stdout.write('Maximum memory used = %d bytes\n' % data.options.max_memory)
    808766        sys.stdout.flush()
    809767    #
Note: See TracChangeset for help on using the changeset viewer.