Changeset 2260


Ignore:
Timestamp:
Feb 4, 2010 11:44:29 PM (10 years ago)
Author:
jwatson
Message:

Refactoring of ph initialization routines to support sampling and bundling.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • coopr.pysp/trunk/coopr/pysp/ph_script.py

    r2255 r2260  
    282282
    283283#
    284 # The main PH initialization / runner routine.
    285 #
    286 
    287 def run_ph(options, args):
    288 
     284# Create the reference model / instance and scenario tree instance for PH.
     285#
     286
     287def load_reference_and_scenario_models(options):
     288
     289   #
     290   # create and populate the reference model/instance pair.
     291   #
     292
     293   reference_model = None
     294   reference_instance = None
     295
     296   try:
     297      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
     298      if options.verbose is True:
     299         print "Scenario reference model filename="+reference_model_filename
     300      model_import = pyutilib.misc.import_file(reference_model_filename)
     301      if "model" not in dir(model_import):
     302         print ""
     303         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
     304         return
     305
     306      if model_import.model is None:
     307         print ""
     308         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
     309         return
     310 
     311      reference_model = model_import.model
     312   except IOError:
     313      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
     314      return
     315
     316   try:
     317      reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
     318      if options.verbose is True:
     319         print "Scenario reference instance filename="+reference_instance_filename
     320      reference_instance = reference_model.create(reference_instance_filename)
     321   except IOError:
     322      print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
     323      return     
     324
     325   #
     326   # create and populate the scenario tree model
     327   #
     328
     329   from coopr.pysp.util.scenariomodels import scenario_tree_model
     330   scenario_tree_instance = None
     331
     332   try:
     333      scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
     334      if options.verbose is True:
     335         print "Scenario tree instance filename="+scenario_tree_instance_filename
     336      scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
     337   except IOError:
     338      print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
     339      return
     340   
     341   #
     342   # construct the scenario tree
     343   #
     344   scenario_tree = ScenarioTree(model=reference_instance,
     345                                nodes=scenario_tree_instance.Nodes,
     346                                nodechildren=scenario_tree_instance.Children,
     347                                nodestages=scenario_tree_instance.NodeStage,
     348                                nodeprobabilities=scenario_tree_instance.ConditionalProbability,
     349                                stages=scenario_tree_instance.Stages,
     350                                stagevariables=scenario_tree_instance.StageVariables,
     351                                stagecostvariables=scenario_tree_instance.StageCostVariable,
     352                                scenarios=scenario_tree_instance.Scenarios,
     353                                scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
     354                                scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
     355
     356   return reference_model, reference_instance, scenario_tree
     357
     358#
     359# Create a PH object from a checkpoint.
     360#
     361def create_ph_from_checkpoint(options):
     362
     363   # we need to load the reference model, as pickle doesn't save contents of .py files!
     364   try:
     365      reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
     366      if options.verbose is True:
     367         print "Scenario reference model filename="+reference_model_filename
     368      model_import = pyutilib.misc.import_file(reference_model_filename)
     369      if "model" not in dir(model_import):
     370         print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
     371         return
     372
     373      if model_import.model is None:
     374         print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
     375         return None
     376 
     377      reference_model = model_import.model
     378   except IOError:
     379      print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
     380      return None
     381
     382   # import the saved state
     383     
     384   try:
     385      checkpoint_file = open(options.restore_from_checkpoint,"r")
     386      ph = pickle.load(checkpoint_file)
     387      checkpoint_file.close()
     388     
     389   except IOError, msg:
     390      raise RuntimeError, msg
     391
     392   # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
     393
     394   raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
     395
     396   return ph
     397
     398#
     399# Create a PH object from a checkpoint.
     400#
     401def create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree):
     402
     403   #
     404   # print the input tree for validation/information purposes.
     405   #
     406   if options.verbose is True:
     407      scenario_tree.pprint()
     408
     409   #
     410   # validate the tree prior to doing anything serious
     411   #
     412   print ""
     413   if scenario_tree.validate() is False:
     414      print "***ERROR: Scenario tree is invalid****"
     415      return None
     416   else:
     417      if options.verbose is True:
     418         print "Scenario tree is valid!"
     419   print ""
     420
     421   #
     422   # if any of the ww extension configuration options are specified without the
     423   # ww extension itself being enabled, halt and warn the user - this has led
     424   # to confusion in the past, and will save user support time.
     425   #
     426   if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
     427      print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
     428      return None
     429
     430   if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
     431      print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"         
     432      return None
     433
     434   #
     435   # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
     436   #
     437   if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
     438      print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
     439      return None
     440
     441   #
     442   # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
     443   # order only matters if both are specified.
     444   #
     445   if options.enable_ww_extensions is True:
     446
     447      from coopr.pysp import wwphextension
     448
     449      plugin = ExtensionPoint(IPHExtension)
     450      if len(options.ww_extension_cfgfile) > 0:
     451         plugin.service()._configuration_filename = options.ww_extension_cfgfile
     452      if len(options.ww_extension_suffixfile) > 0:
     453         plugin.service()._suffix_filename = options.ww_extension_suffixfile
     454
     455   if options.user_defined_extension is not None:
     456      print "Trying to import user-defined PH extension module="+options.user_defined_extension
     457      # JPW removed the exception handling logic, as the module importer
     458      # can raise a broad array of exceptions.
     459      __import__(options.user_defined_extension)
     460      print "Module successfully loaded"
     461
     462   #
     463   # construct the convergence "computer" class.
     464   #
     465   converger = None
     466   # go with the non-defaults first, and then with the default.
     467   if options.enable_free_discrete_count_convergence is True:
     468      converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
     469   elif options.enable_normalized_termdiff_convergence is True:
     470      converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
     471   else:
     472      converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
     473
     474   
     475   #
     476   # construct and initialize PH
     477   #
     478   ph = ProgressiveHedging(max_iterations=options.max_iterations, \
     479                           rho=options.default_rho, \
     480                           rho_setter=options.rho_cfgfile, \
     481                           bounds_setter=options.bounds_cfgfile, \
     482                           solver=options.solver_type, \
     483                           solver_manager=options.solver_manager_type, \
     484                           scenario_solver_options=options.scenario_solver_options, \
     485                           scenario_mipgap=options.scenario_mipgap, \
     486                           keep_solver_files=options.keep_solver_files, \
     487                           output_solver_log=options.output_solver_logs, \
     488                           output_solver_results=options.output_solver_results, \
     489                           verbose=options.verbose, \
     490                           report_solutions=options.report_solutions, \
     491                           report_weights=options.report_weights, \
     492                           output_times=options.output_times, \
     493                           disable_warmstarts=options.disable_warmstarts,
     494                           drop_proximal_terms=options.drop_proximal_terms,
     495                           retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
     496                           linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
     497                           breakpoint_strategy=options.breakpoint_strategy, \
     498                           checkpoint_interval=options.checkpoint_interval)
     499
     500   ph.initialize(scenario_data_directory_name=os.path.expanduser(options.instance_directory), \
     501                 model=reference_model, \
     502                 model_instance=reference_instance, \
     503                 scenario_tree=scenario_tree, \
     504                 converger=converger)
     505
     506   if options.suppress_continuous_variable_output is True:
     507      ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.
     508
     509   return ph
     510
     511#
     512# Given a PH object, execute it and optionally solve the EF at the end.
     513#
     514
     515def run_ph(options, ph):
     516
     517   #
     518   # at this point, we have an initialized PH object by some means.
     519   #
    289520   start_time = time.time()
    290521
    291    ph = None
    292 
    293    # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
    294    if len(options.restore_from_checkpoint) > 0:
    295 
    296       # we need to load the reference model, as pickle doesn't save contents of .py files!
    297       try:
    298          reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
    299          if options.verbose is True:
    300             print "Scenario reference model filename="+reference_model_filename
    301          model_import = pyutilib.misc.import_file(reference_model_filename)
    302          if "model" not in dir(model_import):
    303             print ""
    304             print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
    305             return
    306 
    307          if model_import.model is None:
    308             print ""
    309             print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
    310             return
    311  
    312          reference_model = model_import.model
    313       except IOError:
    314          print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
    315          return     
    316 
    317       # import the saved state
    318      
    319       try:
    320          checkpoint_file = open(options.restore_from_checkpoint,"r")
    321          ph = pickle.load(checkpoint_file)
    322          checkpoint_file.close()
    323          
    324 #         print "PH=",ph
    325 #         ph._scenario_tree.pprint()         
    326       except IOError, msg:
    327          raise RuntimeError, msg
    328 
    329       # tell PH to build the right solver manager and solver TBD - AND PLUGINS, BUT LATER
    330 
    331       raise RuntimeError, "Checkpoint restoration is not fully supported/tested yet!"
    332      
    333    else:
    334       #
    335       # create and populate the reference model/instance pair.
    336       #
    337 
    338       reference_model = None
    339       reference_instance = None
    340 
    341       try:
    342          reference_model_filename = os.path.expanduser(options.model_directory)+os.sep+"ReferenceModel.py"
    343          if options.verbose is True:
    344             print "Scenario reference model filename="+reference_model_filename
    345          model_import = pyutilib.misc.import_file(reference_model_filename)
    346          if "model" not in dir(model_import):
    347             print ""
    348             print "***ERROR: Exiting test driver: No 'model' object created in module "+reference_model_filename
    349             return
    350 
    351          if model_import.model is None:
    352             print ""
    353             print "***ERROR: Exiting test driver: 'model' object equals 'None' in module "+reference_model_filename
    354             return
    355  
    356          reference_model = model_import.model
    357       except IOError:
    358          print "***ERROR: Failed to load scenario reference model from file="+reference_model_filename
    359          return
    360 
    361       try:
    362          reference_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ReferenceModel.dat"
    363          if options.verbose is True:
    364             print "Scenario reference instance filename="+reference_instance_filename
    365          reference_instance = reference_model.create(reference_instance_filename)
    366       except IOError:
    367          print "***ERROR: Failed to load scenario reference instance data from file="+reference_instance_filename
    368          return     
    369 
    370       #
    371       # create and populate the scenario tree model
    372       #
    373 
    374       from coopr.pysp.util.scenariomodels import scenario_tree_model
    375       scenario_tree_instance = None
    376 
    377       try:
    378          scenario_tree_instance_filename = os.path.expanduser(options.instance_directory)+os.sep+"ScenarioStructure.dat"
    379          if options.verbose is True:
    380             print "Scenario tree instance filename="+scenario_tree_instance_filename
    381          scenario_tree_instance = scenario_tree_model.create(scenario_tree_instance_filename)
    382       except IOError:
    383          print "***ERROR: Failed to load scenario tree reference instance data from file="+scenario_tree_instance_filename
    384          return
    385    
    386       #
    387       # construct the scenario tree
    388       #
    389       scenario_tree = ScenarioTree(model=reference_instance,
    390                                    nodes=scenario_tree_instance.Nodes,
    391                                    nodechildren=scenario_tree_instance.Children,
    392                                    nodestages=scenario_tree_instance.NodeStage,
    393                                    nodeprobabilities=scenario_tree_instance.ConditionalProbability,
    394                                    stages=scenario_tree_instance.Stages,
    395                                    stagevariables=scenario_tree_instance.StageVariables,
    396                                    stagecostvariables=scenario_tree_instance.StageCostVariable,
    397                                    scenarios=scenario_tree_instance.Scenarios,
    398                                    scenarioleafs=scenario_tree_instance.ScenarioLeafNode,
    399                                    scenariobaseddata=scenario_tree_instance.ScenarioBasedData)
    400 
    401       #
    402       # print the input tree for validation/information purposes.
    403       #
    404       if options.verbose is True:
    405          scenario_tree.pprint()
    406 
    407       #
    408       # validate the tree prior to doing anything serious
    409       #
    410       print ""
    411       if scenario_tree.validate() is False:
    412          print "***ERROR: Scenario tree is invalid****"
    413          return
    414       else:
    415          if options.verbose is True:
    416             print "Scenario tree is valid!"
    417       print ""
    418 
    419       #
    420       # if any of the ww extension configuration options are specified without the
    421       # ww extension itself being enabled, halt and warn the user - this has led
    422       # to confusion in the past, and will save user support time.
    423       #
    424       if len(options.ww_extension_cfgfile) > 0 and options.enable_ww_extensions is False:
    425          print "***ERROR: A configuration file was specified for the WW extension module, but the WW extensions are not enabled!"
    426          return
    427 
    428       if len(options.ww_extension_suffixfile) > 0 and options.enable_ww_extensions is False:
    429          print "***ERROR: A suffix file was specified for the WW extension module, but the WW extensions are not enabled!"         
    430          return
    431 
    432       #
    433       # if a breakpoint strategy is specified without linearization eanbled, halt and warn the user.
    434       #
    435       if (options.breakpoint_strategy > 0) and (options.linearize_nonbinary_penalty_terms == 0):
    436          print "***ERROR: A breakpoint distribution strategy was specified, but linearization is not enabled!"
    437          return         
    438 
    439       #
    440       # deal with any plugins. ww extension comes first currently, followed by an option user-defined plugin.
    441       # order only matters if both are specified.
    442       #
    443       if options.enable_ww_extensions is True:
    444 
    445          from coopr.pysp import wwphextension
    446 
    447          plugin = ExtensionPoint(IPHExtension)
    448          if len(options.ww_extension_cfgfile) > 0:
    449             plugin.service()._configuration_filename = options.ww_extension_cfgfile
    450          if len(options.ww_extension_suffixfile) > 0:
    451             plugin.service()._suffix_filename = options.ww_extension_suffixfile
    452 
    453       if options.user_defined_extension is not None:
    454          print "Trying to import user-defined PH extension module="+options.user_defined_extension
    455          # JPW removed the exception handling logic, as the module importer
    456          # can raise a broad array of exceptions.
    457          __import__(options.user_defined_extension)
    458          print "Module successfully loaded"
    459 
    460       #
    461       # construct the convergence "computer" class.
    462       #
    463       converger = None
    464       # go with the non-defaults first, and then with the default.
    465       if options.enable_free_discrete_count_convergence is True:
    466          converger = NumFixedDiscreteVarConvergence(convergence_threshold=options.free_discrete_count_threshold)
    467       elif options.enable_normalized_termdiff_convergence is True:
    468          converger = NormalizedTermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
    469       else:
    470          converger = TermDiffConvergence(convergence_threshold=options.termdiff_threshold)     
    471 
    472      
    473       #
    474       # construct and initialize PH
    475       #
    476       ph = ProgressiveHedging(max_iterations=options.max_iterations, \
    477                               rho=options.default_rho, \
    478                               rho_setter=options.rho_cfgfile, \
    479                               bounds_setter=options.bounds_cfgfile, \
    480                               solver=options.solver_type, \
    481                               solver_manager=options.solver_manager_type, \
    482                               scenario_solver_options=options.scenario_solver_options, \
    483                               scenario_mipgap=options.scenario_mipgap, \
    484                               keep_solver_files=options.keep_solver_files, \
    485                               output_solver_log=options.output_solver_logs, \
    486                               output_solver_results=options.output_solver_results, \
    487                               verbose=options.verbose, \
    488                               report_solutions=options.report_solutions, \
    489                               report_weights=options.report_weights, \
    490                               output_times=options.output_times, \
    491                               disable_warmstarts=options.disable_warmstarts,
    492                               drop_proximal_terms=options.drop_proximal_terms,
    493                               retain_quadratic_binary_terms=options.retain_quadratic_binary_terms, \
    494                               linearize_nonbinary_penalty_terms=options.linearize_nonbinary_penalty_terms, \
    495                               breakpoint_strategy=options.breakpoint_strategy, \
    496                               checkpoint_interval=options.checkpoint_interval)
    497    
    498       ph.initialize(scenario_data_directory_name=os.path.expanduser(options.instance_directory), \
    499                     model=reference_model, \
    500                     model_instance=reference_instance, \
    501                     scenario_tree=scenario_tree, \
    502                     converger=converger)
    503 
    504       if options.suppress_continuous_variable_output is True:
    505          ph._output_continuous_variable_stats = False # clutters up the screen, when we really only care about the binaries.     
    506 
    507    #
    508    # at this point, we have a PH object by some means.
    509    #
    510 
    511522   #
    512523   # kick off the solve
     
    514525   ph.solve()
    515526
     527   end_time = time.time()
     528
    516529   print ""
    517    print "DONE..."
    518 
    519    end_time = time.time()
    520 
    521    print ""
    522    print "Total execution time=%8.2f seconds" %(end_time - start_time)
     530   print "Total PH execution time=%8.2f seconds" %(end_time - start_time)
    523531   print ""
    524532   if options.output_times is True:
     
    564572      print "Extensive form solve results:"
    565573      ef_results.write(num=1)
     574   
     575#
     576# The main PH initialization / runner routine.
     577#
     578
     579def exec_ph(options):
     580
     581   ph = None
     582
     583   # if we are restoring from a checkpoint file, do so - otherwise, construct PH from scratch.
     584   if len(options.restore_from_checkpoint) > 0:
     585
     586      ph = create_ph_from_checkpoint(options)
     587     
     588   else:
     589
     590      reference_model, reference_instance, scenario_tree = load_reference_and_scenario_models(options)
     591      ph = create_ph_from_scratch(options, reference_model, reference_instance, scenario_tree)
     592
     593   if ph is None:
     594      print "***FAILED TO CREATE PH OBJECT"
     595      return
     596
     597   run_ph(options, ph)
    566598
    567599def run(args=None):
     
    596628        #
    597629        tfile = pyutilib.services.TempfileManager.create_tempfile(suffix=".profile")
    598         tmp = cProfile.runctx('run_ph(options,args)',globals(),locals(),tfile)
     630        tmp = cProfile.runctx('exec_ph(options,args)',globals(),locals(),tfile)
    599631        p = pstats.Stats(tfile).strip_dirs()
    600632        p.sort_stats('time', 'cum')
     
    616648        # Call the main PH routine without profiling.
    617649        #
    618         ans = run_ph(options, args)
     650        ans = exec_ph(options)
    619651
    620652    gc.enable()
Note: See TracChangeset for help on using the changeset viewer.