source: coopr.plugins/trunk/coopr/plugins/mip/CPLEX.py @ 2284

Last change on this file since 2284 was 2284, checked in by jwatson, 10 years ago

Fix to CPLEX plugin relating to suffix processing - working on a laptop, and couldn't test fully!

File size: 25.6 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import os
13import re
14from coopr.opt.base import *
15from coopr.opt.results import *
16from coopr.opt.solver import *
17from coopr.pyomo.base.var import *
18import mockmip
19import pyutilib.services
20import pyutilib.common
21import pyutilib.misc
22import pyutilib.component.core
23import string
24import re
25
26import xml.dom.minidom
27
28import time
29
30
31class CPLEX(ILMLicensedSystemCallSolver):
32    """The CPLEX LP/MIP solver
33    """
34
35    def __init__(self, **kwds):
36        #
37        # Call base class constructor
38        #
39        kwds['type'] = 'cplex'
40        ILMLicensedSystemCallSolver.__init__(self, **kwds)
41
42        # We are currently invoking CPLEX via the command line, with input re-direction. As opposed
43        # to writing our own ilcplex-based C++ driver. Consequently, we need to define an attribute
44        # to retain the execution script name.
45        self.cplex_script_file_name = None
46
47        # NOTE: eventually both of the following attributes should be migrated to a common base class.
48        # is the current solve warm-started? a transient data member to communicate state information
49        # across the _presolve, _apply_solver, and _postsolve methods.
50        self.warm_start_solve = False
51        # related to the above, the temporary name of the MST warm-start file (if any).
52        self.warm_start_file_name = None
53
54        #
55        # Define valid problem formats and associated results formats
56        #
57        self._valid_problem_formats=[ProblemFormat.cpxlp, ProblemFormat.mps]
58        self._valid_result_formats={}
59        self._valid_result_formats[ProblemFormat.cpxlp] = [ResultsFormat.soln]
60        self._valid_result_formats[ProblemFormat.mps] = [ResultsFormat.soln]
61
62    #
63    # ultimately, this utility should go elsewhere - perhaps on the PyomoModel itself.
64    # in the mean time, it is staying here.
65    #
66    def _hasIntegerVariables(self, instance):
67
68       import coopr.pyomo.base.var
69       from coopr.pyomo.base.set_types import IntegerSet, BooleanSet
70
71       for variable in instance.active_components(Var).values():
72
73           if (isinstance(variable.domain, IntegerSet) is True) or (isinstance(variable.domain, BooleanSet) is True):
74
75               return True
76
77       return False
78
79    #
80    # CPLEX has a simple, easy-to-use warm-start capability.
81    #
82    def warm_start_capable(self):
83       return True
84
85    #
86    # write a warm-start file in the CPLEX MST format.
87    #
88    def write_warmstart_file(self, instance):
89
90       import coopr.pyomo.base.var
91
92       self.warm_start_file_name = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.mst')
93
94       doc = xml.dom.minidom.Document()
95       root_element = doc.createElement("CPLEXSolution")
96       root_element.setAttribute("version","1.0")
97       doc.appendChild(root_element)
98
99       # currently not populated.
100       header_element = doc.createElement("header")
101       # currently not populated.       
102       quality_element = doc.createElement("quality")
103       # definitely populated!
104       variables_element = doc.createElement("variables")
105
106       root_element.appendChild(header_element)
107       root_element.appendChild(quality_element)
108       root_element.appendChild(variables_element)
109
110       # for each variable, add a child to the variables element.
111       # both continuous and discrete are accepted (and required,
112       # depending on other options), according to the CPLEX manual.
113       output_index = 0
114       for variable in instance.active_components(Var).values():
115
116           for index in variable._varval.keys():
117
118               if (variable[index].status != coopr.pyomo.base.var.VarStatus.unused) and (variable[index].value != None) and (variable[index].fixed == False):
119
120                   variable_element = doc.createElement("variable")
121                   name = variable[index].label
122                   name = name.replace('[','(')
123                   name = name.replace(']',')')
124                   variable_element.setAttribute("name", name)
125                   variable_element.setAttribute("index", str(output_index))
126                   variable_element.setAttribute("value", str(variable[index].value))
127
128                   variables_element.appendChild(variable_element)
129
130                   output_index = output_index + 1
131
132       mst_file = open(self.warm_start_file_name,'w')
133       doc.writexml(mst_file, indent="    ", newl="\n")
134       mst_file.close()
135
136    # over-ride presolve to extract the warm-start keyword, if specified.
137    def _presolve(self, *args, **kwds):
138
139       # if the first argument is a string (representing a filename),
140       # then we don't have an instance => the solver is being applied
141       # to a file.
142
143       self.warm_start_solve = False               
144       if "warmstart" in kwds:
145          self.warm_start_solve = kwds["warmstart"]
146          del kwds["warmstart"]
147
148       if (len(args) > 0) and (isinstance(args[0],basestring) is False):
149
150          # write the warm-start file - currently only supports MIPs.
151          # we only know how to deal with a single problem instance.       
152          if self.warm_start_solve is True:
153
154             if len(args) != 1:
155                raise ValueError, "CPLEX _presolve method can only handle a single problem instance - "+str(len(args))+" were supplied"                 
156
157             if self._hasIntegerVariables(args[0]) is True:
158                start_time = time.time()
159                self.write_warmstart_file(args[0])
160                end_time = time.time()
161                if self._report_timing is True:
162                   print "Warm start write time="+str(end_time-start_time)+" seconds"
163         
164       # let the base class handle any remaining keywords/actions.
165       ILMLicensedSystemCallSolver._presolve(self, *args, **kwds)
166
167    def executable(self):
168        executable = pyutilib.services.registered_executable("cplex")
169        if executable is None:
170            pyutilib.component.core.PluginGlobals.env().log.error("Could not locate the 'cplex' executable, which is required for solver %s" % self.name)
171            self.enable = False
172            return None
173        return executable.get_path()
174
175    def create_command_line(self,executable,problem_files):
176        #
177        # Define log file
178        # The log file in CPLEX contains the solution trace, but the solver status can be found in the solution file.
179        #
180        self.log_file = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.log')
181
182        #
183        # Define solution file
184        # As indicated above, contains (in XML) both the solution and solver status.
185        #
186        self.soln_file = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.sol')
187
188        #
189        # Define results file
190        #
191        if self._results_format is None or self._results_format == ResultsFormat.soln:
192           self.results_file = self.soln_file
193        elif self._results_format == ResultsFormat.sol:
194           self.results_file = self.sol_file
195
196        #
197        # Write the CPLEX execution script
198        #
199        self.cplex_script_file_name = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.script')
200        cplex_script_file = open(self.cplex_script_file_name,'w')
201        cplex_script_file.write("set logfile "+self.log_file+"\n")
202        if self._timelimit is not None and self._timelimit > 0.0:
203            cplex_script_file.write("set timelimit "+`self._timelimit`+"\n")
204        if (self.mipgap is not None) and (self.mipgap > 0.0):
205            cplex_script_file.write("set mip tolerances mipgap "+`self.mipgap`+"\n")           
206        for key in self.options:
207                if key in ['relax_integrality']:
208                    pass
209                elif isinstance(self.options[key],basestring) and ' ' in self.options[key]:
210                    opt = " ".join(key.split('_'))+" "+str(self.options[key])
211                else:
212                    opt = " ".join(key.split('_'))+" "+str(self.options[key])
213                cplex_script_file.write("set "+opt+"\n")
214        cplex_script_file.write("read "+problem_files[0]+"\n")
215
216        # if we're dealing with an LP, the MST file will be empty.
217        if (self.warm_start_solve is True) and (self.warm_start_file_name is not None):
218            cplex_script_file.write("read "+self.warm_start_file_name+"\n")
219
220        if 'relax_integrality' in self.options:
221            cplex_script_file.write("change problem lp\n")
222           
223        cplex_script_file.write("display problem stats\n")
224        cplex_script_file.write("optimize\n")
225        cplex_script_file.write("write " + self.soln_file+"\n")
226        cplex_script_file.write("quit\n")
227        cplex_script_file.close()
228
229        # dump the script and warm-start file names for the
230        # user if we're keeping files around.
231        if self.keepFiles:
232           print "Solver script file=" + self.cplex_script_file_name
233           if (self.warm_start_solve is True) and (self.warm_start_file_name is not None):
234              print "Solver warm-start file=" + self.warm_start_file_name
235
236        #
237        # Define command line
238        #
239        if self._problem_format in [ProblemFormat.cpxlp, ProblemFormat.mps]:
240           proc = self._timer + " " + self.executable() + " < " + self.cplex_script_file_name
241        return pyutilib.misc.Bunch(cmd=proc, log_file=self.log_file, env=None)
242
243    def process_logfile(self):
244        """
245        Process logfile
246        """
247        results = SolverResults()
248        results.problem.number_of_variables = None
249        results.problem.number_of_nonzeros = None
250        #
251        # Process logfile
252        #
253        OUTPUT = open(self.log_file)
254        output = "".join(OUTPUT.readlines())
255        OUTPUT.close()
256        #
257        # It is generally useful to know the CPLEX version number for logfile parsing.
258        #
259        cplex_version = None
260       
261        #
262        # Parse logfile lines
263        #
264        for line in output.split("\n"):
265            tokens = re.split('[ \t]+',line.strip())
266            if len(tokens) > 3 and tokens[0] == "CPLEX" and tokens[1] == "Error":
267                # IMPT: See below - cplex can generate an error line and then terminate fine, e.g., in CPLEX 12.1.
268                #       To handle these cases, we should be specifying some kind of termination criterion always
269                #       in the course of parsing a log file (we aren't doing so currently - just in some conditions).
270                results.solver.status=SolverStatus.error
271                results.solver.error = " ".join(tokens)
272            elif len(tokens) >= 3 and tokens[0] == "ILOG" and tokens[1] == "CPLEX":
273                cplex_version = tokens[2].rstrip(',')
274            elif len(tokens) >= 3 and tokens[0] == "Variables":
275                if results.problem.number_of_variables is None: # CPLEX 11.2 and subsequent versions have two Variables sections in the log file output.
276                    results.problem.number_of_variables = tokens[2]
277            # In CPLEX 11 (and presumably before), there was only a single line output to
278            # indicate the constriant count, e.g., "Linear constraints : 16 [Less: 7, Greater: 6, Equal: 3]".
279            # In CPLEX 11.2 (or somewhere in between 11 and 11.2 - I haven't bothered to track it down
280            # in that detail), there is another instance of this line prefix in the min/max problem statistics
281            # block - which we don't care about. In this case, the line looks like: "Linear constraints :" and
282            # that's all.
283            elif len(tokens) >= 4 and tokens[0] == "Linear" and tokens[1] == "constraints":
284                results.problem.number_of_constraints = tokens[3]
285            elif len(tokens) >= 3 and tokens[0] == "Nonzeros":
286                if results.problem.number_of_nonzeros is None: # CPLEX 11.2 and subsequent has two Nonzeros sections.               
287                    results.problem.number_of_nonzeros = tokens[2]
288            elif len(tokens) >= 5 and tokens[4] == "MINIMIZE":
289                results.problem.sense = ProblemSense.minimize
290            elif len(tokens) >= 5 and tokens[4] == "MAXIMIZE":
291                results.problem.sense = ProblemSense.maximize
292            elif len(tokens) >= 4 and tokens[0] == "Solution" and tokens[1] == "time" and tokens[2] == "=":
293               # technically, I'm not sure if this is CPLEX user time or user+system - CPLEX doesn't appear
294               # to differentiate, and I'm not sure we can always provide a break-down.
295               results.solver.user_time = eval(tokens[3])
296            elif len(tokens) >= 4 and tokens[0] == "Dual" and tokens[1] == "simplex" and tokens[3] == "Optimal:":
297                results.solver.termination_condition = TerminationCondition.optimal
298                results.solver.termination_message = ' '.join(tokens)
299            elif len(tokens) >= 4 and tokens[0] == "MIP" and tokens[2] == "Integer" and tokens[3] == "infeasible.":
300                # if CPLEX has previously printed an error message, reduce it to a warning -
301                # there is a strong indication it recovered, but we can't be sure.
302                if results.solver.status == SolverStatus.error:
303                   results.solver.status = SolverStatus.warning
304                else:
305                   results.solver.status = SolverStatus.ok
306                results.solver.termination_condition = TerminationCondition.infeasible
307                results.solver.termination_message = ' '.join(tokens)
308            # for the case below, CPLEX sometimes reports "true" optimal (the first case)
309            # and other times within-tolerance optimal (the second case).
310            elif (len(tokens) >= 4 and tokens[0] == "MIP" and tokens[2] == "Integer" and tokens[3] == "optimal") or \
311                 (len(tokens) >= 4 and tokens[0] == "MIP" and tokens[2] == "Integer" and tokens[3] == "optimal,"):
312                # if CPLEX has previously printed an error message, reduce it to a warning -
313                # there is a strong indication it recovered, but we can't be sure.
314                if results.solver.status == SolverStatus.error:
315                   results.solver.status = SolverStatus.warning
316                else:
317                   results.solver.status = SolverStatus.ok               
318                results.solver.termination_condition = TerminationCondition.optimal
319                results.solver.termination_message = ' '.join(tokens)               
320            elif len(tokens) >= 3 and tokens[0] == "Presolve" and tokens[2] == "Infeasible.":
321                # if CPLEX has previously printed an error message, reduce it to a warning -
322                # there is a strong indication it recovered, but we can't be sure.
323                if results.solver.status == SolverStatus.error:
324                   results.solver.status = SolverStatus.warning
325                else:
326                   results.solver.status = SolverStatus.ok               
327                results.solver.termination_condition = TerminationCondition.infeasible
328                results.solver.termination_message = ' '.join(tokens)
329            elif len(tokens) >= 5 and tokens[0] == "Presolve" and tokens[2] == "Unbounded" and tokens[4] == "infeasible.":
330                # if CPLEX has previously printed an error message, reduce it to a warning -
331                # there is a strong indication it recovered, but we can't be sure.
332                if results.solver.status == SolverStatus.error:
333                   results.solver.status = SolverStatus.warning
334                else:
335                   results.solver.status = SolverStatus.ok               
336                # It isn't clear whether we can determine if the problem is unbounded from
337                # CPLEX's output.
338                results.solver.termination_condition = TerminationCondition.unbounded
339                results.solver.termination_message = ' '.join(tokens)
340        return results
341
342    def process_soln_file(self,results):
343
344        # the only suffixes that we extract from CPLEX are
345        # constraint duals, constraint slacks, and variable
346        # reduced-costs. scan through the solver suffix list
347        # and throw an exception if the user has specified
348        # any others.
349        extract_duals = False
350        extract_slacks = False
351        extract_reduced_costs = False
352        for suffix in self.suffixes:
353           if suffix == "dual":
354              extract_duals = True
355           elif suffix == "slack":
356              extract_slacks = True             
357           elif suffix == "rc":
358              extract_reduced_costs = True
359           else:
360              raise RuntimeError,"***CPLEX solver plugin cannot extract solution suffix="+suffix
361
362        lp_solution = False
363        if not os.path.exists(self.soln_file):
364           return
365
366        soln = Solution()
367        soln.objective['f'].value=None
368        INPUT = open(self.soln_file,"r")
369        results.problem.number_of_objectives=1
370        mip_problem=False
371        for line in INPUT:
372            line = line.strip()
373            line = line.lstrip('<?/')
374            line = line.rstrip('/>?')
375            tokens=line.split(' ')
376
377            if tokens[0] == "variable":
378                variable_name = None
379                variable_value = None
380                variable_reduced_cost = None
381                variable_status = None
382                for i in range(1,len(tokens)):
383                   field_name =  string.strip(tokens[i].split('=')[0])
384                   field_value = (string.strip(tokens[i].split('=')[1])).lstrip("\"").rstrip("\"")
385                   if field_name == "name":
386                      variable_name = field_value
387                   elif field_name == "value":
388                      variable_value = field_value
389                   elif (extract_reduced_costs is True) and (field_name == "reducedCost"):
390                      variable_reduced_cost = field_value
391                   elif (extract_reduced_costs is True) and (field_name == "status"):
392                      variable_status = field_value
393
394                # skip the "constant-one" variable, used to capture/retain objective offsets in the CPLEX LP format.
395                if variable_name != "ONE_VAR_CONSTANT":
396                   variable = None # cache the solution variable reference, as the getattr is expensive.
397                   try:
398                       variable = soln.variable[variable_name]
399                       variable.value = eval(variable_value)
400                   except:
401                       variable.value = variable_value
402                   if (variable_reduced_cost is not None) and (extract_reduced_costs is True):
403                        try:
404                            variable.rc = eval(variable_reduced_cost)
405                            if variable_status is not None:
406                               if variable_status == "LL":
407                                  variable.lrc = eval(variable_reduced_cost)
408                               else:
409                                  variable.lrc = 0.0
410                               if variable_status == "UL":
411                                  variable.urc = eval(variable_reduced_cost)
412                               else:
413                                  variable.urc = 0.0
414                        except:
415                            raise ValueError, "Unexpected reduced-cost value="+str(variable_reduced_cost)+" encountered for variable="+variable_name
416            elif (tokens[0] == "constraint") and ((extract_duals is True) or (extract_slacks is True)):
417                constraint_name = None
418                constraint_dual = None
419                constaint = None # cache the solution constraint reference, as the getattr is expensive.
420                for i in range(1,len(tokens)):
421                   field_name =  string.strip(tokens[i].split('=')[0])
422                   field_value = (string.strip(tokens[i].split('=')[1])).lstrip("\"").rstrip("\"")
423                   if field_name == "name":
424                      constraint_name = field_value
425                      constraint = soln.constraint[constraint_name]
426                   elif (extract_duals is True) and (field_name == "dual"): # for LPs
427                      # assumes the name field is first.
428                      if eval(field_value) != 0.0:
429                        constraint.dual = eval(field_value)
430                   elif (extract_slacks is True) and (field_name == "slack"): # for MIPs
431                      # assumes the name field is first.
432                      if eval(field_value) != 0.0:
433                        constraint.slack = eval(field_value)
434            elif tokens[0].startswith("problemName"):
435                filename = (string.strip(tokens[0].split('=')[1])).lstrip("\"").rstrip("\"")
436                #print "HERE",filename
437                results.problem.name = os.path.basename(filename)
438                if '.' in results.problem.name:
439                    results.problem.name = results.problem.name.split('.')[0]
440                tINPUT=open(filename,"r")
441                for tline in tINPUT:
442                    tline = tline.strip()
443                    if tline == "":
444                        continue
445                    tokens = re.split('[\t ]+',tline)
446                    if tokens[0][0] in ['\\', '*']:
447                        continue
448                    elif tokens[0] == "NAME":
449                        results.problem.name = tokens[1]
450                    else:
451                        sense = tokens[0].lower()
452                        if sense in ['max','maximize']:
453                            results.problem.sense = ProblemSense.maximize
454                        if sense in ['min','minimize']:
455                            results.problem.sense = ProblemSense.minimize
456                    break
457                tINPUT.close()
458               
459            elif tokens[0].startswith("objectiveValue"):
460                objective_value = (string.strip(tokens[0].split('=')[1])).lstrip("\"").rstrip("\"")
461                soln.objective['f'].value = objective_value
462            elif tokens[0].startswith("solutionStatusString"):
463                solution_status = (string.strip(" ".join(tokens).split('=')[1])).lstrip("\"").rstrip("\"")
464                if solution_status in ["optimal", "integer optimal solution", "integer optimal, tolerance"]:
465                    soln.status = SolutionStatus.optimal
466                    soln.gap = 0.0
467                    if results.problem.sense == ProblemSense.minimize:
468                        results.problem.lower_bound = soln.objective['f'].value
469                        if "upper_bound" in dir(results.problem):
470                            del results.problem.upper_bound
471                    else:
472                        results.problem.upper_bound = soln.objective['f'].value
473                        if "lower_bound" in dir(results.problem):
474                            del results.problem.lower_bound
475                    mip_problem=True
476            elif tokens[0].startswith("MIPNodes"):
477                if mip_problem:
478                    n = eval(string.strip(" ".join(tokens).split('=')[1])).lstrip("\"").rstrip("\"")
479                    results.solver.statistics.branch_and_bound.number_of_created_subproblems=n
480                    results.solver.statistics.branch_and_bound.number_of_bounded_subproblems=n
481
482               
483        if not results.solver.status is SolverStatus.error:
484            results.solution.insert(soln)
485        INPUT.close()
486
487    def _postsolve(self):
488
489        # take care of the annoying (and empty) CPLEX temporary files in the current directory.
490        # this approach doesn't seem overly efficient, but python os module functions don't
491        # accept regular expression directly.
492        filename_list = os.listdir(".")
493        for filename in filename_list:
494           # CPLEX temporary files come in two flavors - cplex.log and clone*.log.
495           # the latter is the case for multi-processor environments.
496           # IMPT: trap the possible exception raised by the file not existing.
497           #       this can occur in pyro environments where > 1 workers are
498           #       running CPLEX, and were started from the same directory.
499           #       these logs don't matter anyway (we redirect everything),
500           #       and are largely an annoyance.
501           try:
502              if  re.match('cplex\.log', filename) != None:
503                  os.remove(filename)
504              elif re.match('clone\d+\.log', filename) != None:
505                  os.remove(filename)
506           except OSError:
507              pass
508
509        # let the base class deal with returning results.
510        return ILMLicensedSystemCallSolver._postsolve(self)           
511
512
513class MockCPLEX(CPLEX,mockmip.MockMIP):
514    """A Mock CPLEX solver used for testing
515    """
516
517    def __init__(self, **kwds):
518        try:
519           CPLEX.__init__(self, **kwds)
520        except pyutilib.common.ApplicationError: #pragma:nocover
521           pass                        #pragma:nocover
522        mockmip.MockMIP.__init__(self,"cplex")
523
524    def available(self, exception_flag=True):
525        return CPLEX.available(self,exception_flag)
526
527    def create_command_line(self,executable,problem_files):
528        command = CPLEX.create_command_line(self,executable,problem_files)
529        mockmip.MockMIP.create_command_line(self,executable,problem_files)
530        return command
531
532    def executable(self):
533        return mockmip.MockMIP.executable(self)
534
535    def _execute_command(self,cmd):
536        return mockmip.MockMIP._execute_command(self,cmd)
537
538
539pyutilib.services.register_executable(name="cplex")
540SolverRegistration("cplex", CPLEX)
541SolverRegistration("_mock_cplex", MockCPLEX)
Note: See TracBrowser for help on using the repository browser.