source: coopr.plugins/stable/2.2/coopr/plugins/mip/CPLEX.py @ 2195

Last change on this file since 2195 was 2195, checked in by wehart, 9 years ago

Merged revisions 2112-2194 via svnmerge from
https://software.sandia.gov/svn/public/coopr/coopr.plugins/trunk

........

r2168 | jwatson | 2010-01-24 14:07:25 -0700 (Sun, 24 Jan 2010) | 3 lines


Improvements to the CPLEX plugin, mainly in the area of reporting solver status.

........

r2171 | wehart | 2010-01-25 16:56:28 -0700 (Mon, 25 Jan 2010) | 2 lines


Update CPLEX baselines.

........

r2187 | wehart | 2010-01-27 17:40:16 -0700 (Wed, 27 Jan 2010) | 2 lines


A draft NLOPT solver.

........

File size: 24.6 KB
Line 
1#  _________________________________________________________________________
2#
3#  Coopr: A COmmon Optimization Python Repository
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the Coopr README.txt file.
9#  _________________________________________________________________________
10
11
12import os
13import re
14from coopr.opt.base import *
15from coopr.opt.results import *
16from coopr.opt.solver import *
17from coopr.pyomo.base.var import *
18import mockmip
19import pyutilib.services
20import pyutilib.common
21import pyutilib.misc
22import pyutilib.plugin.core
23import string
24import re
25
26import xml.dom.minidom
27
28import time
29
30
31class CPLEX(ILMLicensedSystemCallSolver):
32    """The CPLEX LP/MIP solver
33    """
34
35    def __init__(self, **kwds):
36        #
37        # Call base class constructor
38        #
39        kwds['type'] = 'cplex'
40        ILMLicensedSystemCallSolver.__init__(self, **kwds)
41
42        # We are currently invoking CPLEX via the command line, with input re-direction. As opposed
43        # to writing our own ilcplex-based C++ driver. Consequently, we need to define an attribute
44        # to retain the execution script name.
45        self.cplex_script_file_name = None
46
47        # NOTE: eventually both of the following attributes should be migrated to a common base class.
48        # is the current solve warm-started? a transient data member to communicate state information
49        # across the _presolve, _apply_solver, and _postsolve methods.
50        self.warm_start_solve = False
51        # related to the above, the temporary name of the MST warm-start file (if any).
52        self.warm_start_file_name = None
53
54        #
55        # Define valid problem formats and associated results formats
56        #
57        self._valid_problem_formats=[ProblemFormat.cpxlp, ProblemFormat.mps]
58        self._valid_result_formats={}
59        self._valid_result_formats[ProblemFormat.cpxlp] = [ResultsFormat.soln]
60        self._valid_result_formats[ProblemFormat.mps] = [ResultsFormat.soln]
61
62    #
63    # ultimately, this utility should go elsewhere - perhaps on the PyomoModel itself.
64    # in the mean time, it is staying here.
65    #
66    def _hasIntegerVariables(self, instance):
67
68       import coopr.pyomo.base.var
69       from coopr.pyomo.base.set_types import IntegerSet, BooleanSet
70
71       for variable in instance.active_components(Var).values():
72
73           if (isinstance(variable.domain, IntegerSet) is True) or (isinstance(variable.domain, BooleanSet) is True):
74
75               return True
76
77       return False
78
79    #
80    # CPLEX has a simple, easy-to-use warm-start capability.
81    #
82    def warm_start_capable(self):
83       return True
84
85    #
86    # write a warm-start file in the CPLEX MST format.
87    #
88    def write_warmstart_file(self, instance):
89
90       import coopr.pyomo.base.var
91
92       self.warm_start_file_name = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.mst')
93
94       doc = xml.dom.minidom.Document()
95       root_element = doc.createElement("CPLEXSolution")
96       root_element.setAttribute("version","1.0")
97       doc.appendChild(root_element)
98
99       # currently not populated.
100       header_element = doc.createElement("header")
101       # currently not populated.       
102       quality_element = doc.createElement("quality")
103       # definitely populated!
104       variables_element = doc.createElement("variables")
105
106       root_element.appendChild(header_element)
107       root_element.appendChild(quality_element)
108       root_element.appendChild(variables_element)
109
110       # for each variable, add a child to the variables element.
111       # both continuous and discrete are accepted (and required,
112       # depending on other options), according to the CPLEX manual.
113       output_index = 0
114       for variable in instance.active_components(Var).values():
115
116           for index in variable._varval.keys():
117
118               if (variable[index].status != coopr.pyomo.base.var.VarStatus.unused) and (variable[index].value != None) and (variable[index].fixed == False):
119
120                   variable_element = doc.createElement("variable")
121                   name = variable[index].label
122                   name = name.replace('[','(')
123                   name = name.replace(']',')')
124                   variable_element.setAttribute("name", name)
125                   variable_element.setAttribute("index", str(output_index))
126                   variable_element.setAttribute("value", str(variable[index].value))
127
128                   variables_element.appendChild(variable_element)
129
130                   output_index = output_index + 1
131
132       mst_file = open(self.warm_start_file_name,'w')
133       doc.writexml(mst_file, indent="    ", newl="\n")
134       mst_file.close()
135
136    # over-ride presolve to extract the warm-start keyword, if specified.
137    def _presolve(self, *args, **kwds):
138
139       # if the first argument is a string (representing a filename),
140       # then we don't have an instance => the solver is being applied
141       # to a file.
142
143       self.warm_start_solve = False               
144       if "warmstart" in kwds:
145          self.warm_start_solve = kwds["warmstart"]
146          del kwds["warmstart"]
147
148       if (len(args) > 0) and (isinstance(args[0],basestring) is False):
149
150          # write the warm-start file - currently only supports MIPs.
151          # we only know how to deal with a single problem instance.       
152          if self.warm_start_solve is True:
153
154             if len(args) != 1:
155                raise ValueError, "CPLEX _presolve method can only handle a single problem instance - "+str(len(args))+" were supplied"                 
156
157             if self._hasIntegerVariables(args[0]) is True:
158                start_time = time.time()
159                self.write_warmstart_file(args[0])
160                end_time = time.time()
161                print "Warm start write time="+str(end_time-start_time)+" seconds"
162         
163       # let the base class handle any remaining keywords/actions.
164       ILMLicensedSystemCallSolver._presolve(self, *args, **kwds)
165
166    def executable(self):
167        executable = pyutilib.services.registered_executable("cplex")
168        if executable is None:
169            pyutilib.plugin.core.PluginGlobals.env().log.error("Could not locate the 'cplex' executable, which is required for solver %s" % self.name)
170            self.enable = False
171            return None
172        return executable.get_path()
173
174    def create_command_line(self,executable,problem_files):
175        #
176        # Define log file
177        # The log file in CPLEX contains the solution trace, but the solver status can be found in the solution file.
178        #
179        self.log_file = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.log')
180
181        #
182        # Define solution file
183        # As indicated above, contains (in XML) both the solution and solver status.
184        #
185        self.soln_file = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.sol')
186
187        #
188        # Define results file
189        #
190        if self._results_format is None or self._results_format == ResultsFormat.soln:
191           self.results_file = self.soln_file
192        elif self._results_format == ResultsFormat.sol:
193           self.results_file = self.sol_file
194
195        #
196        # Write the CPLEX execution script
197        #
198        self.cplex_script_file_name = pyutilib.services.TempfileManager.create_tempfile(suffix = '.cplex.script')
199        cplex_script_file = open(self.cplex_script_file_name,'w')
200        cplex_script_file.write("set logfile "+self.log_file+"\n")
201        if self._timelimit is not None and self._timelimit > 0.0:
202            cplex_script_file.write("set timelimit "+`self._timelimit`+"\n")
203        if (self.mipgap is not None) and (self.mipgap > 0.0):
204            cplex_script_file.write("set mip tolerances mipgap "+`self.mipgap`+"\n")           
205        for key in self.options:
206                if key in ['relax_integrality']:
207                    pass
208                elif isinstance(self.options[key],basestring) and ' ' in self.options[key]:
209                    opt = " ".join(key.split('_'))+" "+str(self.options[key])
210                else:
211                    opt = " ".join(key.split('_'))+" "+str(self.options[key])
212                cplex_script_file.write("set "+opt+"\n")
213        cplex_script_file.write("read "+problem_files[0]+"\n")
214
215        # if we're dealing with an LP, the MST file will be empty.
216        if (self.warm_start_solve is True) and (self.warm_start_file_name is not None):
217            cplex_script_file.write("read "+self.warm_start_file_name+"\n")
218
219        if 'relax_integrality' in self.options:
220            cplex_script_file.write("change problem lp\n")
221           
222        cplex_script_file.write("display problem stats\n")
223        cplex_script_file.write("optimize\n")
224        cplex_script_file.write("write " + self.soln_file+"\n")
225        cplex_script_file.write("quit\n")
226        cplex_script_file.close()
227
228        # dump the script and warm-start file names for the
229        # user if we're keeping files around.
230        if self.keepFiles:
231           print "Solver script file=" + self.cplex_script_file_name
232           if (self.warm_start_solve is True) and (self.warm_start_file_name is not None):
233              print "Solver warm-start file=" + self.warm_start_file_name
234
235        #
236        # Define command line
237        #
238        if self._problem_format in [ProblemFormat.cpxlp, ProblemFormat.mps]:
239           proc = self._timer + " " + self.executable() + " < " + self.cplex_script_file_name
240        return pyutilib.misc.Bunch(cmd=proc, log_file=self.log_file, env=None)
241
242    def process_logfile(self):
243        """
244        Process logfile
245        """
246        results = SolverResults()
247        results.problem.number_of_variables = None
248        results.problem.number_of_nonzeros = None
249        #
250        # Process logfile
251        #
252        OUTPUT = open(self.log_file)
253        output = "".join(OUTPUT.readlines())
254        OUTPUT.close()
255        #
256        # It is generally useful to know the CPLEX version number for logfile parsing.
257        #
258        cplex_version = None
259       
260        #
261        # Parse logfile lines
262        #
263        for line in output.split("\n"):
264            tokens = re.split('[ \t]+',line.strip())
265            if len(tokens) > 3 and tokens[0] == "CPLEX" and tokens[1] == "Error":
266                # IMPT: See below - cplex can generate an error line and then terminate fine, e.g., in CPLEX 12.1.
267                #       To handle these cases, we should be specifying some kind of termination criterion always
268                #       in the course of parsing a log file (we aren't doing so currently - just in some conditions).
269                results.solver.status=SolverStatus.error
270                results.solver.error = " ".join(tokens)
271            elif len(tokens) >= 3 and tokens[0] == "ILOG" and tokens[1] == "CPLEX":
272                cplex_version = tokens[2].rstrip(',')
273            elif len(tokens) >= 3 and tokens[0] == "Variables":
274                if results.problem.number_of_variables is None: # CPLEX 11.2 and subsequent versions have two Variables sections in the log file output.
275                    results.problem.number_of_variables = tokens[2]
276            # In CPLEX 11 (and presumably before), there was only a single line output to
277            # indicate the constriant count, e.g., "Linear constraints : 16 [Less: 7, Greater: 6, Equal: 3]".
278            # In CPLEX 11.2 (or somewhere in between 11 and 11.2 - I haven't bothered to track it down
279            # in that detail), there is another instance of this line prefix in the min/max problem statistics
280            # block - which we don't care about. In this case, the line looks like: "Linear constraints :" and
281            # that's all.
282            elif len(tokens) >= 4 and tokens[0] == "Linear" and tokens[1] == "constraints":
283                results.problem.number_of_constraints = tokens[3]
284            elif len(tokens) >= 3 and tokens[0] == "Nonzeros":
285                if results.problem.number_of_nonzeros is None: # CPLEX 11.2 and subsequent has two Nonzeros sections.               
286                    results.problem.number_of_nonzeros = tokens[2]
287            elif len(tokens) >= 5 and tokens[4] == "MINIMIZE":
288                results.problem.sense = ProblemSense.minimize
289            elif len(tokens) >= 5 and tokens[4] == "MAXIMIZE":
290                results.problem.sense = ProblemSense.maximize
291            elif len(tokens) >= 4 and tokens[0] == "Solution" and tokens[1] == "time" and tokens[2] == "=":
292               # technically, I'm not sure if this is CPLEX user time or user+system - CPLEX doesn't appear
293               # to differentiate, and I'm not sure we can always provide a break-down.
294               results.solver.user_time = eval(tokens[3])
295            elif len(tokens) >= 4 and tokens[0] == "Dual" and tokens[1] == "simplex" and tokens[3] == "Optimal:":
296                results.solver.termination_condition = TerminationCondition.optimal
297                results.solver.termination_message = ' '.join(tokens)
298            elif len(tokens) >= 4 and tokens[0] == "MIP" and tokens[2] == "Integer" and tokens[3] == "infeasible.":
299                # if CPLEX has previously printed an error message, reduce it to a warning -
300                # there is a strong indication it recovered, but we can't be sure.
301                if results.solver.status == SolverStatus.error:
302                   results.solver.status = SolverStatus.warning
303                else:
304                   results.solver.status = SolverStatus.ok
305                results.solver.termination_condition = TerminationCondition.infeasible
306                results.solver.termination_message = ' '.join(tokens)
307            # for the case below, CPLEX sometimes reports "true" optimal (the first case)
308            # and other times within-tolerance optimal (the second case).
309            elif (len(tokens) >= 4 and tokens[0] == "MIP" and tokens[2] == "Integer" and tokens[3] == "optimal") or \
310                 (len(tokens) >= 4 and tokens[0] == "MIP" and tokens[2] == "Integer" and tokens[3] == "optimal,"):
311                # if CPLEX has previously printed an error message, reduce it to a warning -
312                # there is a strong indication it recovered, but we can't be sure.
313                if results.solver.status == SolverStatus.error:
314                   results.solver.status = SolverStatus.warning
315                else:
316                   results.solver.status = SolverStatus.ok               
317                results.solver.termination_condition = TerminationCondition.optimal
318                results.solver.termination_message = ' '.join(tokens)               
319            elif len(tokens) >= 3 and tokens[0] == "Presolve" and tokens[2] == "Infeasible.":
320                # if CPLEX has previously printed an error message, reduce it to a warning -
321                # there is a strong indication it recovered, but we can't be sure.
322                if results.solver.status == SolverStatus.error:
323                   results.solver.status = SolverStatus.warning
324                else:
325                   results.solver.status = SolverStatus.ok               
326                results.solver.termination_condition = TerminationCondition.infeasible
327                results.solver.termination_message = ' '.join(tokens)
328            elif len(tokens) >= 5 and tokens[0] == "Presolve" and tokens[2] == "Unbounded" and tokens[4] == "infeasible.":
329                # if CPLEX has previously printed an error message, reduce it to a warning -
330                # there is a strong indication it recovered, but we can't be sure.
331                if results.solver.status == SolverStatus.error:
332                   results.solver.status = SolverStatus.warning
333                else:
334                   results.solver.status = SolverStatus.ok               
335                # It isn't clear whether we can determine if the problem is unbounded from
336                # CPLEX's output.
337                results.solver.termination_condition = TerminationCondition.unbounded
338                results.solver.termination_message = ' '.join(tokens)
339        return results
340
341    def process_soln_file(self,results):
342        lp_solution=False
343        if not os.path.exists(self.soln_file):
344           return
345
346        soln = Solution()
347        soln.objective['f'].value=None
348        INPUT = open(self.soln_file,"r")
349        results.problem.number_of_objectives=1
350        mip_problem=False
351        for line in INPUT:
352            line = line.strip()
353            line = line.lstrip('<?/')
354            line = line.rstrip('/>?')
355            tokens=line.split(' ')
356
357            if tokens[0] == "variable":
358                variable_name = None
359                variable_value = None
360                variable_reduced_cost = None
361                variable_status = None
362                for i in range(1,len(tokens)):
363                   field_name =  string.strip(tokens[i].split('=')[0])
364                   field_value = (string.strip(tokens[i].split('=')[1])).lstrip("\"").rstrip("\"")
365                   if field_name == "name":
366                      variable_name = field_value
367                   elif field_name == "value":
368                      variable_value = field_value
369                   elif field_name == "reducedCost":
370                      variable_reduced_cost = field_value
371                   elif field_name == "status":
372                      variable_status = field_value
373
374                # skip the "constant-one" variable, used to capture/retain objective offsets in the CPLEX LP format.
375                if variable_name != "ONE_VAR_CONSTANT":
376                   variable = None # cache the solution variable reference, as the getattr is expensive.
377                   try:
378                       variable = soln.variable[variable_name]
379                       variable.value = eval(variable_value)
380                   except:
381                       variable.value = variable_value
382                   if not variable_reduced_cost is None:
383                        try:
384                            variable.rc = eval(variable_reduced_cost)
385                            if variable_status is not None:
386                               if variable_status == "LL":
387                                  variable.lrc = eval(variable_reduced_cost)
388                               else:
389                                  variable.lrc = 0.0
390                               if variable_status == "UL":
391                                  variable.urc = eval(variable_reduced_cost)
392                               else:
393                                  variable.urc = 0.0
394                        except:
395                            raise ValueError, "Unexpected reduced-cost value="+str(variable_reduced_cost)+" encountered for variable="+variable_name
396            elif tokens[0] == "constraint":
397                constraint_name = None
398                constraint_dual = None
399                constaint = None # cache the solution constraint reference, as the getattr is expensive.
400                for i in range(1,len(tokens)):
401                   field_name =  string.strip(tokens[i].split('=')[0])
402                   field_value = (string.strip(tokens[i].split('=')[1])).lstrip("\"").rstrip("\"")
403                   if field_name == "name":
404                      constraint_name = field_value
405                      constraint = soln.constraint[constraint_name]
406                   elif field_name == "dual": # for LPs
407                      # assumes the name field is first.
408                      if eval(field_value) != 0.0:
409                        constraint.dual = eval(field_value)
410                   elif field_name == "slack": # for MIPs
411                      # assumes the name field is first.
412                      if eval(field_value) != 0.0:
413                        constraint.slack = eval(field_value)
414            elif tokens[0].startswith("problemName"):
415                filename = (string.strip(tokens[0].split('=')[1])).lstrip("\"").rstrip("\"")
416                #print "HERE",filename
417                results.problem.name = os.path.basename(filename)
418                if '.' in results.problem.name:
419                    results.problem.name = results.problem.name.split('.')[0]
420                tINPUT=open(filename,"r")
421                for tline in tINPUT:
422                    tline = tline.strip()
423                    if tline == "":
424                        continue
425                    tokens = re.split('[\t ]+',tline)
426                    if tokens[0][0] in ['\\', '*']:
427                        continue
428                    elif tokens[0] == "NAME":
429                        results.problem.name = tokens[1]
430                    else:
431                        sense = tokens[0].lower()
432                        if sense in ['max','maximize']:
433                            results.problem.sense = ProblemSense.maximize
434                        if sense in ['min','minimize']:
435                            results.problem.sense = ProblemSense.minimize
436                    break
437                tINPUT.close()
438               
439            elif tokens[0].startswith("objectiveValue"):
440                objective_value = (string.strip(tokens[0].split('=')[1])).lstrip("\"").rstrip("\"")
441                soln.objective['f'].value = objective_value
442            elif tokens[0].startswith("solutionStatusString"):
443                solution_status = (string.strip(" ".join(tokens).split('=')[1])).lstrip("\"").rstrip("\"")
444                if solution_status in ["optimal", "integer optimal solution", "integer optimal, tolerance"]:
445                    soln.status = SolutionStatus.optimal
446                    soln.gap = 0.0
447                    if results.problem.sense == ProblemSense.minimize:
448                        results.problem.lower_bound = soln.objective['f'].value
449                        if "upper_bound" in dir(results.problem):
450                            del results.problem.upper_bound
451                    else:
452                        results.problem.upper_bound = soln.objective['f'].value
453                        if "lower_bound" in dir(results.problem):
454                            del results.problem.lower_bound
455                    mip_problem=True
456            elif tokens[0].startswith("MIPNodes"):
457                if mip_problem:
458                    n = eval(string.strip(" ".join(tokens).split('=')[1])).lstrip("\"").rstrip("\"")
459                    results.solver.statistics.branch_and_bound.number_of_created_subproblems=n
460                    results.solver.statistics.branch_and_bound.number_of_bounded_subproblems=n
461
462               
463        if not results.solver.status is SolverStatus.error:
464            results.solution.insert(soln)
465        INPUT.close()
466
467    def _postsolve(self):
468
469        # take care of the annoying (and empty) CPLEX temporary files in the current directory.
470        # this approach doesn't seem overly efficient, but python os module functions don't
471        # accept regular expression directly.
472        filename_list = os.listdir(".")
473        for filename in filename_list:
474           # CPLEX temporary files come in two flavors - cplex.log and clone*.log.
475           # the latter is the case for multi-processor environments.
476           # IMPT: trap the possible exception raised by the file not existing.
477           #       this can occur in pyro environments where > 1 workers are
478           #       running CPLEX, and were started from the same directory.
479           #       these logs don't matter anyway (we redirect everything),
480           #       and are largely an annoyance.
481           try:
482              if  re.match('cplex\.log', filename) != None:
483                  os.remove(filename)
484              elif re.match('clone\d+\.log', filename) != None:
485                  os.remove(filename)
486           except OSError:
487              pass
488
489        # let the base class deal with returning results.
490        return ILMLicensedSystemCallSolver._postsolve(self)           
491
492
493class MockCPLEX(CPLEX,mockmip.MockMIP):
494    """A Mock CPLEX solver used for testing
495    """
496
497    def __init__(self, **kwds):
498        try:
499           CPLEX.__init__(self, **kwds)
500        except pyutilib.common.ApplicationError: #pragma:nocover
501           pass                        #pragma:nocover
502        mockmip.MockMIP.__init__(self,"cplex")
503
504    def available(self, exception_flag=True):
505        return CPLEX.available(self,exception_flag)
506
507    def create_command_line(self,executable,problem_files):
508        command = CPLEX.create_command_line(self,executable,problem_files)
509        mockmip.MockMIP.create_command_line(self,executable,problem_files)
510        return command
511
512    def executable(self):
513        return mockmip.MockMIP.executable(self)
514
515    def _execute_command(self,cmd):
516        return mockmip.MockMIP._execute_command(self,cmd)
517
518
519pyutilib.services.register_executable(name="cplex")
520SolverRegistration("cplex", CPLEX)
521SolverRegistration("_mock_cplex", MockCPLEX)
Note: See TracBrowser for help on using the repository browser.