$ontext ==================================================================
LAST MODIFIED: 2008/11/08
FILENAME: timings.gms
PURPOSE: Creates table showing solver resource times for a given model
test set. If model is solved optimally or feasibly, resource
time in seconds is listed. Otherwise, the status is returned.
Also computes shifted geometric means of resource times:
prod(model m, time(i,m) + shift) ** (1/card(m)) - shift
Additionally, model sizes are listed.
Output is in .txt or in .html files. Also lists model
sizes.
Works for all modeltypes.
REQUIRED INPUTS:
--trace1: trace file of solver 1
--trace2: trace file of solver 2
...
--trace8: trace file of solver 8 (min is 1, max is 8 files)
OPTIONAL INPUTS:
--modelfile: modelfile containing models to be considered
in comparison
--outfile: root filename of restimes output file (default
is timings --> timings.txt, timings.htm)
--outdir: output directory (default: no directory)
--reslim limit used for resource value. Unsolved models are
counted with twice the reslim in the calculation
of mean values (default: max resource usage among all data).
--modellib: the model library where the models are taken from.
Valid values are linlib, globallib, minlplib, none.
(default: none)
--colselect: selects a tracefile column to be additionally printed
to resource time (default: ETSolver)
--meanonopt: whether the mean time w.r.t. all models solved to
optimality should be computed (default: 0)
REMARKS: Each tracefile can contain information from only one (1)
solver for a given set of models. For example, tracefiles
containing solver A and solver B will cause error message
upon compilation.
$offtext =================================================================
$if not set colselect $setglobal colselect ETSolver
$setglobal tracecols direction,eqnum,varnum,dvarnum,nz,nlnz,modelstat,solvestat,res,%colselect%
$setglobal alltracecols modelname,modeltype,solvername,nlpdef,mipdef,%tracecols%
*=== Error checks
$if not set trace1 $goto errors_notrace1
$if not exist %trace1% $goto errors_trace1notexist
*== Create master trace file and read in results using readtrace.gms
$echo > %gams.scrdir%tracedata
$if set trace1 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace1% >> %gams.scrdir%tracedata'
$if set trace2 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace2% >> %gams.scrdir%tracedata'
$if set trace3 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace3% >> %gams.scrdir%tracedata'
$if set trace4 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace4% >> %gams.scrdir%tracedata'
$if set trace5 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace5% >> %gams.scrdir%tracedata'
$if set trace6 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace6% >> %gams.scrdir%tracedata'
$if set trace7 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace7% >> %gams.scrdir%tracedata'
$if set trace8 $call 'awk -f readtrace.awk -v gamscolumns="%alltracecols%" %trace8% >> %gams.scrdir%tracedata'
$set tracefile %gams.scrdir%tracedata
$include readtrace.gms
$call 'rm -f %tracefile%'
$ifthen set modelfile
*=== If only the subset of models as listed in "modelfile" is to be analyzed
Set m(*) the subset of models to consider /
$onempty
$include %modelfile%
$offempty
/;
$else
*=== If all models in trace files are to be analyzed
Set m(*);
m(modelname) = yes;
$endif
*=== Set default options
$if not set resmin $set resmin 5e-2
$if not set tsame $set tsame 10
$if not set tfaster $set tfaster 50
$if not set outfile $set outfile timings
$if not set meanonopt $set meanonopt 0
$if not set reslim $set reslim -1
$if not set modellib $set modellib none
$if %modellib% == linlib $set liburl "http://www.gamsworld.org/performance/plib"
$if %modellib% == minlplib $set liburl "http://www.gamsworld.org/minlp/minlplib"
$if %modellib% == globallib $set liburl "http://www.gamsworld.org/global/globallib"
$set outfileroot %outfile%
$if not set outdir $goto nooutdir
$set outfile %outdir%/%outfile%
execute 'mkdir %outdir%'
execute 'cp style.css %outdir%'
$label nooutdir
*=== Possible solver outcomes
Set all_results /opt "optimal",
feas "feasible",
infes "infeasible",
unbnd "unbounded",
fail "fail"
/;
Set result_cont(all_results) "possible results for cont. models"
result_disc(all_results) "possible results for discrete models"
feas_result(all_results) "results where a feasible point was found"
;
result_cont("opt") = yes;
result_cont("feas") = yes;
result_cont("infes") = yes;
result_cont("unbnd") = yes;
result_cont("fail") = yes;
result_disc("opt") = yes;
result_disc("feas") = yes;
result_disc("infes") = yes;
result_disc("unbnd") = yes;
result_disc("fail") = yes;
feas_result("opt") = yes;
feas_result("feas") = yes;
Parameter is_discrete/0/;
Set discrete_mods / MIP, MINLP, MIQCP/;
is_discrete = sum((modeltype,discrete_mods)$sameas(modeltype, discrete_mods), 1);
Set result(*), resultA(*), resultB(*);
if(is_discrete,
result(result_disc) = yes;
resultA(result_disc) = yes;
resultB(result_disc) = yes;
else
result(result_cont) = yes;
resultA(result_cont) = yes;
resultB(result_cont) = yes;
);
Parameter solvernum /0/
nummodels /0/
ratio /0/;
Alias(solvers,solvername)
Parameter timings(*,*,*) GAMS solver results;
timings(modelname,solvername,all_results) = no;
timings(modelname,solvername,"restime") = -1;
timings(modelname,solvername,"%colselect%") = NA;
*=== Determine solvers in tracefiles
Set solverA, solverB;
Parameter i /1/;
loop(solvername,
if(i eq 1,
solverA(solvername) = solvername(solvername);
elseif(i eq 2),
solverB(solvername) = solvername(solvername);
);
i = 2;
);
Parameter modstat /0/;
Parameter solstat /0/;
Parameter have_extracol counts how often we have data in the colselect column /0/;
*=== Compute result totals
loop( (modelname(m),modeltype,solvername,nlpdef,mipdef),
modstat =
tracedata(modelname,modeltype,solvername,nlpdef,mipdef,'modelstat');
solstat =
tracedata(modelname,modeltype,solvername,nlpdef,mipdef,'solvestat');
if (modstat and solstat,
timings(modelname,solvername,"restime")$(modstat and solstat) =
tracedata(modelname,modeltype,solvername,nlpdef,mipdef,'res');
if (mapval(tracedata(modelname,modeltype,solvername,nlpdef,mipdef,'res')),
timings(modelname,solvername,"restime")$(modstat and solstat) = -1);
timings(modelname,solvername,"%colselect%") =
tracedata(modelname,modeltype,solvername,nlpdef,mipdef,'%colselect%');
if (not mapval(timings(modelname,solvername,"%colselect%")), have_extracol = have_extracol + 1);
*=== Assign DISCRETE model timings only if data exists
if(is_discrete,
if( (modstat=1 or modstat=15) and solstat=1,
timings(modelname,solvername,"opt") = yes;
timings(modelname,solvername,"restime") =
tracedata(modelname,modeltype,solvername,nlpdef,mipdef,'res');
elseif( (modstat=2 or modstat=8 or modstat=16) and
(solstat=1 or solstat=2 or solstat=3 or solstat=4 or solstat=5 or solstat=8)),
timings(modelname,solvername,"feas") = yes;
elseif( (modstat=4 or modstat=5 or modstat=10 or modstat=19)and solstat=1),
timings(modelname,solvername,"infes") = yes;
elseif( (modstat=3 or modstat=18) and solstat=1 ),
timings(modelname,solvername,"unbnd") = yes;
else
timings(modelname,solvername,"fail") = yes;
);
*=== Assign CONTINUOUS model square only if data exists
else
if( (modstat=1 or modstat=15) and solstat=1,
timings(modelname,solvername,"opt") = yes;
elseif( (modstat=2 or modstat=7 or modstat=16 or modstat=17) and
(solstat=1 or solstat=2 or solstat=3 or solstat=4 or solstat=5 or solstat=8)),
timings(modelname,solvername,"feas") = yes;
elseif( (modstat=4 or modstat=5 or modstat=19) and solstat=1),
timings(modelname,solvername,"infes") = yes;
elseif( (modstat=3 or modstat=18) and solstat=1),
timings(modelname,solvername,"unbnd") = yes;
else
timings(modelname,solvername,"fail") = yes;
);
);
);
);
*=== Compute solution time mean values
Parameter timeshift / 10 /;
Parameter colselshift / 10 /;
Parameter reslim / %reslim% /;
Parameter colsellim / %reslim% /;
if (reslim=-1,
reslim=smax( (modelname,solvername), timings(modelname,solvername,"restime") );
);
$ifthen not "%colselect%" == "ETSolver"
colselshift = smin( (modelname, solvername)$(not mapval(timings(modelname,solvername,"%colselect%"))),
max(timings(modelname,solvername,"%colselect%"),1e-9) ) / 10;
colsellim = smax( (modelname, solvername)$(not mapval(timings(modelname,solvername,"%colselect%"))),
timings(modelname,solvername,"%colselect%") );
$else
colsellim = reslim;
$endif
Parameter
rescount(*) number of models per solver with resource times data
feascount(*) number of models per solver with feasible points
optcount(*) number of models per solver solved to optimality
cleanrestime(*,*) cleaned value of resource time for each solver-instance pair
cleancolsel(*,*) cleaned value of column-select entry for each solver-instance pair
;
rescount(solvername) = 0;
feascount(solvername) = 0;
optcount(solvername) = 0;
loop((solvername,modelname(m),all_results)$timings(modelname,solvername,all_results),
if (feas_result(all_results),
if (timings(modelname,solvername,"restime") >= 0,
cleanrestime(solvername,modelname) = min(timings(modelname,solvername,"restime"), reslim);
else
cleanrestime(solvername,modelname) = reslim;
);
if (have_extracol,
if (not mapval(timings(modelname,solvername,"%colselect%")),
if (timings(modelname,solvername,"%colselect%") >= 0,
cleancolsel(solvername,modelname) = min(timings(modelname,solvername,"%colselect%"), colsellim);
else
cleancolsel(solvername,modelname) = colsellim );
else
cleancolsel(solvername,modelname) = colsellim );
);
feascount(solvername) = feascount(solvername) + 1;
if (sameas(all_results, "opt"),
optcount(solvername) = optcount(solvername) + 1;
);
else
cleanrestime(solvername,modelname) = reslim;
if (have_extracol, cleancolsel(solvername,modelname) = colsellim );
);
rescount(solvername) = rescount(solvername) + 1;
);
Parameter
resavg(*) average of resource times over all models with feasible points (as shifted geom. mean)
wallavg(*) average of colselect times over all models with feasible points (as shifted geom. mean)
$ifthen not %meanonopt% == 0
optresavg(*) average of resource times over all models solved to optimality (as shifted geom. mean)
optwallavg(*) average of colselect times over all models solved to optimality (as shifted geom. mean)
$endif
;
resavg(solvername) =
prod(modelname(m), (cleanrestime(solvername,modelname) + timeshift)**(1/rescount(solvername))) - timeshift;
if (have_extracol,
wallavg(solvername) =
prod(modelname(m), (cleancolsel(solvername,modelname) + colselshift)**(1/rescount(solvername))) - colselshift;
);
$ifthen not %meanonopt% == 0
*-- set cleanrestime and cleancolsel to column limit if not solved to optimality, then use same formula as above
loop ((solvername,modelname(m),feas_result)$timings(modelname,solvername,feas_result),
if (not sameas(feas_result, "opt"),
cleanrestime(solvername, modelname) = reslim;
if (have_extracol, cleancolsel(solvername, modelname) = colsellim );
)
);
optresavg(solvername) =
prod(modelname(m), (cleanrestime(solvername,modelname) + timeshift)**(1/rescount(solvername))) - timeshift;
if (have_extracol,
optwallavg(solvername) =
prod(modelname(m), (cleancolsel(solvername,modelname) + colselshift)**(1/rescount(solvername))) - colselshift;
);
$endif
*=========================================================================
*==== Create text file with timings
*=========================================================================
file ftxt /"%outfile%.txt"/;
put ftxt;
put 'DATE/TIME: %system.date% %system.time%' //;
put "BENCHMARKS FOR THE FOLLOWING SOLVERS:"//;
loop(solvername,
put '- ' solvername.tl:0 /;
);
put / 'Solver resource times are given in seconds.'
/ 'If no optimal or feasible solution is found,'
/ 'the status is listed instead of the resource time.'/;
put / '* Status details:'//;
loop(all_results,
put all_results.tl:10 ' - ' all_results.te(all_results):15 /;
);
put '(no data)':10 ' - ' 'no data available in trace file' /;
put / "====================";
loop(solvername,
put "====================";
);
put ftxt /'MODELNAME ':20;
loop(solvername,
put solvername.tl:>20
);
put /;
put "--------------------";
loop(solvername,
put "--------------------";
);
put /;
loop(modelname(m),
put modelname.tl:20
loop(solvername,
loop(all_results$timings(modelname,solvername,all_results),
if (timings(modelname,solvername,"restime") >= 0,
put timings(modelname,solvername,"restime"):20;
else
put all_results.tl:>18 ' *';
)
);
if( sum(all_results$timings(modelname,solvername,all_results),1)=0,
put '(no data)':>20;
);
);
put /;
);
put "====================";
loop(solvername,
put "====================";
);
scalar first /1/;
put / "Shifted Geom. Mean (feasible):";
loop(solvername,
if(first,
put resavg(solvername):10;
first = 0;
else
put resavg(solvername):20;
);
);
put / "% no feasible point:";
loop(solvername,
put (100-100*feascount(solvername)/rescount(solvername)):20
);
$ifthen not %meanonopt% == 0
first=1;
put / "Shifted Geom. Mean (optimal):";
loop(solvername,
if(first,
put optresavg(solvername):11;
first=0;
else
put optresavg(solvername):20;
);
);
put / "% not optimal: ";
loop(solvername,
put (100-100*optcount(solvername)/rescount(solvername)):20
);
$endif
* print model statistics only if trace1 set
$ifthen set trace1
put // "PROBLEM SIZES:"/;
put / "============================================================="
"===================================";
put / "PROBLEM":20 "EQUATIONS":>15 "TOTAL VAR.":>15 "DISCR. VAR":>15
"TOTAL NZ":>15 "NONLINEAR NZ":>15;
put / "============================================================="
"==================================="/;
loop((modelname(m),modeltype,solverA,nlpdef,mipdef)
$tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"eqnum"),
put modelname.tl:20;
put tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"eqnum"):15:0;
put tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"varnum"):15:0;
if(tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"dvarnum"),
put tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"dvarnum"):15:0;
else
put " ":15;
)
put tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"nz"):15:0;
if(tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"nlnz"),
put tracedata(modelname,modeltype,solverA,nlpdef,mipdef,"nlnz"):15:0;
else
put " ":15;
);
put /;
);
putclose ftxt;
$endif
*=========================================================================
*=== Create HTML file
*=========================================================================
file fres_htm /"%outfile%.htm"/;
put fres_htm;
put / ''
/ ''
/ '
'
put / 'Instances solved to optimality are highlighted with a '
/ 'green background.';
$ifthen "%colselect%" == "ETSolver"
if (have_extracol,
put / ' ETSolver times below resource time or above 11 seconds and exceeding the resource time by at least 10% are highlighted with red background.';
);
$endif
put / '
The average time is calculated as a shifted geometric mean. '
/ '
The limit in the solver time columns is ' reslim:0 ' seconds and '
/ 'the shift is ' timeshift:0 ' seconds.';
if (have_extracol,
put / ' The limit in the %colselect% columns is ' colsellim:0 ' and '
/ 'the shift is ' colselshift:0 '.';
);
put / '
In the "Shifted Geometric Mean (feasible)" row, instances without feasible solution and instances above '
/ 'the limit are counted with the limit value (' reslim:0;
if (have_extracol,
put ' and ' colsellim:0 ', resp.';
);
put '). ';
$ifthen not %meanonopt% == 0
put / 'In the "Shifted Geometric Mean (optimal)" row, instances not solved to optimality and instances above the limit are counted with the limit value (' reslim:0;
if (have_extracol,
put / ' and ' colsellim:0 ', resp.';
);
put '). ';
$endif
put / '
';
loop(solvername,
loop(all_results$timings(modelname,solvername,all_results),
put / '
';
if (timings(modelname,solvername,"restime") >= 0,
put timings(modelname,solvername,"restime"):10;
else
put "NA";
);
if (not feas_result(all_results),
put ' (' all_results.te(all_results):0 ') ';
);
put '
';
if (have_extracol,
put / '
=0 and
(timings(modelname,solvername,"%colselect%")>=1.1*max(timings(modelname,solvername,"restime"),10) or
timings(modelname,solvername,"%colselect%")<0.99*timings(modelname,solvername,"restime")),
put ' class="redproblem"' ))
$endif
put '>' timings(modelname,solvername,"%colselect%") '
';
)
);
if( sum(all_results$timings(modelname,solvername,all_results),1)=0,
put / '
';
loop(modelname(m),
* make sure that every model is printed at most once
i=0;
loop((modeltype,solvername,nlpdef,mipdef)$
(tracedata(modelname,modeltype,solvername,nlpdef,mipdef,"eqnum") and i=0),
put / '