001package org.cpsolver.studentsct;
002
003import java.io.BufferedReader;
004import java.io.File;
005import java.io.FileInputStream;
006import java.io.FileOutputStream;
007import java.io.FileReader;
008import java.io.FileWriter;
009import java.io.IOException;
010import java.io.PrintWriter;
011import java.text.DecimalFormat;
012import java.util.ArrayList;
013import java.util.Collection;
014import java.util.Collections;
015import java.util.Comparator;
016import java.util.Date;
017import java.util.HashSet;
018import java.util.HashMap;
019import java.util.Iterator;
020import java.util.List;
021import java.util.Map;
022import java.util.Set;
023import java.util.StringTokenizer;
024import java.util.TreeSet;
025
026import org.apache.logging.log4j.Level;
027import org.apache.logging.log4j.core.config.Configurator;
028import org.cpsolver.ifs.assignment.Assignment;
029import org.cpsolver.ifs.assignment.DefaultSingleAssignment;
030import org.cpsolver.ifs.assignment.EmptyAssignment;
031import org.cpsolver.ifs.heuristics.BacktrackNeighbourSelection;
032import org.cpsolver.ifs.model.Neighbour;
033import org.cpsolver.ifs.solution.Solution;
034import org.cpsolver.ifs.solution.SolutionListener;
035import org.cpsolver.ifs.solver.ParallelSolver;
036import org.cpsolver.ifs.solver.Solver;
037import org.cpsolver.ifs.solver.SolverListener;
038import org.cpsolver.ifs.util.DataProperties;
039import org.cpsolver.ifs.util.JProf;
040import org.cpsolver.ifs.util.Progress;
041import org.cpsolver.ifs.util.ProgressWriter;
042import org.cpsolver.ifs.util.ToolBox;
043import org.cpsolver.studentsct.check.CourseLimitCheck;
044import org.cpsolver.studentsct.check.InevitableStudentConflicts;
045import org.cpsolver.studentsct.check.OverlapCheck;
046import org.cpsolver.studentsct.check.SectionLimitCheck;
047import org.cpsolver.studentsct.extension.DistanceConflict;
048import org.cpsolver.studentsct.extension.TimeOverlapsCounter;
049import org.cpsolver.studentsct.filter.CombinedStudentFilter;
050import org.cpsolver.studentsct.filter.FreshmanStudentFilter;
051import org.cpsolver.studentsct.filter.RandomStudentFilter;
052import org.cpsolver.studentsct.filter.ReverseStudentFilter;
053import org.cpsolver.studentsct.filter.StudentFilter;
054import org.cpsolver.studentsct.heuristics.StudentSctNeighbourSelection;
055import org.cpsolver.studentsct.heuristics.selection.BranchBoundSelection;
056import org.cpsolver.studentsct.heuristics.selection.OnlineSelection;
057import org.cpsolver.studentsct.heuristics.selection.SwapStudentSelection;
058import org.cpsolver.studentsct.heuristics.selection.BranchBoundSelection.BranchBoundNeighbour;
059import org.cpsolver.studentsct.heuristics.studentord.StudentOrder;
060import org.cpsolver.studentsct.heuristics.studentord.StudentRandomOrder;
061import org.cpsolver.studentsct.model.AreaClassificationMajor;
062import org.cpsolver.studentsct.model.Course;
063import org.cpsolver.studentsct.model.CourseRequest;
064import org.cpsolver.studentsct.model.Enrollment;
065import org.cpsolver.studentsct.model.Offering;
066import org.cpsolver.studentsct.model.Request;
067import org.cpsolver.studentsct.model.Student;
068import org.cpsolver.studentsct.report.CourseConflictTable;
069import org.cpsolver.studentsct.report.DistanceConflictTable;
070import org.cpsolver.studentsct.report.RequestGroupTable;
071import org.cpsolver.studentsct.report.RequestPriorityTable;
072import org.cpsolver.studentsct.report.SectionConflictTable;
073import org.cpsolver.studentsct.report.TableauReport;
074import org.cpsolver.studentsct.report.TimeOverlapConflictTable;
075import org.cpsolver.studentsct.report.UnbalancedSectionsTable;
076import org.dom4j.Document;
077import org.dom4j.DocumentHelper;
078import org.dom4j.Element;
079import org.dom4j.io.OutputFormat;
080import org.dom4j.io.SAXReader;
081import org.dom4j.io.XMLWriter;
082
083/**
084 * A main class for running of the student sectioning solver from command line. <br>
085 * <br>
086 * Usage:<br>
087 * java -Xmx1024m -jar studentsct-1.1.jar config.properties [input_file]
088 * [output_folder] [batch|online|simple]<br>
089 * <br>
090 * Modes:<br>
091 * &nbsp;&nbsp;batch ... batch sectioning mode (default mode -- IFS solver with
092 * {@link StudentSctNeighbourSelection} is used)<br>
093 * &nbsp;&nbsp;online ... online sectioning mode (students are sectioned one by
094 * one, sectioning info (expected/held space) is used)<br>
095 * &nbsp;&nbsp;simple ... simple sectioning mode (students are sectioned one by
096 * one, sectioning info is not used)<br>
097 * See http://www.unitime.org for example configuration files and benchmark data
098 * sets.<br>
099 * <br>
100 * 
101 * The test does the following steps:
102 * <ul>
103 * <li>Provided property file is loaded (see {@link DataProperties}).
104 * <li>Output folder is created (General.Output property) and logging is setup
105 * (using log4j).
106 * <li>Input data are loaded from the given XML file (calling
107 * {@link StudentSectioningXMLLoader#load()}).
108 * <li>Solver is executed (see {@link Solver}).
109 * <li>Resultant solution is saved to an XML file (calling
110 * {@link StudentSectioningXMLSaver#save()}.
111 * </ul>
112 * Also, a log and some reports (e.g., {@link CourseConflictTable} and
113 * {@link DistanceConflictTable}) are created in the output folder.
114 * 
115 * <br>
116 * <br>
117 * Parameters:
118 * <table border='1'><caption>Related Solver Parameters</caption>
119 * <tr>
120 * <th>Parameter</th>
121 * <th>Type</th>
122 * <th>Comment</th>
123 * </tr>
124 * <tr>
125 * <td>Test.LastLikeCourseDemands</td>
126 * <td>{@link String}</td>
127 * <td>Load last-like course demands from the given XML file (in the format that
128 * is being used for last like course demand table in the timetabling
129 * application)</td>
130 * </tr>
131 * <tr>
132 * <td>Test.StudentInfos</td>
133 * <td>{@link String}</td>
134 * <td>Load last-like course demands from the given XML file (in the format that
135 * is being used for last like course demand table in the timetabling
136 * application)</td>
137 * </tr>
138 * <tr>
139 * <td>Test.CrsReq</td>
140 * <td>{@link String}</td>
141 * <td>Load student requests from the given semi-colon separated list files (in
142 * the format that is being used by the old MSF system)</td>
143 * </tr>
144 * <tr>
145 * <td>Test.EtrChk</td>
146 * <td>{@link String}</td>
147 * <td>Load student information (academic area, classification, major, minor)
148 * from the given semi-colon separated list files (in the format that is being
149 * used by the old MSF system)</td>
150 * </tr>
151 * <tr>
152 * <td>Sectioning.UseStudentPreferencePenalties</td>
153 * <td>{@link Boolean}</td>
154 * <td>If true, {@link StudentPreferencePenalties} are used (applicable only for
155 * online sectioning)</td>
156 * </tr>
157 * <tr>
158 * <td>Test.StudentOrder</td>
159 * <td>{@link String}</td>
160 * <td>A class that is used for ordering of students (must be an interface of
161 * {@link StudentOrder}, default is {@link StudentRandomOrder}, not applicable
162 * only for batch sectioning)</td>
163 * </tr>
164 * <tr>
165 * <td>Test.CombineStudents</td>
166 * <td>{@link File}</td>
167 * <td>If provided, students are combined from the input file (last-like
168 * students) and the provided file (real students). Real non-freshmen students
169 * are taken from real data, last-like data are loaded on top of the real data
170 * (all students, but weighted to occupy only the remaining space).</td>
171 * </tr>
172 * <tr>
173 * <td>Test.CombineStudentsLastLike</td>
174 * <td>{@link File}</td>
175 * <td>If provided (together with Test.CombineStudents), students are combined
176 * from the this file (last-like students) and Test.CombineStudents file (real
177 * students). Real non-freshmen students are taken from real data, last-like
178 * data are loaded on top of the real data (all students, but weighted to occupy
179 * only the remaining space).</td>
180 * </tr>
181 * <tr>
182 * <td>Test.CombineAcceptProb</td>
183 * <td>{@link Double}</td>
184 * <td>Used in combining students, probability of a non-freshmen real student to
185 * be taken into the combined file (default is 1.0 -- all real non-freshmen
186 * students are taken).</td>
187 * </tr>
188 * <tr>
189 * <td>Test.FixPriorities</td>
190 * <td>{@link Boolean}</td>
191 * <td>If true, course/free time request priorities are corrected (to go from
192 * zero, without holes or duplicates).</td>
193 * </tr>
194 * <tr>
195 * <td>Test.ExtraStudents</td>
196 * <td>{@link File}</td>
197 * <td>If provided, students are loaded from the given file on top of the
198 * students loaded from the ordinary input file (students with the same id are
199 * skipped).</td>
200 * </tr>
201 * </table>
202 * <br>
203 * <br>
204 * 
205 * @author  Tomáš Müller
206 * @version StudentSct 1.3 (Student Sectioning)<br>
207 *          Copyright (C) 2007 - 2014 Tomáš Müller<br>
208 *          <a href="mailto:muller@unitime.org">muller@unitime.org</a><br>
209 *          <a href="http://muller.unitime.org">http://muller.unitime.org</a><br>
210 * <br>
211 *          This library is free software; you can redistribute it and/or modify
212 *          it under the terms of the GNU Lesser General Public License as
213 *          published by the Free Software Foundation; either version 3 of the
214 *          License, or (at your option) any later version. <br>
215 * <br>
216 *          This library is distributed in the hope that it will be useful, but
217 *          WITHOUT ANY WARRANTY; without even the implied warranty of
218 *          MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
219 *          Lesser General Public License for more details. <br>
220 * <br>
221 *          You should have received a copy of the GNU Lesser General Public
222 *          License along with this library; if not see
223 *          <a href='http://www.gnu.org/licenses/'>http://www.gnu.org/licenses/</a>.
224 */
225
226public class Test {
227    private static org.apache.logging.log4j.Logger sLog = org.apache.logging.log4j.LogManager.getLogger(Test.class);
228    private static java.text.SimpleDateFormat sDateFormat = new java.text.SimpleDateFormat("yyMMdd_HHmmss",
229            java.util.Locale.US);
230    private static DecimalFormat sDF = new DecimalFormat("0.000");
231
232    /** Load student sectioning model 
233     * @param cfg solver configuration
234     * @return loaded solution
235     **/
236    public static Solution<Request, Enrollment> load(DataProperties cfg) {
237        StudentSectioningModel model = null;
238        Assignment<Request, Enrollment> assignment = null;
239        try {
240            if (cfg.getProperty("Test.CombineStudents") == null) {
241                model = new StudentSectioningModel(cfg);
242                assignment = new DefaultSingleAssignment<Request, Enrollment>();
243                new StudentSectioningXMLLoader(model, assignment).load();
244            } else {
245                Solution<Request, Enrollment> solution = combineStudents(cfg,
246                        new File(cfg.getProperty("Test.CombineStudentsLastLike", cfg.getProperty("General.Input", "." + File.separator + "solution.xml"))),
247                        new File(cfg.getProperty("Test.CombineStudents")));
248                model = (StudentSectioningModel)solution.getModel();
249                assignment = solution.getAssignment();
250            }
251            if (cfg.getProperty("Test.ExtraStudents") != null) {
252                StudentSectioningXMLLoader extra = new StudentSectioningXMLLoader(model, assignment);
253                extra.setInputFile(new File(cfg.getProperty("Test.ExtraStudents")));
254                extra.setLoadOfferings(false);
255                extra.setLoadStudents(true);
256                extra.setStudentFilter(new ExtraStudentFilter(model));
257                extra.load();
258            }
259            if (cfg.getProperty("Test.LastLikeCourseDemands") != null)
260                loadLastLikeCourseDemandsXml(model, new File(cfg.getProperty("Test.LastLikeCourseDemands")));
261            if (cfg.getProperty("Test.CrsReq") != null)
262                loadCrsReqFiles(model, cfg.getProperty("Test.CrsReq"));
263        } catch (Exception e) {
264            sLog.error("Unable to load model, reason: " + e.getMessage(), e);
265            return null;
266        }
267        if (cfg.getPropertyBoolean("Debug.DistanceConflict", false))
268            DistanceConflict.sDebug = true;
269        if (cfg.getPropertyBoolean("Debug.BranchBoundSelection", false))
270            BranchBoundSelection.sDebug = true;
271        if (cfg.getPropertyBoolean("Debug.SwapStudentsSelection", false))
272            SwapStudentSelection.sDebug = true;
273        if (cfg.getPropertyBoolean("Debug.TimeOverlaps", false))
274            TimeOverlapsCounter.sDebug = true;
275        if (cfg.getProperty("CourseRequest.SameTimePrecise") != null)
276            CourseRequest.sSameTimePrecise = cfg.getPropertyBoolean("CourseRequest.SameTimePrecise", false);
277        Configurator.setLevel(BacktrackNeighbourSelection.class.getName(),
278                cfg.getPropertyBoolean("Debug.BacktrackNeighbourSelection", false) ? Level.DEBUG : Level.INFO);
279        if (cfg.getPropertyBoolean("Test.FixPriorities", false))
280            fixPriorities(model);
281        return new Solution<Request, Enrollment>(model, assignment);
282    }
283
284    /** Batch sectioning test 
285     * @param cfg solver configuration
286     * @return resultant solution
287     **/
288    public static Solution<Request, Enrollment> batchSectioning(DataProperties cfg) {
289        Solution<Request, Enrollment> solution = load(cfg);
290        if (solution == null)
291            return null;
292        StudentSectioningModel model = (StudentSectioningModel)solution.getModel();
293
294        if (cfg.getPropertyBoolean("Test.ComputeSectioningInfo", true))
295            model.clearOnlineSectioningInfos();
296        
297        Progress.getInstance(model).addProgressListener(new ProgressWriter(System.out));
298
299        solve(solution, cfg);
300
301        return solution;
302    }
303
304    /** Online sectioning test 
305     * @param cfg solver configuration
306     * @return resultant solution
307     * @throws Exception thrown when the sectioning fails
308     **/
309    public static Solution<Request, Enrollment> onlineSectioning(DataProperties cfg) throws Exception {
310        Solution<Request, Enrollment> solution = load(cfg);
311        if (solution == null)
312            return null;
313        StudentSectioningModel model = (StudentSectioningModel)solution.getModel();
314        Assignment<Request, Enrollment> assignment = solution.getAssignment();
315
316        solution.addSolutionListener(new TestSolutionListener());
317        double startTime = JProf.currentTimeSec();
318
319        Solver<Request, Enrollment> solver = new Solver<Request, Enrollment>(cfg);
320        solver.setInitalSolution(solution);
321        solver.initSolver();
322
323        OnlineSelection onlineSelection = new OnlineSelection(cfg);
324        onlineSelection.init(solver);
325
326        double totalPenalty = 0, minPenalty = 0, maxPenalty = 0;
327        double minAvEnrlPenalty = 0, maxAvEnrlPenalty = 0;
328        double totalPrefPenalty = 0, minPrefPenalty = 0, maxPrefPenalty = 0;
329        double minAvEnrlPrefPenalty = 0, maxAvEnrlPrefPenalty = 0;
330        int nrChoices = 0, nrEnrollments = 0, nrCourseRequests = 0;
331        int chChoices = 0, chCourseRequests = 0, chStudents = 0;
332
333        int choiceLimit = model.getProperties().getPropertyInt("Test.ChoicesLimit", -1);
334
335        File outDir = new File(model.getProperties().getProperty("General.Output", "."));
336        outDir.mkdirs();
337        PrintWriter pw = new PrintWriter(new FileWriter(new File(outDir, "choices.csv")));
338
339        List<Student> students = model.getStudents();
340        try {
341            @SuppressWarnings("rawtypes")
342            Class studentOrdClass = Class.forName(model.getProperties().getProperty("Test.StudentOrder", StudentRandomOrder.class.getName()));
343            @SuppressWarnings("unchecked")
344            StudentOrder studentOrd = (StudentOrder) studentOrdClass.getConstructor(new Class[] { DataProperties.class }).newInstance(new Object[] { model.getProperties() });
345            students = studentOrd.order(model.getStudents());
346        } catch (Exception e) {
347            sLog.error("Unable to reorder students, reason: " + e.getMessage(), e);
348        }
349        
350        ShutdownHook hook = new ShutdownHook(solver);
351        Runtime.getRuntime().addShutdownHook(hook);
352
353        for (Student student : students) {
354            if (student.nrAssignedRequests(assignment) > 0)
355                continue; // skip students with assigned courses (i.e., students
356                          // already assigned by a batch sectioning process)
357            sLog.info("Sectioning student: " + student);
358
359            BranchBoundSelection.Selection selection = onlineSelection.getSelection(assignment, student);
360            BranchBoundNeighbour neighbour = selection.select();
361            if (neighbour != null) {
362                StudentPreferencePenalties penalties = null;
363                if (selection instanceof OnlineSelection.EpsilonSelection) {
364                    OnlineSelection.EpsilonSelection epsSelection = (OnlineSelection.EpsilonSelection) selection;
365                    penalties = epsSelection.getPenalties();
366                    for (int i = 0; i < neighbour.getAssignment().length; i++) {
367                        Request r = student.getRequests().get(i);
368                        if (r instanceof CourseRequest) {
369                            nrCourseRequests++;
370                            chCourseRequests++;
371                            int chChoicesThisRq = 0;
372                            CourseRequest request = (CourseRequest) r;
373                            for (Enrollment x : request.getAvaiableEnrollments(assignment)) {
374                                nrEnrollments++;
375                                if (epsSelection.isAllowed(i, x)) {
376                                    nrChoices++;
377                                    if (choiceLimit <= 0 || chChoicesThisRq < choiceLimit) {
378                                        chChoices++;
379                                        chChoicesThisRq++;
380                                    }
381                                }
382                            }
383                        }
384                    }
385                    chStudents++;
386                    if (chStudents == 100) {
387                        pw.println(sDF.format(((double) chChoices) / chCourseRequests));
388                        pw.flush();
389                        chStudents = 0;
390                        chChoices = 0;
391                        chCourseRequests = 0;
392                    }
393                }
394                for (int i = 0; i < neighbour.getAssignment().length; i++) {
395                    if (neighbour.getAssignment()[i] == null)
396                        continue;
397                    Enrollment enrollment = neighbour.getAssignment()[i];
398                    if (enrollment.getRequest() instanceof CourseRequest) {
399                        CourseRequest request = (CourseRequest) enrollment.getRequest();
400                        double[] avEnrlMinMax = getMinMaxAvailableEnrollmentPenalty(assignment, request);
401                        minAvEnrlPenalty += avEnrlMinMax[0];
402                        maxAvEnrlPenalty += avEnrlMinMax[1];
403                        totalPenalty += enrollment.getPenalty();
404                        minPenalty += request.getMinPenalty();
405                        maxPenalty += request.getMaxPenalty();
406                        if (penalties != null) {
407                            double[] avEnrlPrefMinMax = penalties.getMinMaxAvailableEnrollmentPenalty(assignment, enrollment.getRequest());
408                            minAvEnrlPrefPenalty += avEnrlPrefMinMax[0];
409                            maxAvEnrlPrefPenalty += avEnrlPrefMinMax[1];
410                            totalPrefPenalty += penalties.getPenalty(enrollment);
411                            minPrefPenalty += penalties.getMinPenalty(enrollment.getRequest());
412                            maxPrefPenalty += penalties.getMaxPenalty(enrollment.getRequest());
413                        }
414                    }
415                }
416                neighbour.assign(assignment, solution.getIteration());
417                sLog.info("Student " + student + " enrolls into " + neighbour);
418                onlineSelection.updateSpace(assignment, student);
419            } else {
420                sLog.warn("No solution found.");
421            }
422            solution.update(JProf.currentTimeSec() - startTime);
423            solution.saveBest();
424        }
425
426        if (chCourseRequests > 0)
427            pw.println(sDF.format(((double) chChoices) / chCourseRequests));
428
429        pw.flush();
430        pw.close();
431        
432        HashMap<String, String> extra = new HashMap<String, String>();
433        sLog.info("Overall penalty is " + getPerc(totalPenalty, minPenalty, maxPenalty) + "% ("
434                + sDF.format(totalPenalty) + "/" + sDF.format(minPenalty) + ".." + sDF.format(maxPenalty) + ")");
435        extra.put("Overall penalty", getPerc(totalPenalty, minPenalty, maxPenalty) + "% (" + sDF.format(totalPenalty)
436                + "/" + sDF.format(minPenalty) + ".." + sDF.format(maxPenalty) + ")");
437        extra.put("Overall available enrollment penalty", getPerc(totalPenalty, minAvEnrlPenalty, maxAvEnrlPenalty)
438                + "% (" + sDF.format(totalPenalty) + "/" + sDF.format(minAvEnrlPenalty) + ".." + sDF.format(maxAvEnrlPenalty) + ")");
439        if (onlineSelection.isUseStudentPrefPenalties()) {
440            sLog.info("Overall preference penalty is " + getPerc(totalPrefPenalty, minPrefPenalty, maxPrefPenalty)
441                    + "% (" + sDF.format(totalPrefPenalty) + "/" + sDF.format(minPrefPenalty) + ".." + sDF.format(maxPrefPenalty) + ")");
442            extra.put("Overall preference penalty", getPerc(totalPrefPenalty, minPrefPenalty, maxPrefPenalty) + "% ("
443                    + sDF.format(totalPrefPenalty) + "/" + sDF.format(minPrefPenalty) + ".." + sDF.format(maxPrefPenalty) + ")");
444            extra.put("Overall preference available enrollment penalty", getPerc(totalPrefPenalty,
445                    minAvEnrlPrefPenalty, maxAvEnrlPrefPenalty)
446                    + "% (" + sDF.format(totalPrefPenalty) + "/" + sDF.format(minAvEnrlPrefPenalty) + ".." + sDF.format(maxAvEnrlPrefPenalty) + ")");
447            extra.put("Average number of choices", sDF.format(((double) nrChoices) / nrCourseRequests) + " ("
448                    + nrChoices + "/" + nrCourseRequests + ")");
449            extra.put("Average number of enrollments", sDF.format(((double) nrEnrollments) / nrCourseRequests) + " ("
450                    + nrEnrollments + "/" + nrCourseRequests + ")");
451        }
452        hook.setExtra(extra);
453
454        return solution;
455    }
456
457    /**
458     * Minimum and maximum enrollment penalty, i.e.,
459     * {@link Enrollment#getPenalty()} of all enrollments
460     * @param request a course request
461     * @return minimum and maximum of the enrollment penalty
462     */
463    public static double[] getMinMaxEnrollmentPenalty(CourseRequest request) {
464        List<Enrollment> enrollments = request.values(new EmptyAssignment<Request, Enrollment>());
465        if (enrollments.isEmpty())
466            return new double[] { 0, 0 };
467        double min = Double.MAX_VALUE, max = Double.MIN_VALUE;
468        for (Enrollment enrollment : enrollments) {
469            double penalty = enrollment.getPenalty();
470            min = Math.min(min, penalty);
471            max = Math.max(max, penalty);
472        }
473        return new double[] { min, max };
474    }
475
476    /**
477     * Minimum and maximum available enrollment penalty, i.e.,
478     * {@link Enrollment#getPenalty()} of all available enrollments
479     * @param assignment current assignment
480     * @param request a course request
481     * @return minimum and maximum of the available enrollment penalty
482     */
483    public static double[] getMinMaxAvailableEnrollmentPenalty(Assignment<Request, Enrollment> assignment, CourseRequest request) {
484        List<Enrollment> enrollments = request.getAvaiableEnrollments(assignment);
485        if (enrollments.isEmpty())
486            return new double[] { 0, 0 };
487        double min = Double.MAX_VALUE, max = Double.MIN_VALUE;
488        for (Enrollment enrollment : enrollments) {
489            double penalty = enrollment.getPenalty();
490            min = Math.min(min, penalty);
491            max = Math.max(max, penalty);
492        }
493        return new double[] { min, max };
494    }
495
496    /**
497     * Compute percentage
498     * 
499     * @param value
500     *            current value
501     * @param min
502     *            minimal bound
503     * @param max
504     *            maximal bound
505     * @return (value-min)/(max-min)
506     */
507    public static String getPerc(double value, double min, double max) {
508        if (max == min)
509            return sDF.format(100.0);
510        return sDF.format(100.0 - 100.0 * (value - min) / (max - min));
511    }
512
513    /**
514     * Print some information about the solution
515     * 
516     * @param solution
517     *            given solution
518     * @param computeTables
519     *            true, if reports {@link CourseConflictTable} and
520     *            {@link DistanceConflictTable} are to be computed as well
521     * @param computeSectInfos
522     *            true, if online sectioning infou is to be computed as well
523     *            (see
524     *            {@link StudentSectioningModel#computeOnlineSectioningInfos(Assignment)})
525     * @param runChecks
526     *            true, if checks {@link OverlapCheck} and
527     *            {@link SectionLimitCheck} are to be performed as well
528     */
529    public static void printInfo(Solution<Request, Enrollment> solution, boolean computeTables, boolean computeSectInfos, boolean runChecks) {
530        StudentSectioningModel model = (StudentSectioningModel) solution.getModel();
531
532        if (computeTables) {
533            if (solution.getModel().assignedVariables(solution.getAssignment()).size() > 0) {
534                try {
535                    DataProperties lastlike = new DataProperties();
536                    lastlike.setProperty("lastlike", "true");
537                    lastlike.setProperty("real", "false");
538                    lastlike.setProperty("useAmPm", "true");
539                    DataProperties real = new DataProperties();
540                    real.setProperty("lastlike", "false");
541                    real.setProperty("real", "true");
542                    real.setProperty("useAmPm", "true");
543                    
544                    File outDir = new File(model.getProperties().getProperty("General.Output", "."));
545                    outDir.mkdirs();
546                    CourseConflictTable cct = new CourseConflictTable((StudentSectioningModel) solution.getModel());
547                    cct.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "conflicts-lastlike.csv"));
548                    cct.createTable(solution.getAssignment(), real).save(new File(outDir, "conflicts-real.csv"));
549
550                    DistanceConflictTable dct = new DistanceConflictTable((StudentSectioningModel) solution.getModel());
551                    dct.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "distances-lastlike.csv"));
552                    dct.createTable(solution.getAssignment(), real).save(new File(outDir, "distances-real.csv"));
553                    
554                    SectionConflictTable sct = new SectionConflictTable((StudentSectioningModel) solution.getModel(), SectionConflictTable.Type.OVERLAPS);
555                    sct.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "time-conflicts-lastlike.csv"));
556                    sct.createTable(solution.getAssignment(), real).save(new File(outDir, "time-conflicts-real.csv"));
557                    
558                    SectionConflictTable ust = new SectionConflictTable((StudentSectioningModel) solution.getModel(), SectionConflictTable.Type.UNAVAILABILITIES);
559                    ust.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "availability-conflicts-lastlike.csv"));
560                    ust.createTable(solution.getAssignment(), real).save(new File(outDir, "availability-conflicts-real.csv"));
561                    
562                    SectionConflictTable ct = new SectionConflictTable((StudentSectioningModel) solution.getModel(), SectionConflictTable.Type.OVERLAPS_AND_UNAVAILABILITIES);
563                    ct.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "section-conflicts-lastlike.csv"));
564                    ct.createTable(solution.getAssignment(), real).save(new File(outDir, "section-conflicts-real.csv"));
565                    
566                    UnbalancedSectionsTable ubt = new UnbalancedSectionsTable((StudentSectioningModel) solution.getModel());
567                    ubt.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "unbalanced-lastlike.csv"));
568                    ubt.createTable(solution.getAssignment(), real).save(new File(outDir, "unbalanced-real.csv"));
569                    
570                    TimeOverlapConflictTable toc = new TimeOverlapConflictTable((StudentSectioningModel) solution.getModel());
571                    toc.createTable(solution.getAssignment(), lastlike).save(new File(outDir, "time-overlaps-lastlike.csv"));
572                    toc.createTable(solution.getAssignment(), real).save(new File(outDir, "time-overlaps-real.csv"));
573                    
574                    RequestGroupTable rqt = new RequestGroupTable((StudentSectioningModel) solution.getModel());
575                    rqt.create(solution.getAssignment(), model.getProperties()).save(new File(outDir, "request-groups.csv"));
576                    
577                    RequestPriorityTable rpt = new RequestPriorityTable((StudentSectioningModel) solution.getModel());
578                    rpt.create(solution.getAssignment(), model.getProperties()).save(new File(outDir, "request-priorities.csv"));
579                    
580                    TableauReport tr = new TableauReport((StudentSectioningModel) solution.getModel());
581                    tr.create(solution.getAssignment(), model.getProperties()).save(new File(outDir, "tableau.csv"));
582                } catch (IOException e) {
583                    sLog.error(e.getMessage(), e);
584                }
585            }
586
587            solution.saveBest();
588        }
589
590        if (computeSectInfos)
591            model.computeOnlineSectioningInfos(solution.getAssignment());
592
593        if (runChecks) {
594            try {
595                if (model.getProperties().getPropertyBoolean("Test.InevitableStudentConflictsCheck", false)) {
596                    InevitableStudentConflicts ch = new InevitableStudentConflicts(model);
597                    if (!ch.check(solution.getAssignment()))
598                        ch.getCSVFile().save(
599                                new File(new File(model.getProperties().getProperty("General.Output", ".")),
600                                        "inevitable-conflicts.csv"));
601                }
602            } catch (IOException e) {
603                sLog.error(e.getMessage(), e);
604            }
605            new OverlapCheck(model).check(solution.getAssignment());
606            new SectionLimitCheck(model).check(solution.getAssignment());
607            try {
608                CourseLimitCheck ch = new CourseLimitCheck(model);
609                if (!ch.check())
610                    ch.getCSVFile().save(
611                            new File(new File(model.getProperties().getProperty("General.Output", ".")),
612                                    "course-limits.csv"));
613            } catch (IOException e) {
614                sLog.error(e.getMessage(), e);
615            }
616        }
617
618        sLog.info("Best solution found after " + solution.getBestTime() + " seconds (" + solution.getBestIteration()
619                + " iterations).");
620        sLog.info("Info: " + ToolBox.dict2string(solution.getExtendedInfo(), 2));
621    }
622
623    /** Solve the student sectioning problem using IFS solver 
624     * @param solution current solution
625     * @param cfg solver configuration
626     * @return resultant solution
627     **/
628    public static Solution<Request, Enrollment> solve(Solution<Request, Enrollment> solution, DataProperties cfg) {
629        int nrSolvers = cfg.getPropertyInt("Parallel.NrSolvers", 1);
630        Solver<Request, Enrollment> solver = (nrSolvers == 1 ? new Solver<Request, Enrollment>(cfg) : new ParallelSolver<Request, Enrollment>(cfg));
631        solver.setInitalSolution(solution);
632        if (cfg.getPropertyBoolean("Test.Verbose", false)) {
633            solver.addSolverListener(new SolverListener<Request, Enrollment>() {
634                @Override
635                public boolean variableSelected(Assignment<Request, Enrollment> assignment, long iteration, Request variable) {
636                    return true;
637                }
638
639                @Override
640                public boolean valueSelected(Assignment<Request, Enrollment> assignment, long iteration, Request variable, Enrollment value) {
641                    return true;
642                }
643
644                @Override
645                public boolean neighbourSelected(Assignment<Request, Enrollment> assignment, long iteration, Neighbour<Request, Enrollment> neighbour) {
646                    sLog.debug("Select[" + iteration + "]: " + neighbour);
647                    return true;
648                }
649
650                @Override
651                public void neighbourFailed(Assignment<Request, Enrollment> assignment, long iteration, Neighbour<Request, Enrollment> neighbour) {
652                    sLog.debug("Failed[" + iteration + "]: " + neighbour);
653                }
654            });
655        }
656        solution.addSolutionListener(new TestSolutionListener());
657        
658        Runtime.getRuntime().addShutdownHook(new ShutdownHook(solver));
659
660        solver.start();
661        try {
662            solver.getSolverThread().join();
663        } catch (InterruptedException e) {
664        }
665
666        return solution;
667    }
668
669    /**
670     * Compute last-like student weight for the given course
671     * 
672     * @param course
673     *            given course
674     * @param real
675     *            number of real students for the course
676     * @param lastLike
677     *            number of last-like students for the course
678     * @return weight of a student request for the given course
679     */
680    public static double getLastLikeStudentWeight(Course course, int real, int lastLike) {
681        int projected = course.getProjected();
682        int limit = course.getLimit();
683        if (course.getLimit() < 0) {
684            sLog.debug("  -- Course " + course.getName() + " is unlimited.");
685            return 1.0;
686        }
687        if (projected <= 0) {
688            sLog.warn("  -- No projected demand for course " + course.getName() + ", using course limit (" + limit
689                    + ")");
690            projected = limit;
691        } else if (limit < projected) {
692            sLog.warn("  -- Projected number of students is over course limit for course " + course.getName() + " ("
693                    + Math.round(projected) + ">" + limit + ")");
694            projected = limit;
695        }
696        if (lastLike == 0) {
697            sLog.warn("  -- No last like info for course " + course.getName());
698            return 1.0;
699        }
700        double weight = ((double) Math.max(0, projected - real)) / lastLike;
701        sLog.debug("  -- last like student weight for " + course.getName() + " is " + weight + " (lastLike=" + lastLike
702                + ", real=" + real + ", projected=" + projected + ")");
703        return weight;
704    }
705
706    /**
707     * Load last-like students from an XML file (the one that is used to load
708     * last like course demands table in the timetabling application)
709     * @param model problem model
710     * @param xml an XML file
711     */
712    public static void loadLastLikeCourseDemandsXml(StudentSectioningModel model, File xml) {
713        try {
714            Document document = (new SAXReader()).read(xml);
715            Element root = document.getRootElement();
716            HashMap<Course, List<Request>> requests = new HashMap<Course, List<Request>>();
717            long reqId = 0;
718            for (Iterator<?> i = root.elementIterator("student"); i.hasNext();) {
719                Element studentEl = (Element) i.next();
720                Student student = new Student(Long.parseLong(studentEl.attributeValue("externalId")));
721                student.setDummy(true);
722                int priority = 0;
723                HashSet<Course> reqCourses = new HashSet<Course>();
724                for (Iterator<?> j = studentEl.elementIterator("studentCourse"); j.hasNext();) {
725                    Element courseEl = (Element) j.next();
726                    String subjectArea = courseEl.attributeValue("subject");
727                    String courseNbr = courseEl.attributeValue("courseNumber");
728                    Course course = null;
729                    offerings: for (Offering offering : model.getOfferings()) {
730                        for (Course c : offering.getCourses()) {
731                            if (c.getSubjectArea().equals(subjectArea) && c.getCourseNumber().equals(courseNbr)) {
732                                course = c;
733                                break offerings;
734                            }
735                        }
736                    }
737                    if (course == null && courseNbr.charAt(courseNbr.length() - 1) >= 'A'
738                            && courseNbr.charAt(courseNbr.length() - 1) <= 'Z') {
739                        String courseNbrNoSfx = courseNbr.substring(0, courseNbr.length() - 1);
740                        offerings: for (Offering offering : model.getOfferings()) {
741                            for (Course c : offering.getCourses()) {
742                                if (c.getSubjectArea().equals(subjectArea)
743                                        && c.getCourseNumber().equals(courseNbrNoSfx)) {
744                                    course = c;
745                                    break offerings;
746                                }
747                            }
748                        }
749                    }
750                    if (course == null) {
751                        sLog.warn("Course " + subjectArea + " " + courseNbr + " not found.");
752                    } else {
753                        if (!reqCourses.add(course)) {
754                            sLog.warn("Course " + subjectArea + " " + courseNbr + " already requested.");
755                        } else {
756                            List<Course> courses = new ArrayList<Course>(1);
757                            courses.add(course);
758                            CourseRequest request = new CourseRequest(reqId++, priority++, false, student, courses, false, null);
759                            List<Request> requestsThisCourse = requests.get(course);
760                            if (requestsThisCourse == null) {
761                                requestsThisCourse = new ArrayList<Request>();
762                                requests.put(course, requestsThisCourse);
763                            }
764                            requestsThisCourse.add(request);
765                        }
766                    }
767                }
768                if (!student.getRequests().isEmpty())
769                    model.addStudent(student);
770            }
771            for (Map.Entry<Course, List<Request>> entry : requests.entrySet()) {
772                Course course = entry.getKey();
773                List<Request> requestsThisCourse = entry.getValue();
774                double weight = getLastLikeStudentWeight(course, 0, requestsThisCourse.size());
775                for (Request request : requestsThisCourse) {
776                    request.setWeight(weight);
777                }
778            }
779        } catch (Exception e) {
780            sLog.error(e.getMessage(), e);
781        }
782    }
783
784    /**
785     * Load course request from the given files (in the format being used by the
786     * old MSF system)
787     * 
788     * @param model
789     *            student sectioning model (with offerings loaded)
790     * @param files
791     *            semi-colon separated list of files to be loaded
792     */
793    public static void loadCrsReqFiles(StudentSectioningModel model, String files) {
794        try {
795            boolean lastLike = model.getProperties().getPropertyBoolean("Test.CrsReqIsLastLike", true);
796            boolean shuffleIds = model.getProperties().getPropertyBoolean("Test.CrsReqShuffleStudentIds", true);
797            boolean tryWithoutSuffix = model.getProperties().getPropertyBoolean("Test.CrsReqTryWithoutSuffix", false);
798            HashMap<Long, Student> students = new HashMap<Long, Student>();
799            long reqId = 0;
800            for (StringTokenizer stk = new StringTokenizer(files, ";"); stk.hasMoreTokens();) {
801                String file = stk.nextToken();
802                sLog.debug("Loading " + file + " ...");
803                BufferedReader in = new BufferedReader(new FileReader(file));
804                String line;
805                int lineIndex = 0;
806                while ((line = in.readLine()) != null) {
807                    lineIndex++;
808                    if (line.length() <= 150)
809                        continue;
810                    char code = line.charAt(13);
811                    if (code == 'H' || code == 'T')
812                        continue; // skip header and tail
813                    long studentId = Long.parseLong(line.substring(14, 23));
814                    Student student = students.get(Long.valueOf(studentId));
815                    if (student == null) {
816                        student = new Student(studentId);
817                        if (lastLike)
818                            student.setDummy(true);
819                        students.put(Long.valueOf(studentId), student);
820                        sLog.debug("  -- loading student " + studentId + " ...");
821                    } else
822                        sLog.debug("  -- updating student " + studentId + " ...");
823                    line = line.substring(150);
824                    while (line.length() >= 20) {
825                        String subjectArea = line.substring(0, 4).trim();
826                        String courseNbr = line.substring(4, 8).trim();
827                        if (subjectArea.length() == 0 || courseNbr.length() == 0) {
828                            line = line.substring(20);
829                            continue;
830                        }
831                        /*
832                         * // UNUSED String instrSel = line.substring(8,10);
833                         * //ZZ - Remove previous instructor selection char
834                         * reqPDiv = line.charAt(10); //P - Personal preference;
835                         * C - Conflict resolution; //0 - (Zero) used by program
836                         * only, for change requests to reschedule division //
837                         * (used to reschedule canceled division) String reqDiv
838                         * = line.substring(11,13); //00 - Reschedule division
839                         * String reqSect = line.substring(13,15); //Contains
840                         * designator for designator-required courses String
841                         * credit = line.substring(15,19); char nameRaise =
842                         * line.charAt(19); //N - Name raise
843                         */
844                        char action = line.charAt(19); // A - Add; D - Drop; C -
845                                                       // Change
846                        sLog.debug("    -- requesting " + subjectArea + " " + courseNbr + " (action:" + action
847                                + ") ...");
848                        Course course = null;
849                        offerings: for (Offering offering : model.getOfferings()) {
850                            for (Course c : offering.getCourses()) {
851                                if (c.getSubjectArea().equals(subjectArea) && c.getCourseNumber().equals(courseNbr)) {
852                                    course = c;
853                                    break offerings;
854                                }
855                            }
856                        }
857                        if (course == null && tryWithoutSuffix && courseNbr.charAt(courseNbr.length() - 1) >= 'A'
858                                && courseNbr.charAt(courseNbr.length() - 1) <= 'Z') {
859                            String courseNbrNoSfx = courseNbr.substring(0, courseNbr.length() - 1);
860                            offerings: for (Offering offering : model.getOfferings()) {
861                                for (Course c : offering.getCourses()) {
862                                    if (c.getSubjectArea().equals(subjectArea)
863                                            && c.getCourseNumber().equals(courseNbrNoSfx)) {
864                                        course = c;
865                                        break offerings;
866                                    }
867                                }
868                            }
869                        }
870                        if (course == null) {
871                            if (courseNbr.charAt(courseNbr.length() - 1) >= 'A'
872                                    && courseNbr.charAt(courseNbr.length() - 1) <= 'Z') {
873                            } else {
874                                sLog.warn("      -- course " + subjectArea + " " + courseNbr + " not found (file "
875                                        + file + ", line " + lineIndex + ")");
876                            }
877                        } else {
878                            CourseRequest courseRequest = null;
879                            for (Request request : student.getRequests()) {
880                                if (request instanceof CourseRequest
881                                        && ((CourseRequest) request).getCourses().contains(course)) {
882                                    courseRequest = (CourseRequest) request;
883                                    break;
884                                }
885                            }
886                            if (action == 'A') {
887                                if (courseRequest == null) {
888                                    List<Course> courses = new ArrayList<Course>(1);
889                                    courses.add(course);
890                                    courseRequest = new CourseRequest(reqId++, student.getRequests().size(), false, student, courses, false, null);
891                                } else {
892                                    sLog.warn("      -- request for course " + course + " is already present");
893                                }
894                            } else if (action == 'D') {
895                                if (courseRequest == null) {
896                                    sLog.warn("      -- request for course " + course
897                                            + " is not present -- cannot be dropped");
898                                } else {
899                                    student.getRequests().remove(courseRequest);
900                                }
901                            } else if (action == 'C') {
902                                if (courseRequest == null) {
903                                    sLog.warn("      -- request for course " + course
904                                            + " is not present -- cannot be changed");
905                                } else {
906                                    // ?
907                                }
908                            } else {
909                                sLog.warn("      -- unknown action " + action);
910                            }
911                        }
912                        line = line.substring(20);
913                    }
914                }
915                in.close();
916            }
917            HashMap<Course, List<Request>> requests = new HashMap<Course, List<Request>>();
918            Set<Long> studentIds = new HashSet<Long>();
919            for (Student student: students.values()) {
920                if (!student.getRequests().isEmpty())
921                    model.addStudent(student);
922                if (shuffleIds) {
923                    long newId = -1;
924                    while (true) {
925                        newId = 1 + (long) (999999999L * Math.random());
926                        if (studentIds.add(Long.valueOf(newId)))
927                            break;
928                    }
929                    student.setId(newId);
930                }
931                if (student.isDummy()) {
932                    for (Request request : student.getRequests()) {
933                        if (request instanceof CourseRequest) {
934                            Course course = ((CourseRequest) request).getCourses().get(0);
935                            List<Request> requestsThisCourse = requests.get(course);
936                            if (requestsThisCourse == null) {
937                                requestsThisCourse = new ArrayList<Request>();
938                                requests.put(course, requestsThisCourse);
939                            }
940                            requestsThisCourse.add(request);
941                        }
942                    }
943                }
944            }
945            Collections.sort(model.getStudents(), new Comparator<Student>() {
946                @Override
947                public int compare(Student o1, Student o2) {
948                    return Double.compare(o1.getId(), o2.getId());
949                }
950            });
951            for (Map.Entry<Course, List<Request>> entry : requests.entrySet()) {
952                Course course = entry.getKey();
953                List<Request> requestsThisCourse = entry.getValue();
954                double weight = getLastLikeStudentWeight(course, 0, requestsThisCourse.size());
955                for (Request request : requestsThisCourse) {
956                    request.setWeight(weight);
957                }
958            }
959            if (model.getProperties().getProperty("Test.EtrChk") != null) {
960                for (StringTokenizer stk = new StringTokenizer(model.getProperties().getProperty("Test.EtrChk"), ";"); stk
961                        .hasMoreTokens();) {
962                    String file = stk.nextToken();
963                    sLog.debug("Loading " + file + " ...");
964                    BufferedReader in = new BufferedReader(new FileReader(file));
965                    try {
966                        String line;
967                        while ((line = in.readLine()) != null) {
968                            if (line.length() < 55)
969                                continue;
970                            char code = line.charAt(12);
971                            if (code == 'H' || code == 'T')
972                                continue; // skip header and tail
973                            if (code == 'D' || code == 'K')
974                                continue; // skip delete nad cancel
975                            long studentId = Long.parseLong(line.substring(2, 11));
976                            Student student = students.get(Long.valueOf(studentId));
977                            if (student == null) {
978                                sLog.info("  -- student " + studentId + " not found");
979                                continue;
980                            }
981                            sLog.info("  -- reading student " + studentId);
982                            String area = line.substring(15, 18).trim();
983                            if (area.length() == 0)
984                                continue;
985                            String clasf = line.substring(18, 20).trim();
986                            String major = line.substring(21, 24).trim();
987                            String minor = line.substring(24, 27).trim();
988                            student.getAreaClassificationMajors().clear();
989                            student.getAreaClassificationMinors().clear();
990                            if (major.length() > 0)
991                                student.getAreaClassificationMajors().add(new AreaClassificationMajor(area, clasf, major));
992                            if (minor.length() > 0)
993                                student.getAreaClassificationMajors().add(new AreaClassificationMajor(area, clasf, minor));
994                        }
995                    } finally {
996                        in.close();
997                    }
998                }
999            }
1000            int without = 0;
1001            for (Student student: students.values()) {
1002                if (student.getAreaClassificationMajors().isEmpty())
1003                    without++;
1004            }
1005            fixPriorities(model);
1006            sLog.info("Students without academic area: " + without);
1007        } catch (Exception e) {
1008            sLog.error(e.getMessage(), e);
1009        }
1010    }
1011
1012    public static void fixPriorities(StudentSectioningModel model) {
1013        for (Student student : model.getStudents()) {
1014            Collections.sort(student.getRequests(), new Comparator<Request>() {
1015                @Override
1016                public int compare(Request r1, Request r2) {
1017                    int cmp = Double.compare(r1.getPriority(), r2.getPriority());
1018                    if (cmp != 0)
1019                        return cmp;
1020                    return Double.compare(r1.getId(), r2.getId());
1021                }
1022            });
1023            int priority = 0;
1024            for (Request request : student.getRequests()) {
1025                if (priority != request.getPriority()) {
1026                    sLog.debug("Change priority of " + request + " to " + priority);
1027                    request.setPriority(priority);
1028                }
1029            }
1030        }
1031    }
1032
1033    /** Save solution info as XML 
1034     * @param solution current solution
1035     * @param extra solution extra info
1036     * @param file file to write
1037     **/
1038    public static void saveInfoToXML(Solution<Request, Enrollment> solution, Map<String, String> extra, File file) {
1039        FileOutputStream fos = null;
1040        try {
1041            Document document = DocumentHelper.createDocument();
1042            document.addComment("Solution Info");
1043
1044            Element root = document.addElement("info");
1045            TreeSet<Map.Entry<String, String>> entrySet = new TreeSet<Map.Entry<String, String>>(
1046                    new Comparator<Map.Entry<String, String>>() {
1047                        @Override
1048                        public int compare(Map.Entry<String, String> e1, Map.Entry<String, String> e2) {
1049                            return e1.getKey().compareTo(e2.getKey());
1050                        }
1051                    });
1052            entrySet.addAll(solution.getExtendedInfo().entrySet());
1053            if (extra != null)
1054                entrySet.addAll(extra.entrySet());
1055            for (Map.Entry<String, String> entry : entrySet) {
1056                root.addElement("property").addAttribute("name", entry.getKey()).setText(entry.getValue());
1057            }
1058
1059            fos = new FileOutputStream(file);
1060            (new XMLWriter(fos, OutputFormat.createPrettyPrint())).write(document);
1061            fos.flush();
1062            fos.close();
1063            fos = null;
1064        } catch (Exception e) {
1065            sLog.error("Unable to save info, reason: " + e.getMessage(), e);
1066        } finally {
1067            try {
1068                if (fos != null)
1069                    fos.close();
1070            } catch (IOException e) {
1071            }
1072        }
1073    }
1074
1075    private static void fixWeights(StudentSectioningModel model) {
1076        HashMap<Course, Integer> lastLike = new HashMap<Course, Integer>();
1077        HashMap<Course, Integer> real = new HashMap<Course, Integer>();
1078        HashSet<Long> lastLikeIds = new HashSet<Long>();
1079        HashSet<Long> realIds = new HashSet<Long>();
1080        for (Student student : model.getStudents()) {
1081            if (student.isDummy()) {
1082                if (!lastLikeIds.add(Long.valueOf(student.getId()))) {
1083                    sLog.error("Two last-like student with id " + student.getId());
1084                }
1085            } else {
1086                if (!realIds.add(Long.valueOf(student.getId()))) {
1087                    sLog.error("Two real student with id " + student.getId());
1088                }
1089            }
1090            for (Request request : student.getRequests()) {
1091                if (request instanceof CourseRequest) {
1092                    CourseRequest courseRequest = (CourseRequest) request;
1093                    Course course = courseRequest.getCourses().get(0);
1094                    Integer cnt = (student.isDummy() ? lastLike : real).get(course);
1095                    (student.isDummy() ? lastLike : real).put(course, Integer.valueOf(
1096                            (cnt == null ? 0 : cnt.intValue()) + 1));
1097                }
1098            }
1099        }
1100        for (Student student : new ArrayList<Student>(model.getStudents())) {
1101            if (student.isDummy() && realIds.contains(Long.valueOf(student.getId()))) {
1102                sLog.warn("There is both last-like and real student with id " + student.getId());
1103                long newId = -1;
1104                while (true) {
1105                    newId = 1 + (long) (999999999L * Math.random());
1106                    if (!realIds.contains(Long.valueOf(newId)) && !lastLikeIds.contains(Long.valueOf(newId)))
1107                        break;
1108                }
1109                lastLikeIds.remove(Long.valueOf(student.getId()));
1110                lastLikeIds.add(Long.valueOf(newId));
1111                student.setId(newId);
1112                sLog.warn("  -- last-like student id changed to " + student.getId());
1113            }
1114            for (Request request : new ArrayList<Request>(student.getRequests())) {
1115                if (!student.isDummy()) {
1116                    request.setWeight(1.0);
1117                    continue;
1118                }
1119                if (request instanceof CourseRequest) {
1120                    CourseRequest courseRequest = (CourseRequest) request;
1121                    Course course = courseRequest.getCourses().get(0);
1122                    Integer lastLikeCnt = lastLike.get(course);
1123                    Integer realCnt = real.get(course);
1124                    courseRequest.setWeight(getLastLikeStudentWeight(course, realCnt == null ? 0 : realCnt.intValue(),
1125                            lastLikeCnt == null ? 0 : lastLikeCnt.intValue()));
1126                } else
1127                    request.setWeight(1.0);
1128                if (request.getWeight() <= 0.0) {
1129                    model.removeVariable(request);
1130                    student.getRequests().remove(request);
1131                }
1132            }
1133            if (student.getRequests().isEmpty()) {
1134                model.getStudents().remove(student);
1135            }
1136        }
1137    }
1138
1139    /** Combine students from the provided two files 
1140     * @param cfg solver configuration
1141     * @param lastLikeStudentData a file containing last-like student data
1142     * @param realStudentData a file containing real student data
1143     * @return combined solution
1144     **/
1145    public static Solution<Request, Enrollment> combineStudents(DataProperties cfg, File lastLikeStudentData, File realStudentData) {
1146        try {
1147            RandomStudentFilter rnd = new RandomStudentFilter(1.0);
1148
1149            StudentSectioningModel model = null;
1150            Assignment<Request, Enrollment> assignment = new DefaultSingleAssignment<Request, Enrollment>();
1151
1152            for (StringTokenizer stk = new StringTokenizer(cfg.getProperty("Test.CombineAcceptProb", "1.0"), ","); stk.hasMoreTokens();) {
1153                double acceptProb = Double.parseDouble(stk.nextToken());
1154                sLog.info("Test.CombineAcceptProb=" + acceptProb);
1155                rnd.setProbability(acceptProb);
1156
1157                StudentFilter batchFilter = new CombinedStudentFilter(new ReverseStudentFilter(
1158                        new FreshmanStudentFilter()), rnd, CombinedStudentFilter.OP_AND);
1159
1160                model = new StudentSectioningModel(cfg);
1161                StudentSectioningXMLLoader loader = new StudentSectioningXMLLoader(model, assignment);
1162                loader.setLoadStudents(false);
1163                loader.load();
1164
1165                StudentSectioningXMLLoader lastLikeLoader = new StudentSectioningXMLLoader(model, assignment);
1166                lastLikeLoader.setInputFile(lastLikeStudentData);
1167                lastLikeLoader.setLoadOfferings(false);
1168                lastLikeLoader.setLoadStudents(true);
1169                lastLikeLoader.load();
1170
1171                StudentSectioningXMLLoader realLoader = new StudentSectioningXMLLoader(model, assignment);
1172                realLoader.setInputFile(realStudentData);
1173                realLoader.setLoadOfferings(false);
1174                realLoader.setLoadStudents(true);
1175                realLoader.setStudentFilter(batchFilter);
1176                realLoader.load();
1177
1178                fixWeights(model);
1179
1180                fixPriorities(model);
1181
1182                Solver<Request, Enrollment> solver = new Solver<Request, Enrollment>(model.getProperties());
1183                solver.setInitalSolution(model);
1184                new StudentSectioningXMLSaver(solver).save(new File(new File(model.getProperties().getProperty(
1185                        "General.Output", ".")), "solution-r" + ((int) (100.0 * acceptProb)) + ".xml"));
1186
1187            }
1188
1189            return model == null ? null : new Solution<Request, Enrollment>(model, assignment);
1190
1191        } catch (Exception e) {
1192            sLog.error("Unable to combine students, reason: " + e.getMessage(), e);
1193            return null;
1194        }
1195    }
1196    
1197    /** Main 
1198     * @param args program arguments
1199     **/
1200    public static void main(String[] args) {
1201        try {
1202            DataProperties cfg = new DataProperties();
1203            cfg.setProperty("Termination.Class", "org.cpsolver.ifs.termination.GeneralTerminationCondition");
1204            cfg.setProperty("Termination.StopWhenComplete", "true");
1205            cfg.setProperty("Termination.TimeOut", "600");
1206            cfg.setProperty("Comparator.Class", "org.cpsolver.ifs.solution.GeneralSolutionComparator");
1207            cfg.setProperty("Value.Class", "org.cpsolver.studentsct.heuristics.EnrollmentSelection");// org.cpsolver.ifs.heuristics.GeneralValueSelection
1208            cfg.setProperty("Value.WeightConflicts", "1.0");
1209            cfg.setProperty("Value.WeightNrAssignments", "0.0");
1210            cfg.setProperty("Variable.Class", "org.cpsolver.ifs.heuristics.GeneralVariableSelection");
1211            cfg.setProperty("Neighbour.Class", "org.cpsolver.studentsct.heuristics.StudentSctNeighbourSelection");
1212            cfg.setProperty("General.SaveBestUnassigned", "0");
1213            cfg.setProperty("Extensions.Classes",
1214                    "org.cpsolver.ifs.extension.ConflictStatistics;org.cpsolver.studentsct.extension.DistanceConflict" +
1215                    ";org.cpsolver.studentsct.extension.TimeOverlapsCounter");
1216            cfg.setProperty("Data.Initiative", "puWestLafayetteTrdtn");
1217            cfg.setProperty("Data.Term", "Fal");
1218            cfg.setProperty("Data.Year", "2007");
1219            cfg.setProperty("General.Input", "pu-sectll-fal07-s.xml");
1220            if (args.length >= 1) {
1221                cfg.load(new FileInputStream(args[0]));
1222            }
1223            cfg.putAll(System.getProperties());
1224
1225            if (args.length >= 2) {
1226                cfg.setProperty("General.Input", args[1]);
1227            }
1228
1229            File outDir = null;
1230            if (args.length >= 3) {
1231                outDir = new File(args[2], sDateFormat.format(new Date()));
1232            } else if (cfg.getProperty("General.Output") != null) {
1233                outDir = new File(cfg.getProperty("General.Output", "."), sDateFormat.format(new Date()));
1234            } else {
1235                outDir = new File(System.getProperty("user.home", ".") + File.separator + "Sectioning-Test" + File.separator + (sDateFormat.format(new Date())));
1236            }
1237            outDir.mkdirs();
1238            ToolBox.setupLogging(new File(outDir, "debug.log"), "true".equals(System.getProperty("debug", "false")));
1239            cfg.setProperty("General.Output", outDir.getAbsolutePath());
1240
1241            if (args.length >= 4 && "online".equals(args[3])) {
1242                onlineSectioning(cfg);
1243            } else if (args.length >= 4 && "simple".equals(args[3])) {
1244                cfg.setProperty("Sectioning.UseOnlinePenalties", "false");
1245                onlineSectioning(cfg);
1246            } else {
1247                batchSectioning(cfg);
1248            }
1249        } catch (Exception e) {
1250            sLog.error(e.getMessage(), e);
1251            e.printStackTrace();
1252        }
1253    }
1254
1255    public static class ExtraStudentFilter implements StudentFilter {
1256        HashSet<Long> iIds = new HashSet<Long>();
1257
1258        public ExtraStudentFilter(StudentSectioningModel model) {
1259            for (Student student : model.getStudents()) {
1260                iIds.add(Long.valueOf(student.getId()));
1261            }
1262        }
1263
1264        @Override
1265        public boolean accept(Student student) {
1266            return !iIds.contains(Long.valueOf(student.getId()));
1267        }
1268
1269        @Override
1270        public String getName() {
1271            return "Extra";
1272        }
1273    }
1274
1275    public static class TestSolutionListener implements SolutionListener<Request, Enrollment> {
1276        @Override
1277        public void solutionUpdated(Solution<Request, Enrollment> solution) {
1278            StudentSectioningModel m = (StudentSectioningModel) solution.getModel();
1279            if (m.getTimeOverlaps() != null && TimeOverlapsCounter.sDebug)
1280                m.getTimeOverlaps().checkTotalNrConflicts(solution.getAssignment());
1281            if (m.getDistanceConflict() != null && DistanceConflict.sDebug)
1282                m.getDistanceConflict().checkAllConflicts(solution.getAssignment());
1283            if (m.getStudentQuality() != null && m.getStudentQuality().isDebug())
1284                m.getStudentQuality().checkTotalPenalty(solution.getAssignment());
1285        }
1286
1287        @Override
1288        public void getInfo(Solution<Request, Enrollment> solution, Map<String, String> info) {
1289        }
1290
1291        @Override
1292        public void getInfo(Solution<Request, Enrollment> solution, Map<String, String> info, Collection<Request> variables) {
1293        }
1294
1295        @Override
1296        public void bestCleared(Solution<Request, Enrollment> solution) {
1297        }
1298
1299        @Override
1300        public void bestSaved(Solution<Request, Enrollment> solution) {
1301            sLog.info("**BEST** " + ((StudentSectioningModel)solution.getModel()).toString(solution.getAssignment()) + ", TM:" + sDF.format(solution.getTime() / 3600.0) + "h" +
1302                    (solution.getFailedIterations() > 0 ? ", F:" + sDF.format(100.0 * solution.getFailedIterations() / solution.getIteration()) + "%" : ""));
1303        }
1304
1305        @Override
1306        public void bestRestored(Solution<Request, Enrollment> solution) {
1307        }
1308    }
1309    
1310    private static class ShutdownHook extends Thread {
1311        Solver<Request, Enrollment> iSolver = null;
1312        Map<String, String> iExtra = null;
1313
1314        private ShutdownHook(Solver<Request, Enrollment> solver) {
1315            setName("ShutdownHook");
1316            iSolver = solver;
1317        }
1318        
1319        void setExtra(Map<String, String> extra) { iExtra = extra; }
1320        
1321        @Override
1322        public void run() {
1323            try {
1324                if (iSolver.isRunning()) iSolver.stopSolver();
1325                Solution<Request, Enrollment> solution = iSolver.lastSolution();
1326                solution.restoreBest();
1327                DataProperties cfg = iSolver.getProperties();
1328                
1329                printInfo(solution,
1330                        cfg.getPropertyBoolean("Test.CreateReports", true),
1331                        cfg.getPropertyBoolean("Test.ComputeSectioningInfo", true),
1332                        cfg.getPropertyBoolean("Test.RunChecks", true));
1333
1334                try {
1335                    new StudentSectioningXMLSaver(iSolver).save(new File(new File(cfg.getProperty("General.Output", ".")), "solution.xml"));
1336                } catch (Exception e) {
1337                    sLog.error("Unable to save solution, reason: " + e.getMessage(), e);
1338                }
1339                
1340                saveInfoToXML(solution, iExtra, new File(new File(cfg.getProperty("General.Output", ".")), "info.xml"));
1341                
1342                Progress.removeInstance(solution.getModel());
1343            } catch (Throwable t) {
1344                sLog.error("Test failed.", t);
1345            }
1346        }
1347    }
1348
1349}