NIHVIVO-222 Complete the batch version.

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/FileHelper.java

@@ -0,0 +1,133 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Some utility methods for dealing with files and directories.
+ */
+public class FileHelper {
+
+	/**
+	 * Delete a file. If it can't be deleted, complain.
+	 */
+	public static void deleteFile(File file) throws IOException {
+		if (file.exists()) {
+			file.delete();
+		}
+		if (!file.exists()) {
+			return;
+		}
+
+		/*
+		 * If we were unable to delete the file, is it because it's a non-empty
+		 * directory?
+		 */
+		if (file.isDirectory()) {
+			final StringBuffer message = new StringBuffer(
+					"Can't delete directory '" + file.getPath() + "'\n");
+			file.listFiles(new FileFilter() {
+				public boolean accept(File pathname) {
+					message.append("   contains file '" + pathname + "'\n");
+					return true;
+				}
+			});
+			throw new IOException(message.toString().trim());
+		} else {
+			throw new IOException("Unable to delete file '" + file.getPath()
+					+ "'");
+		}
+	}
+
+	/**
+	 * Delete all of the files in a directory, any sub-directories, and the
+	 * directory itself.
+	 */
+	public static void purgeDirectoryRecursively(File directory)
+			throws IOException {
+		File[] files = directory.listFiles();
+		for (File file : files) {
+			if (file.isDirectory()) {
+				purgeDirectoryRecursively(file);
+			} else {
+				deleteFile(file);
+			}
+		}
+		deleteFile(directory);
+	}
+
+	/**
+	 * Confirm that this is an existing, readable file.
+	 */
+	public static void checkReadableFile(File file, String label) {
+		if (!file.exists()) {
+			throw new IllegalArgumentException(label + " does not exist.");
+		}
+		if (!file.isFile()) {
+			throw new IllegalArgumentException(label + " is not a file.");
+		}
+		if (!file.canRead()) {
+			throw new IllegalArgumentException(label + " is not readable.");
+		}
+	}
+
+	/**
+	 * Get the name of this file, without the extension.
+	 */
+	public static String baseName(File file) {
+		String name = file.getName();
+		int periodHere = name.indexOf('.');
+		if (periodHere == -1) {
+			return name;
+		} else {
+			return name.substring(0, periodHere);
+		}
+	}
+
+	/**
+	 * Copy the contents of a file to a new location. If the target file already
+	 * exists, it will be over-written.
+	 */
+	public static void copy(File source, File target) throws IOException {
+		InputStream input = null;
+		OutputStream output = null;
+
+		try {
+			input = new FileInputStream(source);
+			output = new FileOutputStream(target);
+			int howMany;
+			byte[] buffer = new byte[4096];
+			while (-1 != (howMany = input.read(buffer))) {
+				output.write(buffer, 0, howMany);
+			}
+		}finally {
+			if (input != null) {
+				try {
+					input.close();
+				} catch (IOException e1) {
+					e1.printStackTrace();
+				}
+			}
+			if (output != null) {
+				try {
+					output.close();
+				} catch (IOException e1) {
+					e1.printStackTrace();
+				}
+			}
+		}
+	}
+
+	/** No need to instantiate this, since all methods are static. */
+	private FileHelper() {
+		// Nothing to initialize.
+	}
+
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java

@@ -0,0 +1,130 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A list of tests to be ignored - they are expected to fail, and their failure
+ * is logged with a warning, not an error.
+ */
+public class IgnoredTests {
+	private final File file;
+	private final List<IgnoredTestInfo> tests;
+
+	/**
+	 * <p>
+	 * Parse the file of ignored tests.
+	 * </p>
+	 * <p>
+	 * Ignore any blank line, or any line starting with '#' or '!'
+	 * </p>
+	 * <p>
+	 * Each other line describes an ignored test. The line contains the suite
+	 * name, a comma (with optional space), the test name (with optional space)
+	 * and optionally a comment, starting with a '#'.
+	 * </p>
+	 */
+	public IgnoredTests(File file) {
+		this.file = file;
+		List<IgnoredTestInfo> tests = new ArrayList<IgnoredTestInfo>();
+
+		BufferedReader reader = null;
+		try {
+			reader = new BufferedReader(new FileReader(file));
+			String line;
+			while (null != (line = reader.readLine())) {
+				line = line.trim();
+				if ((line.length() == 0) || (line.charAt(0) == '#')
+						|| (line.charAt(0) == '!')) {
+					continue;
+				}
+				Pattern p = Pattern.compile("([^,#]+),([^,#]+)(#(.*))?");
+				Matcher m = p.matcher(line);
+				if (m.matches()) {
+					tests.add(new IgnoredTestInfo(m.group(1), m.group(2), m
+							.group(4)));
+				} else {
+					throw new FatalException(
+							"Bad format on ignored test description: '" + line
+									+ "', should be "
+									+ "<suite name>, <test name> [# comment]");
+				}
+			}
+		} catch (IOException e) {
+			throw new FatalException(
+					"Failed to parse the list of ignored tests: '"
+							+ file.getPath() + "'", e);
+		} finally {
+			if (reader != null) {
+				try {
+					reader.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		this.tests = Collections.unmodifiableList(tests);
+	}
+
+	/**
+	 * Is this test ignored or not?
+	 */
+	public boolean isIgnored(String suiteName, String testName) {
+		for (IgnoredTestInfo test : tests) {
+			if (test.matches(suiteName, testName)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * If this test is ignored, what is the reason? If not, return an empty
+	 * string.
+	 */
+	public String getReasonForIgnoring(String suiteName, String testName) {
+		for (IgnoredTestInfo test : tests) {
+			if (test.matches(suiteName, testName)) {
+				return test.comment;
+			}
+		}
+		return "";
+	}
+
+	public String toString() {
+		String s = "  ignored tests from " + file.getPath() + "\n";
+		for (IgnoredTestInfo test : tests) {
+			s += "      " + test.suiteName + ", " + test.testName + "\n";
+		}
+		return s;
+	}
+
+	private static class IgnoredTestInfo {
+		final String suiteName;
+		final String testName;
+		final String comment;
+
+		public IgnoredTestInfo(String suiteName, String testName, String comment) {
+			this.suiteName = suiteName.trim();
+			this.testName = testName.trim();
+			this.comment = (comment == null) ? "" : comment.trim();
+		}
+
+		public boolean matches(String suiteName, String testName) {
+			return this.suiteName.equals(suiteName)
+					&& this.testName.equals(testName);
+		}
+
+	}
+
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Listener.java

@@ -49,9 +49,26 @@ public class Listener {
 		log(e);
 	}
 
+	public void runEndTime() {
+		log("Testing complete.");
+	}
+
 	public void runStopped() {
 		log("Run stopped.");
 	}
+	
+	public void cleanOutputStart(File outputDirectory) {
+		log("Output area cleaning started: " + outputDirectory.getPath());
+	}
+
+	public void cleanOutputFailed(File outputDirectory, IOException e) {
+		log("Output area cleaning failed: " + outputDirectory.getPath());
+		log(e);
+	}
+
+	public void cleanOutputStop(File outputDirectory) {
+		log("Output area cleaning stopped: " + outputDirectory.getPath());
+	}
 
 	public void webappStopping(String tomcatStopCommand) {
 		log("Stopping tomcat: " + tomcatStopCommand);

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/LogStats.java

@@ -0,0 +1,133 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Extract any summary information from the log file.
+ */
+public class LogStats {
+	private static final Pattern START_TIME_PATTERN = Pattern
+			.compile("(.*) Run started.");
+	private static final Pattern END_TIME_PATTERN = Pattern
+			.compile("(.*) Testing complete.");
+	private static final Pattern SUITE_NAME_PATTERN = Pattern
+			.compile("Running suite (.*)");
+	private static final Pattern ERROR_PATTERN = Pattern
+			.compile("ERROR\\s+(.*)");
+	private static final Pattern WARNING_PATTERN = Pattern
+			.compile("WARN\\s+(.*)");
+	private static final SimpleDateFormat dateParser = new SimpleDateFormat(
+			"yyyy-MM-dd HH:mm:ss.SSS");
+
+	/**
+	 * Factory method.
+	 */
+	public static LogStats parse(File logFile) {
+		return new LogStats(logFile);
+	}
+
+	private long startTime;
+	private long endTime;
+	private final List<String> suiteNames = new ArrayList<String>();
+	private final List<String> errors = new ArrayList<String>();
+	private final List<String> warnings = new ArrayList<String>();
+
+	private LogStats(File logFile) {
+
+		BufferedReader reader = null;
+		String line;
+		try {
+			reader = new BufferedReader(new FileReader(logFile));
+			while (null != (line = reader.readLine())) {
+				Matcher m;
+				m = START_TIME_PATTERN.matcher(line);
+				if (m.matches()) {
+					startTime = parseTime(m.group(1));
+				} else {
+					m = END_TIME_PATTERN.matcher(line);
+					if (m.matches()) {
+						endTime = parseTime(m.group(1));
+					} else {
+						m = SUITE_NAME_PATTERN.matcher(line);
+						if (m.matches()) {
+							suiteNames.add(m.group(1));
+						} else {
+							m = ERROR_PATTERN.matcher(line);
+							if (m.matches()) {
+								errors.add(m.group(1));
+							} else {
+								m = WARNING_PATTERN.matcher(line);
+								if (m.matches()) {
+									warnings.add(m.group(1));
+								}
+							}
+						}
+					}
+				}
+			}
+
+		} catch (IOException e) {
+			// Can't give up - I need to create as much output as I can.
+			e.printStackTrace();
+		} finally {
+			if (reader != null) {
+				try {
+					reader.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
+
+	private long parseTime(String timeString) {
+		try {
+			return dateParser.parse(timeString).getTime();
+		} catch (ParseException e) {
+			e.printStackTrace();
+			return 0L;
+		}
+	}
+
+	public boolean hasErrors() {
+		return !errors.isEmpty();
+	}
+
+	public Collection<String> getErrors() {
+		return Collections.unmodifiableCollection(errors);
+	}
+
+	public boolean hasWarnings() {
+		return !warnings.isEmpty();
+	}
+
+	public Collection<String> getWarnings() {
+		return Collections.unmodifiableCollection(warnings);
+	}
+
+	public long getStartTime() {
+		return startTime;
+	}
+
+	public long getEndTime() {
+		return endTime;
+	}
+
+	public long getElapsedTime() {
+		return Math.abs(endTime - startTime);
+	}
+
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputManager.java

@@ -0,0 +1,83 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Manages the contents of the output area. Removes old files prior to a run.
+ * Creates a unified summary of the test suite outputs.
+ */
+public class OutputManager {
+	private final SeleniumRunnerParameters parms;
+	private final Listener listener;
+
+	public OutputManager(SeleniumRunnerParameters parms) {
+		this.parms = parms;
+		this.listener = parms.getListener();
+	}
+
+	/**
+	 * Delete any output files from previous runs.
+	 */
+	public void cleanOutputDirectory() throws IOException {
+		File outputDirectory = parms.getOutputDirectory();
+		listener.cleanOutputStart(outputDirectory);
+
+		try {
+			for (File file : outputDirectory.listFiles()) {
+				// Skip the log file, since we are already over-writing it.
+				if (file.equals(parms.getLogFile())) {
+					continue;
+				}
+				// Skip any hidden files (like .svn)
+				if (file.getPath().startsWith(".")) {
+					continue;
+				}
+				// Delete all of the others.
+				if (file.isFile()) {
+					FileHelper.deleteFile(file);
+				} else {
+					FileHelper.purgeDirectoryRecursively(file);
+				}
+			}
+		} catch (IOException e) {
+			listener.cleanOutputFailed(outputDirectory, e);
+			throw e;
+		} finally {
+			listener.cleanOutputStop(outputDirectory);
+		}
+	}
+
+	/**
+	 * Parse each of the output files from the test suites, and create a unified
+	 * output file.
+	 */
+	public void summarizeOutput() {
+		LogStats log = LogStats.parse(parms.getLogFile());
+		
+		List<SuiteStats> suites = new ArrayList<SuiteStats>();
+		for (File outputFile : parms.getOutputDirectory().listFiles(
+				new HtmlFileFilter())) {
+			SuiteStats suite = SuiteStats.parse(parms, outputFile);
+			if (suite != null) {
+				suites.add(suite);
+			}
+		}
+		
+		OutputSummaryFormatter formatter = new OutputSummaryFormatter(parms);
+		formatter.format(log, suites);
+	}
+
+	private static class HtmlFileFilter implements FileFilter {
+		public boolean accept(File path) {
+			return path.getName().endsWith(".html")
+					|| path.getName().endsWith(".htm");
+		}
+
+	}
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputSummaryFormatter.java

@@ -0,0 +1,356 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import edu.cornell.mannlib.vitro.utilities.testrunner.SuiteStats.TestInfo;
+
+/**
+ * Creates the summary HTML file.
+ */
+public class OutputSummaryFormatter {
+	public static final String SUMMARY_HTML_FILENAME = "summary.html";
+	public static final String SUMMARY_CSS_FILENAME = "summary.css";
+	private final SimpleDateFormat dateFormat = new SimpleDateFormat(
+			"yyyy-MM-dd HH:mm:ss");
+	private final SeleniumRunnerParameters parms;
+
+	private LogStats log;
+	private List<SuiteStats> suites;
+	private Status runStatus;
+	private List<TestInfo> allTests = new ArrayList<TestInfo>();
+	private int passingTestCount;
+	private List<TestInfo> failingTests = new ArrayList<TestInfo>();
+	private List<TestInfo> ignoredTests = new ArrayList<TestInfo>();
+
+	public OutputSummaryFormatter(SeleniumRunnerParameters parms) {
+		this.parms = parms;
+	}
+
+	/**
+	 * Create a summary HTML file from the info contained in this log file and
+	 * these suite outputs.
+	 */
+	public void format(LogStats log, List<SuiteStats> suites) {
+		this.log = log;
+		this.suites = suites;
+		this.runStatus = figureOverallStatus(log, suites);
+		tallyTests();
+
+		PrintWriter writer = null;
+		try {
+			copyCssFile();
+
+			File outputFile = new File(parms.getOutputDirectory(),
+					SUMMARY_HTML_FILENAME);
+			writer = new PrintWriter(outputFile);
+
+			writeHeader(writer);
+			writeStatsSection(writer);
+			writeErrorMessagesSection(writer);
+			writeFailureSection(writer);
+			writeIgnoreSection(writer);
+			writeSuitesSection(writer);
+			writeAllTestsSection(writer);
+			writeFooter(writer);
+		} catch (IOException e) {
+			// There is no appeal for any problems here. Just report them.
+			e.printStackTrace();
+		} finally {
+			if (writer != null) {
+				writer.close();
+			}
+		}
+	}
+
+	/**
+	 * Copy the CSS file into the output directory.
+	 */
+	private void copyCssFile() {
+		File cssSource = parms.getSummaryCssFile();
+		File cssTarget = new File(parms.getOutputDirectory(),
+				SUMMARY_CSS_FILENAME);
+		try {
+			FileHelper.copy(cssSource, cssTarget);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	/**
+	 * The overall status for the run is the worst status of any component.
+	 */
+	private Status figureOverallStatus(LogStats log, List<SuiteStats> suites) {
+		if (log.hasErrors()) {
+			return Status.ERROR;
+		}
+		boolean hasWarnings = log.hasWarnings();
+
+		for (SuiteStats s : suites) {
+			if (s.getStatus() == Status.ERROR) {
+				return Status.ERROR;
+			} else if (s.getStatus() == Status.WARN) {
+				hasWarnings = true;
+			}
+		}
+
+		if (hasWarnings) {
+			return Status.WARN;
+		} else {
+			return Status.OK;
+		}
+	}
+
+	private void tallyTests() {
+		for (SuiteStats s : suites) {
+			for (TestInfo t : s.getTests()) {
+				this.allTests.add(t);
+				if (t.getStatus() == Status.OK) {
+					this.passingTestCount++;
+				} else if (t.getStatus() == Status.WARN) {
+					this.ignoredTests.add(t);
+				} else {
+					this.failingTests.add(t);
+				}
+			}
+		}
+	}
+
+	private void writeHeader(PrintWriter writer) {
+		String startString = formatDateTime(log.getStartTime());
+
+		writer.println("<html>");
+		writer.println("<head>");
+		writer.println("  <title>Summary of Acceptance Tests " + startString
+				+ "</title>");
+		writer.println("  <link rel=\"stylesheet\" type=\"text/css\" "
+				+ "href=\"summary.css\">");
+		writer.println("</head>");
+		writer.println("<body>");
+		writer.println();
+		writer.println("  <div class=\"heading\">");
+		writer.println("    Acceptance test results: " + startString);
+		writer.println("    <div class=\"" + this.runStatus.getHtmlClass()
+				+ " one-word\">" + this.runStatus + "</div>");
+		writer.println("  </div>");
+	}
+
+	private void writeStatsSection(PrintWriter writer) {
+		String passClass = Status.OK.getHtmlClass();
+		String failClass = this.failingTests.isEmpty() ? "" : Status.ERROR
+				.getHtmlClass();
+		String ignoreClass = this.ignoredTests.isEmpty() ? "" : Status.WARN
+				.getHtmlClass();
+
+		String start = formatDateTime(log.getStartTime());
+		String end = formatDateTime(log.getEndTime());
+		String elapsed = formatElapsedTime(log.getElapsedTime());
+
+		writer.println("  <div class=\"section\">Summary</div>");
+		writer.println();
+		writer.println("  <table class=\"summary\" cellspacing=\"0\">");
+		writer.println("    <tr>");
+		writer.println("      <td>");
+		writer.println("        <table cellspacing=\"0\">");
+		writer.println("  	      <tr><td>Start time:</td><td>" + start
+				+ "</td></tr>");
+		writer.println("  	      <tr><td>End time:</td><td>" + end
+				+ "</td></tr>");
+		writer.println("  	      <tr><td>Elapsed time</td><td>" + elapsed
+				+ "</td></tr>");
+		writer.println("  	    </table>");
+		writer.println("      </td>");
+		writer.println("      <td>");
+		writer.println("        <table cellspacing=\"0\">");
+		writer.println("  	      <tr><td>Suites</td><td>" + this.suites.size()
+				+ "</td></tr>");
+		writer.println("  	      <tr><td>Total tests</td><td>"
+				+ this.allTests.size() + "</td></tr>");
+		writer.println("  	      <tr class=\"" + passClass
+				+ "\"><td>Passing tests</td><td>" + this.passingTestCount
+				+ "</td></tr>");
+		writer.println("  	      <tr class=\"" + failClass
+				+ "\"><td>Failing tests</td><td>" + this.failingTests.size()
+				+ "</td></tr>");
+		writer.println("  	      <tr class=\"" + ignoreClass
+				+ "\"><td>Ignored tests</td><td>" + this.ignoredTests.size()
+				+ "</td></tr>");
+		writer.println("  	    </table>");
+		writer.println("      </td>");
+		writer.println("    </tr>");
+		writer.println("  </table>");
+		writer.println();
+	}
+
+	private void writeErrorMessagesSection(PrintWriter writer) {
+		String errorClass = Status.ERROR.getHtmlClass();
+		String warnClass = Status.WARN.getHtmlClass();
+
+		writer.println("  <div class=section>Errors and warnings</div>");
+		writer.println();
+		writer.println("  <table cellspacing=\"0\">");
+
+		if ((!log.hasErrors()) && (!log.hasWarnings())) {
+			writer
+					.println("      <tr><td colspan=\"2\">No errors or warnings</td></tr>");
+		} else {
+			for (String e : log.getErrors()) {
+				writer.println("      <tr class=\"" + errorClass
+						+ "\"><td>ERROR</td><td>" + e + "</td></tr>");
+			}
+			for (String w : log.getWarnings()) {
+				writer.println("      <tr class=\"" + warnClass
+						+ "\"><td>ERROR</td><td>" + w + "</td></tr>");
+			}
+		}
+		writer.println("    </table>");
+		writer.println();
+	}
+
+	private void writeFailureSection(PrintWriter writer) {
+		String errorClass = Status.ERROR.getHtmlClass();
+
+		writer.println("  <div class=section>Failing tests</div>");
+		writer.println();
+		writer.println("  <table cellspacing=\"0\">");
+		writer.println("    <tr><th>Suite name</th><th>Test name</th></tr>\n");
+		if (failingTests.isEmpty()) {
+			writer.println("    <tr><td colspan=\"2\">No tests failed.</td>"
+					+ "</tr>");
+		} else {
+			for (TestInfo t : failingTests) {
+				writer.println("    <tr class=\"" + errorClass + "\">");
+				writer.println("      <td>" + t.getSuiteName() + "</td>");
+				writer.println("      <td><a href=\"" + t.getOutputLink()
+						+ "\">" + t.getTestName() + "</a></td>");
+				writer.println("    </tr>");
+			}
+		}
+		writer.println("  </table>");
+		writer.println();
+	}
+
+	private void writeIgnoreSection(PrintWriter writer) {
+		String warnClass = Status.WARN.getHtmlClass();
+
+		writer.println("  <div class=section>Ignored tests</div>");
+		writer.println();
+		writer.println("  <table cellspacing=\"0\">");
+		writer.println("    <tr><th>Suite name</th><th>Test name</th>"
+				+ "<th>Reason for ignoring</th></tr>\n");
+		if (ignoredTests.isEmpty()) {
+			writer.println("    <tr><td colspan=\"3\">No tests ignored.</td>"
+					+ "</tr>");
+		} else {
+			for (TestInfo t : ignoredTests) {
+				writer.println("    <tr class=\"" + warnClass + "\">");
+				writer.println("      <td>" + t.getSuiteName() + "</td>");
+				writer.println("      <td><a href=\"" + t.getOutputLink()
+						+ "\">" + t.getTestName() + "</a></td>");
+				writer.println("      <td>" + t.getReasonForIgnoring()
+						+ "</td>");
+				writer.println("    </tr>");
+			}
+		}
+		writer.println("  </table>");
+		writer.println();
+	}
+
+	private void writeSuitesSection(PrintWriter writer) {
+		writer.println("  <div class=section>Suites</div>");
+		writer.println();
+		writer.println("  <table cellspacing=\"0\">");
+
+		for (SuiteStats s : suites) {
+			writer.println("    <tr class=\"" + s.getStatus().getHtmlClass()
+					+ "\">");
+			writer.println("      <td><a href=\"" + s.getOutputLink() + "\">"
+					+ s.getName() + "</a></td>");
+			writer.println("    </tr>");
+		}
+
+		writer.println("  </table>");
+		writer.println();
+	}
+
+	private void writeAllTestsSection(PrintWriter writer) {
+		writer.println("  <div class=section>All tests</div>");
+		writer.println();
+		writer.println("  <table cellspacing=\"0\">");
+
+		writer.println("    <tr><th>Suite name</th><th>Test name</th></tr>\n");
+		for (TestInfo t : allTests) {
+			writer.println("    <tr class=\"" + t.getStatus().getHtmlClass()
+					+ "\">");
+			writer.println("      <td>" + t.getSuiteName() + "</td>");
+			writer.println("      <td><a href=\"" + t.getOutputLink() + "\">"
+					+ t.getTestName() + "</a></td>");
+			writer.println("    </tr>");
+		}
+
+		writer.println("  </table>");
+		writer.println();
+	}
+
+	private void writeFooter(PrintWriter writer) {
+		writer.println("  <div class=section>Log</div>");
+		writer.println("  <pre>");
+
+		Reader reader = null;
+		try {
+			reader = new FileReader(parms.getLogFile());
+			char[] buffer = new char[4096];
+			int howMany;
+			while (-1 != (howMany = reader.read(buffer))) {
+				writer.write(buffer, 0, howMany);
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			if (reader != null) {
+				try {
+					reader.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		writer.println("  </pre>");
+		writer.println("</body>");
+		writer.println("</html>");
+	}
+
+	private String formatElapsedTime(long elapsed) {
+		long elapsedSeconds = elapsed / 1000L;
+		long seconds = elapsedSeconds % 60L;
+		long elapsedMinutes = elapsedSeconds / 60L;
+		long minutes = elapsedMinutes % 60L;
+		long hours = elapsedMinutes / 60L;
+
+		String elapsedTime = "";
+		if (hours > 0) {
+			elapsedTime += hours + "h ";
+		}
+		if (minutes > 0 || hours > 0) {
+			elapsedTime += minutes + "m ";
+		}
+		elapsedTime += seconds + "s";
+
+		return elapsedTime;
+	}
+
+	private String formatDateTime(long dateTime) {
+		return dateFormat.format(new Date(dateTime));
+	}
+
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java

@@ -19,6 +19,7 @@ public class SeleniumRunner {
 	private final UploadAreaCleaner uploadCleaner;
 	private final ModelCleaner modelCleaner;
 	private final SuiteRunner suiteRunner;
+	private final OutputManager outputManager;
 
 	public SeleniumRunner(SeleniumRunnerParameters parms) {
 		this.parms = parms;
@@ -26,34 +27,42 @@ public class SeleniumRunner {
 		this.uploadCleaner = new UploadAreaCleaner(parms);
 		this.modelCleaner = new ModelCleaner(parms);
 		this.suiteRunner = new SuiteRunner(parms);
+		this.outputManager = new OutputManager(parms);
 	}
 
 	public void runSelectedSuites() {
-		listener.runStarted();
-		for (File suiteDir : parms.getSelectedSuites()) {
-			listener.suiteStarted(suiteDir);
-			try {
-				if (parms.isCleanModel()) {
-					modelCleaner.clean();
+		try {
+			listener.runStarted();
+			outputManager.cleanOutputDirectory();
+			for (File suiteDir : parms.getSelectedSuites()) {
+				listener.suiteStarted(suiteDir);
+				try {
+					if (parms.isCleanModel()) {
+						modelCleaner.clean();
+					}
+					if (parms.isCleanUploads()) {
+						uploadCleaner.clean();
+					}
+					suiteRunner.runSuite(suiteDir);
+				} catch (IOException e) {
+					listener.suiteFailed(suiteDir, e);
+				} catch (CommandRunnerException e) {
+					listener.suiteFailed(suiteDir, e);
 				}
-				if (parms.isCleanUploads()) {
-					uploadCleaner.clean();
-				}
-				suiteRunner.runSuite(suiteDir);
-			} catch (IOException e) {
-				listener.suiteFailed(suiteDir, e);
-			} catch (CommandRunnerException e) {
-				listener.suiteFailed(suiteDir, e);
-			} catch (FatalException e) {
-				listener.runFailed(e);
-				e.printStackTrace();
-				break;
+				listener.suiteStopped(suiteDir);
 			}
-			listener.suiteStopped(suiteDir);
+			listener.runEndTime();
+			outputManager.summarizeOutput();
+		} catch (IOException e) {
+			listener.runFailed(e);
+			e.printStackTrace();
+		} catch (FatalException e) {
+			listener.runFailed(e);
+			e.printStackTrace();
 		}
 		listener.runStopped();
 	}
-
+	
 	private static void selectAllSuites(SeleniumRunnerParameters parms) {
 		List<File> suites = new ArrayList<File>();
 		for (File parentDir : parms.getSuiteParentDirectories()) {

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunnerParameters.java

@@ -27,6 +27,8 @@ public class SeleniumRunnerParameters {
 	public static final String PROP_FIREFOX_PROFILE_PATH = "firefox_profile_template_path";
 	public static final String PROP_SUITE_TIMEOUT_LIMIT = "suite_timeout_limit";
 	public static final String PROP_SELENIUM_JAR_PATH = "selenium_jar_path";
+	public static final String PROP_IGNORED_TESTS = "ignored_tests_file";
+	public static final String PROP_SUMMARY_CSS = "summary_css_file";
 
 	public static final String LOGFILE_NAME = "log_file.txt";
 
@@ -40,6 +42,8 @@ public class SeleniumRunnerParameters {
 	private final File logFile;
 	private final Collection<File> suiteParentDirectories;
 	private final ModelCleanerProperties modelCleanerProperties;
+	private final IgnoredTests ignoredTests;
+	private final File summaryCssFile;
 
 	private Collection<File> selectedSuites = Collections.emptySet();
 	private boolean cleanModel = true;
@@ -70,6 +74,8 @@ public class SeleniumRunnerParameters {
 			this.uploadDirectory = checkReadWriteDirectory(props,
 					PROP_UPLOAD_DIRECTORY);
 
+			this.summaryCssFile = checkSummaryCssFile(props);
+
 			this.outputDirectory = checkReadWriteDirectory(props,
 					PROP_OUTPUT_DIRECTORY);
 			this.logFile = new File(this.outputDirectory, LOGFILE_NAME);
@@ -78,6 +84,14 @@ public class SeleniumRunnerParameters {
 			this.suiteParentDirectories = checkSuiteParentDirectories(props);
 
 			this.modelCleanerProperties = new ModelCleanerProperties(props);
+
+			// Get the list of ignored tests.
+			String ignoredFilesPath = getRequiredProperty(props,
+					PROP_IGNORED_TESTS);
+			File ignoredFilesFile = new File(ignoredFilesPath);
+			FileHelper.checkReadableFile(ignoredFilesFile, "File '"
+					+ ignoredFilesPath + "'");
+			this.ignoredTests = new IgnoredTests(ignoredFilesFile);
 		} finally {
 			if (propsReader != null) {
 				try {
@@ -89,6 +103,17 @@ public class SeleniumRunnerParameters {
 		}
 	}
 
+	/**
+	 * The CSS file must be specified, must exist, and must be readable.
+	 */
+	private File checkSummaryCssFile(Properties props) {
+		String summaryCssPath = getRequiredProperty(props, PROP_SUMMARY_CSS);
+		File cssFile = new File(summaryCssPath);
+		FileHelper.checkReadableFile(cssFile, "File '" + summaryCssPath
+				+ "'");
+		return cssFile;
+	}
+
 	/**
 	 * If there is a parameter for this key, it should point to a readable
 	 * directory.
@@ -120,8 +145,8 @@ public class SeleniumRunnerParameters {
 	}
 
 	/**
-	 * Check that there is a property for the output directory, and that it
-	 * points to a valid directory.
+	 * Check that there is a property for the required directory path, and that
+	 * it points to a valid directory.
 	 */
 	private File checkReadWriteDirectory(Properties props, String key) {
 		String value = getRequiredProperty(props, key);
@@ -265,6 +290,14 @@ public class SeleniumRunnerParameters {
 		return outputDirectory;
 	}
 
+	public File getLogFile() {
+		return logFile;
+	}
+
+	public File getSummaryCssFile() {
+		return summaryCssFile;
+	}
+
 	public Collection<File> getSuiteParentDirectories() {
 		return suiteParentDirectories;
 	}
@@ -273,6 +306,10 @@ public class SeleniumRunnerParameters {
 		return modelCleanerProperties;
 	}
 
+	public IgnoredTests getIgnoredTests() {
+		return ignoredTests;
+	}
+
 	public void setSelectedSuites(Collection<File> selectedSuites) {
 		this.selectedSuites = selectedSuites;
 	}
@@ -307,9 +344,9 @@ public class SeleniumRunnerParameters {
 				+ "\n  outputDirectory: " + outputDirectory.getPath()
 				+ "\n  suiteParentDirectories: " + suiteParentDirectories
 				+ "\n  modelCleanerProperties: " + modelCleanerProperties
-				+ "\n\n  selectedSuites: " + showSelectedSuites()
-				+ "\n  cleanModel: " + cleanModel + "\n  cleanUploads: "
-				+ cleanUploads;
+				+ "\n" + ignoredTests + "\n\n  selectedSuites: "
+				+ showSelectedSuites() + "\n  cleanModel: " + cleanModel
+				+ "\n  cleanUploads: " + cleanUploads;
 	}
 
 	private String showSelectedSuites() {

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Status.java

@@ -0,0 +1,33 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+/**
+ * Status for each test, each suite, and the entire run.
+ */
+public enum Status {
+	/** All tests passed, and there were no warnings or errors. */
+	OK("good"),
+
+	/**
+	 * Any test failure was ignored, and any messages were no worse than
+	 * warnings.
+	 */
+	WARN("fair"),
+
+	/**
+	 * A test failed and could not be ignored, or an error message was
+	 * generated.
+	 */
+	ERROR("bad");
+	
+	private final String htmlClass;
+	
+	private Status(String htmlClass) {
+		this.htmlClass = htmlClass;
+	}
+	
+	public String getHtmlClass() {
+		return this.htmlClass;
+	}
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteRunner.java

@@ -37,10 +37,12 @@ public class SuiteRunner {
 		cmd.add("-userExtensions");
 		cmd.add(parms.getUserExtensionsFile().getPath());
 
-		if (parms.hasFirefoxProfileDir()) {
-			cmd.add("-firefoxProfileTemplate");
-			cmd.add(parms.getFirefoxProfileDir().getPath());
-		}
+		// TODO - figure out why the use of a template means running the test
+		// twice in simultaneous tabs.
+		// if (parms.hasFirefoxProfileDir()) {
+		// cmd.add("-firefoxProfileTemplate");
+		// cmd.add(parms.getFirefoxProfileDir().getPath());
+		// }
 
 		String suiteName = suiteDir.getName();
 		File outputFile = new File(parms.getOutputDirectory(), suiteName

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteStats.java

@@ -0,0 +1,186 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.utilities.testrunner;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Extract any summary information from an HTML output file, produced by a test
+ * suite.
+ */
+public class SuiteStats {
+	/**
+	 * If the file doesn't contain a line that includes this pattern, it is not
+	 * a suite output file.
+	 */
+	private static final Pattern TITLE_LINE_PATTERN = Pattern
+			.compile("<title>Test suite results</title>");
+
+	/**
+	 * A test line looks something like this example:
+	 */
+	public static final String EXAMPLE_TEST_LINE = ""
+			+ "<pre><tr class=\"  status_passed\"><td><a href=\"#testresult0\">MyTest</a></td></tr></pre>";
+
+	/**
+	 * So here is the pattern to match it:
+	 */
+	private static final Pattern TEST_LINE_PATTERN = Pattern
+			.compile("<tr class=\"\\s*(\\w+)\"><td><a href=\"(#\\w+)\">([^<]*)</a></td></tr>");
+
+	/**
+	 * Parse the fields from this file and attempt to produce a
+	 * {@link SuiteStats} object. If this is not an appropriate file, just
+	 * return null.
+	 */
+	public static SuiteStats parse(SeleniumRunnerParameters parms,
+			File outputFile) {
+		IgnoredTests ignoredTests = parms.getIgnoredTests();
+
+		boolean isSuiteOutputFile = false;
+		Status status = Status.ERROR;
+
+		List<TestInfo> tests = new ArrayList<TestInfo>();
+		String suiteName = FileHelper.baseName(outputFile);
+		String outputLink = outputFile.getName();
+
+		BufferedReader reader = null;
+		String line;
+		try {
+			reader = new BufferedReader(new FileReader(outputFile));
+			while (null != (line = reader.readLine())) {
+				if (TITLE_LINE_PATTERN.matcher(line).find()) {
+					isSuiteOutputFile = true;
+				}
+
+				Matcher m;
+				m = TEST_LINE_PATTERN.matcher(line);
+				if (m.matches()) {
+					String testName = m.group(3);
+					String testLink = outputLink + m.group(2);
+
+					Status testStatus;
+					String reasonForIgnoring;
+					if ("status_passed".equals(m.group(1))) {
+						testStatus = Status.OK;
+						reasonForIgnoring = "";
+					} else if (ignoredTests.isIgnored(suiteName, testName)) {
+						testStatus = Status.WARN;
+						reasonForIgnoring = ignoredTests.getReasonForIgnoring(
+								suiteName, testName);
+					} else {
+						testStatus = Status.ERROR;
+						reasonForIgnoring = "";
+					}
+
+					tests.add(new TestInfo(testName, suiteName, testLink,
+							testStatus, reasonForIgnoring));
+				}
+			}
+
+			status = Status.OK;
+			for (TestInfo t : tests) {
+				if (t.status == Status.ERROR) {
+					status = Status.ERROR;
+				} else if ((t.status == Status.WARN) && (status == Status.OK)) {
+					status = Status.WARN;
+				}
+			}
+
+			if (isSuiteOutputFile) {
+				return new SuiteStats(suiteName, outputLink, tests, status);
+			} else {
+				return null;
+			}
+
+		} catch (IOException e) {
+			// Can't give up - I need to create as much output as I can.
+			e.printStackTrace();
+			return null;
+		} finally {
+			if (reader != null) {
+				try {
+					reader.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
+
+	private final String suiteName;
+	private final String outputLink;
+	private final List<TestInfo> tests;
+	private final Status status;
+
+	public SuiteStats(String suiteName, String outputLink,
+			List<TestInfo> tests, Status status) {
+		this.suiteName = suiteName;
+		this.outputLink = outputLink;
+		this.tests = tests;
+		this.status = status;
+	}
+
+	public String getName() {
+		return suiteName;
+	}
+
+	public Status getStatus() {
+		return status;
+	}
+
+	public String getOutputLink() {
+		return outputLink;
+	}
+
+	public Collection<TestInfo> getTests() {
+		return Collections.unmodifiableCollection(tests);
+	}
+
+	public static class TestInfo {
+		private final String name;
+		private final String suite;
+		private final String outputLink;
+		private final Status status;
+		private final String reasonForIgnoring;
+
+		public TestInfo(String name, String suite, String outputLink,
+				Status status, String reasonForIgnoring) {
+			this.name = name;
+			this.suite = suite;
+			this.outputLink = outputLink;
+			this.status = status;
+			this.reasonForIgnoring = reasonForIgnoring;
+		}
+
+		public Status getStatus() {
+			return status;
+		}
+
+		public String getSuiteName() {
+			return suite;
+		}
+
+		public String getTestName() {
+			return name;
+		}
+
+		public String getOutputLink() {
+			return outputLink;
+		}
+
+		public String getReasonForIgnoring() {
+			return reasonForIgnoring;
+		}
+	}
+
+}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/UploadAreaCleaner.java

@@ -3,7 +3,6 @@
 package edu.cornell.mannlib.vitro.utilities.testrunner;
 
 import java.io.File;
-import java.io.FileFilter;
 import java.io.IOException;
 
 /**
@@ -34,9 +33,9 @@ public class UploadAreaCleaner {
 		try {
 			for (File file : uploadDirectory.listFiles()) {
 				if (file.isFile()) {
-					deleteFile(file);
+					FileHelper.deleteFile(file);
 				} else {
-					purgeDirectoryRecursively(uploadDirectory);
+					FileHelper.purgeDirectoryRecursively(file);
 				}
 			}
 		} catch (IOException e) {
@@ -47,53 +46,4 @@ public class UploadAreaCleaner {
 		}
 	}
 
-	/**
-	 * Delete all of the files in a directory, any sub-directories, and the
-	 * directory itself.
-	 */
-	protected static void purgeDirectoryRecursively(File directory)
-			throws IOException {
-		File[] files = directory.listFiles();
-		for (File file : files) {
-			if (file.isDirectory()) {
-				purgeDirectoryRecursively(file);
-			} else {
-				deleteFile(file);
-			}
-		}
-		deleteFile(directory);
-	}
-
-	/**
-	 * Delete a file, either before or after the test. If it can't be deleted,
-	 * complain.
-	 */
-	protected static void deleteFile(File file) throws IOException {
-		if (file.exists()) {
-			file.delete();
-		}
-		if (!file.exists()) {
-			return;
-		}
-
-		/*
-		 * If we were unable to delete the file, is it because it's a non-empty
-		 * directory?
-		 */
-		if (!file.isDirectory()) {
-			final StringBuffer message = new StringBuffer(
-					"Can't delete directory '" + file.getPath() + "'\n");
-			file.listFiles(new FileFilter() {
-				public boolean accept(File pathname) {
-					message.append("   contains file '" + pathname + "'\n");
-					return true;
-				}
-			});
-			throw new IOException(message.toString().trim());
-		} else {
-			throw new IOException("Unable to delete file '" + file.getPath()
-					+ "'");
-		}
-	}
-
 }