NIHVIVO-222 Consolidate pre-run and post-run data into SuiteData, backing away from SuiteContents and SuiteResults, except as data-gathering classes. Distinguish between INGORED and WARN as two valid statuses with distinct meanings.

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/ModelCleaner.java

@@ -29,6 +29,13 @@ public class ModelCleaner {
 		this.tomcatController = tomcatController;
 
 		sanityCheck();
+		try {
+			tomcatController.stopTheWebapp();
+			tomcatController.startTheWebapp();
+		} catch (CommandRunnerException e) {
+			throw new FatalException(
+					"sanityCheck: Failed to stop and start Tomcat.", e);
+		}
 	}
 
 	private void sanityCheck() {

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java

@@ -61,7 +61,7 @@ public class SeleniumRunner {
 
 			listener.runEndTime();
 			outputManager.summarizeOutput(dataModel);
-			success = (dataModel.getRunStatus() == Status.OK);
+			success = Status.isSuccess(dataModel.getRunStatus());
 		} catch (IOException e) {
 			listener.runFailed(e);
 			success = false;

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunnerParameters.java

@@ -11,7 +11,6 @@ import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
 
@@ -411,7 +410,6 @@ public class SeleniumRunnerParameters {
 	 * recognize a suite directory because it contains a file named Suite.html.
 	 */
 	public Collection<File> findSuiteDirs(File parentDir) {
-		System.out.println("parentDir: " + parentDir);
 		return Arrays.asList(parentDir.listFiles(new FileFilter() {
 			public boolean accept(File pathname) {
 				if (!pathname.isDirectory()) {

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Status.java

@@ -12,13 +12,13 @@ public enum Status {
 	/**
 	 * One or more tests have not been run yet.
 	 */
-	PENDING(""),
+	PENDING("pending"),
 
 	/**
-	 * Any test failure was ignored, and any messages were no worse than
+	 * Will not run because it is ignored, or has run and failed but the failure
-	 * warnings.
+	 * is ignored.
 	 */
-	WARN("fair"),
+	IGNORED("fair"),
 
 	/**
 	 * A test failed and could not be ignored, or an error message was
@@ -44,4 +44,9 @@ public enum Status {
 			return s2;
 		}
 	}
+
+	/** Anything except ERROR is considered to be a success. */
+	public static boolean isSuccess(Status status) {
+		return status != Status.ERROR;
+	}
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/DataModel.java

@@ -6,6 +6,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -16,10 +17,10 @@ import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests;
 import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo;
 import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats;
 import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
+import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.SuiteData.TestData;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults;
-import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults;
 
 /**
  * Collect all that we know about suites, tests, and their current status.
@@ -38,16 +39,12 @@ public class DataModel {
 	private Status runStatus = Status.PENDING;
 
 	private final SortedMap<String, SuiteData> suiteDataMap = new TreeMap<String, SuiteData>();
-	private final List<SuiteData> pendingSuites = new ArrayList<SuiteData>();
+	private final EnumMap<Status, List<SuiteData>> suiteMapByStatus = new EnumMap<Status, List<SuiteData>>(
-	private final List<SuiteData> passingSuites = new ArrayList<SuiteData>();
+			Status.class);
-	private final List<SuiteData> failingSuites = new ArrayList<SuiteData>();
-	private final List<SuiteData> ignoredSuites = new ArrayList<SuiteData>();
 
-	private final List<TestResults> allTests = new ArrayList<TestResults>();
+	private final List<TestData> allTests = new ArrayList<TestData>();
-	private final List<TestResults> pendingTests = new ArrayList<TestResults>();
+	private final EnumMap<Status, List<TestData>> testMapByStatus = new EnumMap<Status, List<TestData>>(
-	private final List<TestResults> passingTests = new ArrayList<TestResults>();
+			Status.class);
-	private final List<TestResults> failingTests = new ArrayList<TestResults>();
-	private final List<TestResults> ignoredTests = new ArrayList<TestResults>();
 
 	// ----------------------------------------------------------------------
 	// Constructor
@@ -107,20 +104,19 @@ public class DataModel {
 		runStatus = Status.OK;
 
 		suiteDataMap.clear();
-
+		suiteMapByStatus.clear();
-		ignoredSuites.clear();
+		for (Status s : Status.values()) {
-		pendingSuites.clear();
+			suiteMapByStatus.put(s, new ArrayList<SuiteData>());
-		failingSuites.clear();
+		}
-		passingSuites.clear();
 
 		allTests.clear();
-		ignoredTests.clear();
+		testMapByStatus.clear();
-		pendingTests.clear();
+		for (Status s : Status.values()) {
-		failingTests.clear();
+			testMapByStatus.put(s, new ArrayList<TestData>());
-		passingTests.clear();
+		}
 
 		/*
-		 * Suite data.
+		 * Populate the Suite map with all Suites.
 		 */
 		Map<String, SuiteResults> resultsMap = new HashMap<String, SuiteResults>();
 		for (SuiteResults result : suiteResults) {
@@ -142,90 +138,30 @@ public class DataModel {
 		}
 
 		/*
-		 * Tallys of suites and tests.
+		 * Map the Suites by status.
 		 */
-		for (SuiteData sd : suiteDataMap.values()) {
+		for (SuiteData s : suiteDataMap.values()) {
-			switch (sd.getSuiteStatus()) {
+			getSuites(s.getStatus()).add(s);
-			case ERROR:
+		}
-				failingSuites.add(sd);
+
-				break;
+		/**
-			case PENDING:
+		 * Populate the Test map with all Tests, and map by status.
-				pendingSuites.add(sd);
+		 */
-				break;
+		for (SuiteData s : suiteDataMap.values()) {
-			case WARN:
+			for (TestData t : s.getTestMap().values()) {
-				ignoredSuites.add(sd);
+				allTests.add(t);
-				break;
+				getTests(t.getStatus()).add(t);
-			default: // Status.OK
-				passingSuites.add(sd);
-				break;
 			}
 		}
 
-		for (SuiteData sd : suiteDataMap.values()) {
+		if (logStats.hasErrors() || !getSuites(Status.ERROR).isEmpty()) {
-			SuiteResults sResult = sd.getResults();
-			if (sResult != null) {
-				tallyTestResults(sResult);
-			} else if (sd.getContents() != null) {
-				tallyTestContents(sd);
-			}
-		}
-		for (TestResults tResult : allTests) {
-			switch (tResult.getStatus()) {
-			case OK:
-				passingTests.add(tResult);
-				break;
-			case PENDING:
-				pendingTests.add(tResult);
-				break;
-			case WARN:
-				ignoredTests.add(tResult);
-				break;
-			default: // Status.ERROR
-				failingTests.add(tResult);
-				break;
-			}
-		}
-
-		/*
-		 * Overall status. Warnings in the log are scary, but ignored tests are
-		 * OK.
-		 */
-		if (logStats.hasErrors() || !failingSuites.isEmpty()) {
 			runStatus = Status.ERROR;
-		} else {
+		} else if (!getSuites(Status.PENDING).isEmpty()) {
-			if (logStats.hasWarnings()) {
-				runStatus = Status.WARN;
-			} else {
-				if (!pendingSuites.isEmpty()) {
 			runStatus = Status.PENDING;
 		} else {
 			runStatus = Status.OK;
 		}
 	}
-		}
-	}
-
-	/**
-	 * Categorize all test results according to status.
-	 */
-	private void tallyTestResults(SuiteResults sResult) {
-		for (TestResults tResult : sResult.getTests()) {
-			allTests.add(tResult);
-		}
-	}
-
-	/**
-	 * Populate {@link #allTests} with the tests for which we have no results.
-	 */
-	private void tallyTestContents(SuiteData suiteData) {
-		Status suiteStatus = suiteData.getSuiteStatus();
-
-		for (String testName : suiteData.getContents().getTestNames()) {
-			TestResults t = new TestResults(testName, suiteData.getName(), "",
-					suiteStatus, "");
-			allTests.add(t);
-		}
-	}
 
 	// ----------------------------------------------------------------------
 	// Access the derived data.
@@ -248,19 +184,19 @@ public class DataModel {
 	}
 
 	public boolean isAnyPasses() {
-		return !(passingSuites.isEmpty() && passingTests.isEmpty());
+		return !getTests(Status.OK).isEmpty();
 	}
 
 	public boolean isAnyFailures() {
-		return !(failingSuites.isEmpty() && failingTests.isEmpty());
+		return !getTests(Status.ERROR).isEmpty();
 	}
 
 	public boolean isAnyIgnores() {
-		return !(ignoredSuites.isEmpty() && ignoredTests.isEmpty());
+		return !getTests(Status.IGNORED).isEmpty();
 	}
 
 	public boolean isAnyPending() {
-		return !pendingSuites.isEmpty();
+		return !getTests(Status.PENDING).isEmpty();
 	}
 
 	public int getTotalSuiteCount() {
@@ -268,23 +204,33 @@ public class DataModel {
 	}
 
 	public int getPassingSuiteCount() {
-		return passingSuites.size();
+		return getSuites(Status.OK).size();
 	}
 
 	public int getFailingSuiteCount() {
-		return failingSuites.size();
+		return getSuites(Status.ERROR).size();
 	}
 
 	public int getIgnoredSuiteCount() {
-		return ignoredSuites.size();
+		return getSuites(Status.IGNORED).size();
 	}
 
-	public int getPendingSuitesCount() {
+	public int getPendingSuiteCount() {
-		return pendingSuites.size();
+		return getSuites(Status.PENDING).size();
 	}
 
-	public Collection<SuiteResults> getSuiteResults() {
+	public Collection<SuiteData> getAllSuites() {
-		return Collections.unmodifiableCollection(suiteResults);
+		return suiteDataMap.values();
+	}
+
+	public Map<String, SuiteData> getSuitesWithFailureMessages() {
+		Map<String, SuiteData> map = new TreeMap<String, SuiteData>();
+		for (SuiteData s : suiteDataMap.values()) {
+			if (s.getFailureMessages() != null) {
+				map.put(s.getName(), s);
+			}
+		}
+		return map;
 	}
 
 	public int getTotalTestCount() {
@@ -292,57 +238,57 @@ public class DataModel {
 	}
 
 	public int getPassingTestCount() {
-		return passingTests.size();
+		return getTests(Status.OK).size();
 	}
 
 	public int getFailingTestCount() {
-		return failingTests.size();
+		return getTests(Status.ERROR).size();
 	}
 
 	public int getIgnoredTestCount() {
-		return ignoredTests.size();
+		return getTests(Status.IGNORED).size();
 	}
 
-	public int getPendingTestsCount() {
+	public int getPendingTestCount() {
-		return pendingTests.size();
+		return getTests(Status.PENDING).size();
 	}
 
-	public Collection<TestResults> getAllTests() {
+	public Collection<TestData> getAllTests() {
 		return Collections.unmodifiableCollection(allTests);
 	}
 
-	public Collection<TestResults> getFailingTests() {
+	public Collection<TestData> getFailingTests() {
-		return Collections.unmodifiableCollection(failingTests);
+		return Collections.unmodifiableCollection(getTests(Status.ERROR));
 	}
 
-	public Collection<TestResults> getIgnoredTests() {
+	public Collection<TestData> getIgnoredTests() {
-		return Collections.unmodifiableCollection(ignoredTests);
+		return Collections.unmodifiableCollection(getTests(Status.IGNORED));
 	}
 
 	public Collection<IgnoredTestInfo> getIgnoredTestInfo() {
 		return ignoredTestList.getList();
 	}
 
-	public String getOutputLink(String suiteName, String testName) {
-		SuiteData sd = suiteDataMap.get(suiteName);
-		if (sd != null) {
-			SuiteResults s = sd.getResults();
-			if (s != null) {
-				if (testName.equals("*")) {
-					return s.getOutputLink();
-				} else {
-					TestResults t = s.getTest(testName);
-					if (t != null) {
-						return t.getOutputLink();
-					}
-				}
-			}
-		}
-		return "";
-	}
-
 	public String getReasonForIgnoring(String suiteName, String testName) {
 		return ignoredTestList.getReasonForIgnoring(suiteName, testName);
 	}
 
+	// ----------------------------------------------------------------------
+	// Helper methods
+	// ----------------------------------------------------------------------
+
+	/**
+	 * Get the list of suites that have this status.
+	 */
+	private List<SuiteData> getSuites(Status st) {
+		return suiteMapByStatus.get(st);
+	}
+
+	/**
+	 * Get the list of tests that have this status.
+	 */
+	private List<TestData> getTests(Status st) {
+		return testMapByStatus.get(st);
+	}
+
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/SuiteData.java

@@ -2,6 +2,9 @@
 
 package edu.cornell.mannlib.vitro.utilities.testrunner.datamodel;
 
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults;
@@ -11,56 +14,168 @@ import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestRe
  * What do we know about this suite, both before it runs and after it has run?
  */
 public class SuiteData {
+	/**
+	 * If this suite has failure messages, the output link is to an anchor on
+	 * the same page.
+	 */
+	public static String failureMessageAnchor(SuiteData s) {
+		return "suiteFailure_" + s.getName();
+	}
+
 	private final String name;
-	private final boolean ignored;
+	private final Status status;
-	private final SuiteContents contents;
+	private final String outputLink;
-	private final SuiteResults results;
 	private final ProcessOutput failureMessages;
 
+	/**
+	 * This map iterates according to the order that the tests were specified in
+	 * the suite file.
+	 */
+	private final Map<String, TestData> testMap;
+
 	public SuiteData(String name, boolean ignored, SuiteContents contents,
 			SuiteResults results, ProcessOutput failureMessages) {
 		this.name = name;
-		this.ignored = ignored;
-		this.contents = contents;
-		this.results = results;
 		this.failureMessages = failureMessages;
+
+		if (ignored) {
+			this.status = Status.IGNORED;
+			this.outputLink = null;
+			this.testMap = buildTestMap(contents, results);
+		} else if (failureMessages != null) {
+			this.status = Status.ERROR;
+			this.outputLink = "#" + failureMessageAnchor(this);
+			this.testMap = buildTestMap(contents, results);
+		} else if (results != null) {
+			this.testMap = buildTestMap(contents, results);
+			this.status = buildStatusFromTestMap();
+			this.outputLink = results.getOutputLink();
+		} else {
+			this.status = Status.PENDING;
+			this.outputLink = null;
+			this.testMap = buildTestMap(contents, results);
+		}
+	}
+
+	/**
+	 * Build the test map. Do we have test results, or only the advance list of
+	 * tests?
+	 */
+	private Map<String, TestData> buildTestMap(SuiteContents contents,
+			SuiteResults results) {
+		if (results == null) {
+			return buildTestMapFromContents(contents);
+		} else {
+			return buildTestMapFromResults(contents, results);
+		}
+	}
+
+	/**
+	 * All we have to build from is the contents of the Suite HTML file.
+	 */
+	private Map<String, TestData> buildTestMapFromContents(
+			SuiteContents contents) {
+		Map<String, TestData> map = new LinkedHashMap<String, TestData>();
+		for (String testName : contents.getTestNames()) {
+			map.put(testName, new TestData(testName, this.name, this.status,
+					null));
+		}
+		return map;
+	}
+
+	/**
+	 * We can build from both the contents of the Suite HTML file and from the
+	 * test results output file.
+	 */
+	private Map<String, TestData> buildTestMapFromResults(
+			SuiteContents contents, SuiteResults results) {
+		Map<String, TestData> map = new LinkedHashMap<String, TestData>();
+		for (String testName : contents.getTestNames()) {
+			TestResults testResult = results.getTest(testName);
+			if (testResult == null) {
+				// This shouldn't happen. How do we show it?
+				map.put(testName, new TestData(testName, this.name,
+						Status.PENDING, null));
+			} else {
+				map.put(testName,
+						new TestData(testName, this.name, testResult
+								.getStatus(), testResult.getOutputLink()));
+			}
+		}
+		return map;
+	}
+
+	/**
+	 * The suite ran to completion, so its status is the worst of the individual
+	 * test statuses.
+	 */
+	private Status buildStatusFromTestMap() {
+		Status status = Status.OK;
+		for (TestData t : this.testMap.values()) {
+			status = Status.combine(status, t.getStatus());
+		}
+		return status;
 	}
 
 	public String getName() {
 		return name;
 	}
 
-	public boolean isIgnored() {
+	public Status getStatus() {
-		return ignored;
-	}
-
-	public SuiteContents getContents() {
-		return contents;
-	}
-
-	public SuiteResults getResults() {
-		return results;
-	}
-
-	public Status getSuiteStatus() {
-		if (ignored) {
-			return Status.WARN;
-		}
-		if (failureMessages != null) {
-			return Status.ERROR;
-		}
-		if (results == null) {
-			return Status.PENDING;
-		}
-
-		/*
-		 * If we have results and no failure messages, scan the results for the
-		 * worst status.
-		 */
-		Status status = Status.OK;
-		for (TestResults t : results.getTests()) {
-			status = Status.combine(status, t.getStatus());
-		}
 		return status;
 	}
+
+	public String getOutputLink() {
+		return outputLink;
+	}
+
+	public ProcessOutput getFailureMessages() {
+		return failureMessages;
+	}
+
+	public Map<String, TestData> getTestMap() {
+		return testMap;
+	}
+
+	/**
+	 * What do we know about this test, both before it runs and after it has
+	 * run?
+	 */
+	public static class TestData {
+		private final String testName;
+		private final String suiteName;
+		private final Status status;
+		private final String outputLink;
+
+		public TestData(String testName, String suiteName, Status status,
+				String outputLink) {
+			this.testName = testName;
+			this.suiteName = suiteName;
+			this.status = status;
+			this.outputLink = outputLink;
+		}
+
+		public String getTestName() {
+			return testName;
+		}
+
+		public String getSuiteName() {
+			return suiteName;
+		}
+
+		public Status getStatus() {
+			return status;
+		}
+
+		public String getOutputLink() {
+			return outputLink;
+		}
+
+		@Override
+		public String toString() {
+			return "TestData[testName=" + testName + ", suiteName=" + suiteName
+					+ ", status=" + status + ", outputLink=" + outputLink + "]";
+		}
+
+	}
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputDataListener.java

@@ -9,6 +9,8 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper;
 import edu.cornell.mannlib.vitro.utilities.testrunner.listener.Listener;
@@ -138,6 +140,7 @@ public class OutputDataListener implements Listener {
 	 * interesting if it indicates a suite failure.
 	 */
 	public static class ProcessOutput {
+		private static final String SUITE_FAILURE_PATTERN = "exception|error(?i)";
 		private final String suiteName;
 		private final StringBuilder stdout = new StringBuilder();
 		private final StringBuilder errout = new StringBuilder();
@@ -167,7 +170,9 @@ public class OutputDataListener implements Listener {
 		}
 
 		public boolean isSuiteFailure() {
-			return errout.length() > 0;
+			Pattern p = Pattern.compile(SUITE_FAILURE_PATTERN);
+			Matcher m = p.matcher(errout);
+			return m.find();
 		}
 
 	}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputSummaryFormatter.java

@@ -11,6 +11,7 @@ import java.io.Reader;
 import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.Date;
+import java.util.Map;
 
 import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper;
 import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo;
@@ -18,7 +19,9 @@ import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats;
 import edu.cornell.mannlib.vitro.utilities.testrunner.SeleniumRunnerParameters;
 import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
 import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.DataModel;
-import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults;
+import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.SuiteData;
+import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.SuiteData.TestData;
+import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
 
 /**
  * Creates the summary HTML file.
@@ -60,6 +63,7 @@ public class OutputSummaryFormatter {
 			writeIgnoreSection(writer);
 			writeSuitesSection(writer);
 			writeAllTestsSection(writer);
+			writeSuiteErrorMessagesSection(writer);
 			writeFooter(writer);
 		} catch (IOException e) {
 			// There is no appeal for any problems here. Just report them.
@@ -92,207 +96,234 @@ public class OutputSummaryFormatter {
 		}
 	}
 
-	private void writeHeader(PrintWriter writer) {
+	private void writeHeader(PrintWriter w) {
 		Status runStatus = dataModel.getRunStatus();
 		String statusString = (runStatus == Status.PENDING) ? "IN PROGRESS"
 				: runStatus.toString();
 		String startString = formatDateTime(dataModel.getStartTime());
 
-		writer.println("<html>");
+		w.println("<html>");
-		writer.println("<head>");
+		w.println("<head>");
-		writer.println("  <title>Summary of Acceptance Tests " + startString
+		w.println("  <title>Summary of Acceptance Tests " + startString
 				+ "</title>");
-		writer.println("  <link rel=\"stylesheet\" type=\"text/css\" "
+		w.println("  <link rel=\"stylesheet\" type=\"text/css\" "
 				+ "href=\"summary.css\">");
-		writer.println("</head>");
+		w.println("</head>");
-		writer.println("<body>");
+		w.println("<body>");
-		writer.println();
+		w.println();
-		writer.println("  <div class=\"heading\">");
+		w.println("  <div class=\"heading\">");
-		writer.println("    Acceptance test results: " + startString);
+		w.println("    Acceptance test results: " + startString);
-		writer.println("    <div class=\"" + runStatus.getHtmlClass()
+		w.println("    <div class=\"" + runStatus.getHtmlClass()
 				+ " one-word\">" + statusString + "</div>");
-		writer.println("  </div>");
+		w.println("  </div>");
 	}
 
-	private void writeStatsSection(PrintWriter writer) {
+	private void writeStatsSection(PrintWriter w) {
 		String passClass = dataModel.isAnyPasses() ? Status.OK.getHtmlClass()
 				: "";
 		String failClass = dataModel.isAnyFailures() ? Status.ERROR
 				.getHtmlClass() : "";
-		String ignoreClass = dataModel.isAnyIgnores() ? Status.WARN
+		String ignoreClass = dataModel.isAnyIgnores() ? Status.IGNORED
 				.getHtmlClass() : "";
 
 		String start = formatDateTime(dataModel.getStartTime());
 		String end = formatDateTime(dataModel.getEndTime());
 		String elapsed = formatElapsedTime(dataModel.getElapsedTime());
 
-		writer.println("  <div class=\"section\">Summary</div>");
+		w.println("  <div class=\"section\">Summary</div>");
-		writer.println();
+		w.println();
-		writer.println("  <table class=\"summary\" cellspacing=\"0\">");
+		w.println("  <table class=\"summary\" cellspacing=\"0\">");
-		writer.println("    <tr>");
+		w.println("    <tr>");
-		writer.println("      <td>");
+		w.println("      <td>");
-		writer.println("        <table cellspacing=\"0\">");
+		w.println("        <table cellspacing=\"0\">");
-		writer.println("  	      <tr><td>Start time:</td><td>" + start
+		w.println("  	      <tr><td>Start time:</td><td>" + start
 				+ "</td></tr>");
-		writer.println("  	      <tr><td>End time:</td><td>" + end
+		w.println("  	      <tr><td>End time:</td><td>" + end + "</td></tr>");
+		w.println("  	      <tr><td>Elapsed time</td><td>" + elapsed
 				+ "</td></tr>");
-		writer.println("  	      <tr><td>Elapsed time</td><td>" + elapsed
+		w.println("  	    </table>");
-				+ "</td></tr>");
+		w.println("      </td>");
-		writer.println("  	    </table>");
+		w.println("      <td>");
-		writer.println("      </td>");
+		w.println("        <table class=\"tallys\" cellspacing=\"0\">");
-		writer.println("      <td>");
+		w.println("          <tr><th>&nbsp;</th><th>Suites</th><th>Tests</th>");
-		writer.println("        <table class=\"tallys\" cellspacing=\"0\">");
+		w.println("          <tr class=\"" + passClass
-		writer.println("          <tr><th>&nbsp;</th><th>Suites</th><th>Tests</th>");
-		writer.println("          <tr class=\"" + passClass
 				+ "\"><td>Passed</td><td>" + dataModel.getPassingSuiteCount()
 				+ "</td><td>" + dataModel.getPassingTestCount() + "</td>");
-		writer.println("          <tr class=\"" + failClass
+		w.println("          <tr class=\"" + failClass
 				+ "\"><td>Failed</td><td>" + dataModel.getFailingSuiteCount()
 				+ "</td><td>" + dataModel.getFailingTestCount() + "</td>");
-		writer.println("          <tr class=\"" + ignoreClass
+		w.println("          <tr class=\"" + ignoreClass
 				+ "\"><td>Ignored</td><td>" + dataModel.getIgnoredSuiteCount()
 				+ "</td><td>" + dataModel.getIgnoredTestCount() + "</td>");
 		if (dataModel.isAnyPending()) {
-			writer.println("          <tr><td>Pending</td><td>"
+			w.println("          <tr class=\"" + Status.PENDING.getHtmlClass()
-					+ dataModel.getPendingSuitesCount() + "</td><td>"
+					+ "\"><td>Pending</td><td>"
-					+ dataModel.getPendingTestsCount() + "</td>");
+					+ dataModel.getPendingSuiteCount() + "</td><td>"
+					+ dataModel.getPendingTestCount() + "</td>");
 		}
-		writer.println("          <tr><td class=\"total\">Total</td><td>"
+		w.println("          <tr><td class=\"total\">Total</td><td>"
 				+ dataModel.getTotalSuiteCount() + "</td><td>"
 				+ dataModel.getTotalTestCount() + "</td>");
-		writer.println("  	    </table>");
+		w.println("  	    </table>");
-		writer.println("      </td>");
+		w.println("      </td>");
-		writer.println("    </tr>");
+		w.println("    </tr>");
-		writer.println("  </table>");
+		w.println("  </table>");
-		writer.println();
+		w.println();
 	}
 
-	private void writeErrorMessagesSection(PrintWriter writer) {
+	private void writeErrorMessagesSection(PrintWriter w) {
 		String errorClass = Status.ERROR.getHtmlClass();
-		String warnClass = Status.WARN.getHtmlClass();
 
-		writer.println("  <div class=section>Errors and warnings</div>");
+		w.println("  <div class=section>Errors and warnings</div>");
-		writer.println();
+		w.println();
-		writer.println("  <table cellspacing=\"0\">");
+		w.println("  <table cellspacing=\"0\">");
 
 		if ((!logStats.hasErrors()) && (!logStats.hasWarnings())) {
-			writer.println("      <tr><td colspan=\"2\">No errors or warnings</td></tr>");
+			w.println("      <tr><td colspan=\"2\">No errors or warnings</td></tr>");
 		} else {
 			for (String e : logStats.getErrors()) {
-				writer.println("      <tr class=\"" + errorClass
+				w.println("      <tr class=\"" + errorClass
 						+ "\"><td>ERROR</td><td>" + e + "</td></tr>");
 			}
-			for (String w : logStats.getWarnings()) {
-				writer.println("      <tr class=\"" + warnClass
-						+ "\"><td>ERROR</td><td>" + w + "</td></tr>");
 		}
-		}
+		w.println("    </table>");
-		writer.println("    </table>");
+		w.println();
-		writer.println();
 	}
 
-	private void writeFailureSection(PrintWriter writer) {
+	private void writeFailureSection(PrintWriter w) {
 		String errorClass = Status.ERROR.getHtmlClass();
-		Collection<TestResults> failingTests = dataModel.getFailingTests();
+		Collection<TestData> failingTests = dataModel.getFailingTests();
 
-		writer.println("  <div class=section>Failing tests</div>");
+		w.println("  <div class=section>Failures</div>");
-		writer.println();
+		w.println();
-		writer.println("  <table cellspacing=\"0\">");
+		w.println("  <table cellspacing=\"0\">");
-		writer.println("    <tr><th>Suite name</th><th>Test name</th></tr>\n");
+		w.println("    <tr><th>Suite name</th><th>Test name</th></tr>\n");
 		if (failingTests.isEmpty()) {
-			writer.println("    <tr><td colspan=\"2\">No tests failed.</td>"
+			w.println("    <tr><td colspan=\"2\">No tests failed.</td>"
 					+ "</tr>");
 		} else {
-			for (TestResults t : failingTests) {
+			Map<String, SuiteData> failedSuiteMap = dataModel
-				writer.println("    <tr class=\"" + errorClass + "\">");
+					.getSuitesWithFailureMessages();
-				writer.println("      <td>" + t.getSuiteName() + "</td>");
+			for (SuiteData s : failedSuiteMap.values()) {
-				writer.println("      <td><a href=\"" + t.getOutputLink()
+				w.println("    <tr class=\"" + errorClass + "\">");
-						+ "\">" + t.getTestName() + "</a></td>");
+				w.println("      <td>" + s.getName() + "</td>");
-				writer.println("    </tr>");
+				w.println("      <td>" + outputLink(s) + "</td>");
+				w.println("    </tr>");
+			}
+			for (TestData t : failingTests) {
+				if (!failedSuiteMap.containsKey(t.getSuiteName())) {
+					w.println("    <tr class=\"" + errorClass + "\">");
+					w.println("      <td>" + t.getSuiteName() + "</td>");
+					w.println("      <td>" + outputLink(t) + "</td>");
+					w.println("    </tr>");
 				}
 			}
-		writer.println("  </table>");
+		}
-		writer.println();
+		w.println("  </table>");
+		w.println();
 	}
 
-	private void writeIgnoreSection(PrintWriter writer) {
+	private void writeIgnoreSection(PrintWriter w) {
-		String warnClass = Status.WARN.getHtmlClass();
+		String warnClass = Status.IGNORED.getHtmlClass();
 		Collection<IgnoredTestInfo> ignoredTests = dataModel
 				.getIgnoredTestInfo();
 
-		writer.println("  <div class=section>Ignored tests</div>");
+		w.println("  <div class=section>Ignored</div>");
-		writer.println();
+		w.println();
-		writer.println("  <table cellspacing=\"0\">");
+		w.println("  <table cellspacing=\"0\">");
-		writer.println("    <tr><th>Suite name</th><th>Test name</th>"
+		w.println("    <tr><th>Suite name</th><th>Test name</th>"
 				+ "<th>Reason for ignoring</th></tr>\n");
 		if (ignoredTests.isEmpty()) {
-			writer.println("    <tr><td colspan=\"3\">No tests ignored.</td>"
+			w.println("    <tr><td colspan=\"3\">No tests ignored.</td>"
 					+ "</tr>");
 		} else {
 			for (IgnoredTestInfo info : ignoredTests) {
 				String suiteName = info.suiteName;
 				String testName = info.testName;
-				String link = dataModel.getOutputLink(suiteName, testName);
 				String reason = dataModel.getReasonForIgnoring(suiteName,
 						testName);
 
-				writer.println("    <tr class=\"" + warnClass + "\">");
+				w.println("    <tr class=\"" + warnClass + "\">");
-				writer.println("      <td>" + suiteName + "</td>");
+				w.println("      <td>" + suiteName + "</td>");
-				if (link.isEmpty()) {
+				w.println("      <td>" + testName + "</td>");
-					writer.println("      <td>" + testName + "</td>");
+				w.println("      <td>" + reason + "</td>");
-				} else {
+				w.println("    </tr>");
-					writer.println("      <td><a href=\"" + link + "\">"
-							+ testName + "</a></td>");
-				}
-				writer.println("      <td>" + reason + "</td>");
-				writer.println("    </tr>");
 			}
 		}
-		writer.println("  </table>");
+		w.println("  </table>");
-		writer.println();
+		w.println();
 	}
 
-	private void writeSuitesSection(PrintWriter writer) {
+	private void writeSuitesSection(PrintWriter w) {
-		writer.println("  <div class=section>Suites</div>");
+		w.println("  <div class=section>Suites Summary</div>");
-		writer.println();
+		w.println();
-		writer.println("  <table cellspacing=\"0\">");
+		w.println("  <table cellspacing=\"0\">");
 
-		for (SuiteResults s : dataModel.getSuiteResults()) {
+		for (SuiteData s : dataModel.getAllSuites()) {
-			writer.println("    <tr class=\"" + s.getStatus().getHtmlClass()
+			w.println("    <tr class=\"" + s.getStatus().getHtmlClass() + "\">");
+			w.println("      <td>" + outputLink(s) + "</td>");
+			w.println("      <td>" + s.getStatus() + "</td>");
+			w.println("    </tr>");
+		}
+
+		w.println("  </table>");
+		w.println();
+	}
+
+	private void writeAllTestsSection(PrintWriter w) {
+		Collection<TestData> allTests = dataModel.getAllTests();
+
+		w.println("  <div class=section>All tests</div>");
+		w.println();
+		w.println("  <table cellspacing=\"0\">");
+
+		w.println("    <tr><th>Suite name</th><th>Test name</th><th>&nbsp;</th></tr>\n");
+		for (TestData t : allTests) {
+			w.println("    <tr class=\"" + t.getStatus().getHtmlClass() + "\">");
+			w.println("      <td>" + t.getSuiteName() + "</td>");
+			w.println("      <td>" + outputLink(t) + "</td>");
+			w.println("      <td>" + t.getStatus() + "</td>");
+			w.println("    </tr>");
+		}
+
+		w.println("  </table>");
+		w.println();
+	}
+
+	private void writeSuiteErrorMessagesSection(PrintWriter w) {
+		Map<String, SuiteData> failedSuiteMap = dataModel
+				.getSuitesWithFailureMessages();
+		if (failedSuiteMap.isEmpty()) {
+			return;
+		}
+
+		w.println("  <div class=section>All tests</div>");
+		w.println();
+		for (SuiteData s : failedSuiteMap.values()) {
+			ProcessOutput output = s.getFailureMessages();
+
+			w.println("  <a name=\"" + SuiteData.failureMessageAnchor(s)
 					+ "\">");
-			writer.println("      <td><a href=\"" + s.getOutputLink() + "\">"
+			w.println("  <table cellspacing=\"0\">");
-					+ s.getName() + "</a></td>");
+			w.println("    <tr><th>Standard Output</th></tr>\n");
-			writer.println("    </tr>");
+			w.println("    <tr><td><pre>" + output.getStdout()
+					+ "</pre></td></tr>\n");
+			w.println("  </table>");
+			w.println("<br/>&nbsp;<br/>");
+
+			w.println("  <table cellspacing=\"0\">");
+			w.println("    <tr><th>Error Output</th></tr>\n");
+			w.println("    <tr><td><pre>" + output.getErrout()
+					+ "</pre></td></tr>\n");
+			w.println("  </table>");
+			w.println("<br/>&nbsp;<br/>");
+			w.println();
+		}
 	}
 
-		writer.println("  </table>");
+	private void writeFooter(PrintWriter w) {
-		writer.println();
+		w.println("  <div class=section>Log</div>");
-	}
+		w.println("  <pre>");
-
-	private void writeAllTestsSection(PrintWriter writer) {
-		Collection<TestResults> allTests = dataModel.getAllTests();
-
-		writer.println("  <div class=section>All tests</div>");
-		writer.println();
-		writer.println("  <table cellspacing=\"0\">");
-
-		writer.println("    <tr><th>Suite name</th><th>Test name</th></tr>\n");
-		for (TestResults t : allTests) {
-			writer.println("    <tr class=\"" + t.getStatus().getHtmlClass()
-					+ "\">");
-			writer.println("      <td>" + t.getSuiteName() + "</td>");
-			writer.println("      <td><a href=\"" + t.getOutputLink() + "\">"
-					+ t.getTestName() + "</a></td>");
-			writer.println("    </tr>");
-		}
-
-		writer.println("  </table>");
-		writer.println();
-	}
-
-	private void writeFooter(PrintWriter writer) {
-		writer.println("  <div class=section>Log</div>");
-		writer.println("  <pre>");
 
 		Reader reader = null;
 		try {
@@ -300,7 +331,7 @@ public class OutputSummaryFormatter {
 			char[] buffer = new char[4096];
 			int howMany;
 			while (-1 != (howMany = reader.read(buffer))) {
-				writer.write(buffer, 0, howMany);
+				w.write(buffer, 0, howMany);
 			}
 		} catch (IOException e) {
 			e.printStackTrace();
@@ -314,9 +345,9 @@ public class OutputSummaryFormatter {
 			}
 		}
 
-		writer.println("  </pre>");
+		w.println("  </pre>");
-		writer.println("</body>");
+		w.println("</body>");
-		writer.println("</html>");
+		w.println("</html>");
 	}
 
 	private String formatElapsedTime(long elapsed) {
@@ -350,4 +381,21 @@ public class OutputSummaryFormatter {
 		return dateFormat.format(new Date(dateTime));
 	}
 
+	private String outputLink(SuiteData s) {
+		if (s.getOutputLink() == null) {
+			return s.getName();
+		} else {
+			return "<a href=\"" + s.getOutputLink() + "\">" + s.getName()
+					+ "</a>";
+		}
+	}
+
+	private String outputLink(TestData t) {
+		if (t.getOutputLink() == null) {
+			return t.getTestName();
+		} else {
+			return "<a href=\"" + t.getOutputLink() + "\">" + t.getTestName()
+					+ "</a>";
+		}
+	}
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/SuiteResults.java

@@ -76,21 +76,16 @@ public class SuiteResults {
 					String testLink = outputLink + m.group(2);
 
 					Status testStatus;
-					String reasonForIgnoring;
 					if ("status_passed".equals(m.group(1))) {
 						testStatus = Status.OK;
-						reasonForIgnoring = "";
 					} else if (ignoredTests.isIgnored(suiteName, testName)) {
-						testStatus = Status.WARN;
+						testStatus = Status.IGNORED;
-						reasonForIgnoring = ignoredTests.getReasonForIgnoring(
-								suiteName, testName);
 					} else {
 						testStatus = Status.ERROR;
-						reasonForIgnoring = "";
 					}
 
 					tests.add(new TestResults(testName, suiteName, testLink,
-							testStatus, reasonForIgnoring));
+							testStatus));
 				}
 			}
 
@@ -163,15 +158,13 @@ public class SuiteResults {
 		private final String suite;
 		private final String outputLink;
 		private final Status status;
-		private final String reasonForIgnoring;
 
 		public TestResults(String name, String suite, String outputLink,
-				Status status, String reasonForIgnoring) {
+				Status status) {
 			this.name = name;
 			this.suite = suite;
 			this.outputLink = outputLink;
 			this.status = status;
-			this.reasonForIgnoring = reasonForIgnoring;
 		}
 
 		public Status getStatus() {
@@ -190,9 +183,6 @@ public class SuiteResults {
 			return outputLink;
 		}
 
-		public String getReasonForIgnoring() {
-			return reasonForIgnoring;
-		}
 	}
 
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/summary.css

@@ -67,6 +67,10 @@ table.tallys td.total {
 	background: rgb(100%, 100%, 60%);
 }
 
+.pending {
+	background: rgb(90%, 90%, 100%);
+}
+
 .one-word {
 	width: 20%;
 	text-align: center;