NIHVIVO-222 Capture the output of the test suite sub-process, to detect whether the suite has failed.

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java

@@ -17,9 +17,19 @@ import java.util.regex.Pattern;
  * is logged with a warning, not an error.
  */
 public class IgnoredTests {
-	private final File file;
+	public static final IgnoredTests EMPTY_LIST = new IgnoredTests();
+
+	private final String filePath;
 	private final List<IgnoredTestInfo> tests;
 
+	/**
+	 * Create an empty instance.
+	 */
+	private IgnoredTests() {
+		this.filePath = "NO FILE";
+		this.tests = Collections.emptyList();
+	}
+
 	/**
 	 * <p>
 	 * Parse the file of ignored tests.
@@ -38,7 +48,8 @@ public class IgnoredTests {
 	 * </p>
 	 */
 	public IgnoredTests(File file) {
-		this.file = file;
+		this.filePath = file.getAbsolutePath();
+
 		List<IgnoredTestInfo> tests = new ArrayList<IgnoredTestInfo>();
 
 		BufferedReader reader = null;
@@ -81,9 +92,9 @@ public class IgnoredTests {
 	}
 
 	/**
-	 * Package access -- only used in unit tests.
+	 * Get a copy of the whole list.
 	 */
-	List<IgnoredTestInfo> getList() {
+	public List<IgnoredTestInfo> getList() {
 		return new ArrayList<IgnoredTestInfo>(tests);
 	}
 
@@ -138,7 +149,7 @@ public class IgnoredTests {
 	}
 
 	public String toString() {
-		String s = "  ignored tests from " + file.getPath() + "\n";
+		String s = "  ignored tests from " + this.filePath + "\n";
 		for (IgnoredTestInfo test : tests) {
 			s += "      " + test.suiteName + ", " + test.testName + "\n";
 		}
@@ -146,12 +157,13 @@ public class IgnoredTests {
 	}
 
 	/**
-	 * Package access so it can be used in unit tests.
+	 * Encapsulates a line from the file with suite name, test name, and
+	 * comment.
 	 */
-	static class IgnoredTestInfo {
-		final String suiteName;
-		final String testName;
-		final String comment;
+	public static class IgnoredTestInfo {
+		public final String suiteName;
+		public final String testName;
+		public final String comment;
 
 		public IgnoredTestInfo(String suiteName, String testName, String comment) {
 			this.suiteName = suiteName.trim();

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java

@@ -36,8 +36,9 @@ public class SeleniumRunner {
 		this.modelCleaner = new ModelCleaner(parms, this.tomcatController);
 		this.suiteRunner = new SuiteRunner(parms);
 		this.outputManager = new OutputManager(parms);
-		this.dataModel = new DataModel();
 
+		this.dataModel = new DataModel();
+		this.dataModel.setIgnoredTestList(parms.getIgnoredTests());
 	}
 
 	/**

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/DataModel.java

@@ -9,10 +9,15 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
+import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests;
+import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo;
 import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats;
 import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener;
+import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults;
 
@@ -26,12 +31,13 @@ public class DataModel {
 	private Collection<File> selectedSuites = Collections.emptyList();
 	private Collection<SuiteResults> suiteResults = Collections.emptyList();
 	private OutputDataListener.Info dataListenerInfo = OutputDataListener.Info.EMPTY_INFO;
+	private IgnoredTests ignoredTestList = IgnoredTests.EMPTY_LIST;
 	private LogStats logStats = LogStats.EMPTY_LOG_STATS; // TODO
 
 	/* derived data */
 	private Status runStatus = Status.PENDING;
 
-	private final List<SuiteData> allSuiteData = new ArrayList<SuiteData>();
+	private final SortedMap<String, SuiteData> suiteDataMap = new TreeMap<String, SuiteData>();
 	private final List<SuiteData> pendingSuites = new ArrayList<SuiteData>();
 	private final List<SuiteData> passingSuites = new ArrayList<SuiteData>();
 	private final List<SuiteData> failingSuites = new ArrayList<SuiteData>();
@@ -79,6 +85,11 @@ public class DataModel {
 		calculate();
 	}
 
+	public void setIgnoredTestList(IgnoredTests ignoredTestList) {
+		this.ignoredTestList = ignoredTestList;
+		calculate();
+	}
+
 	public void setLogStats(LogStats logStats) { // TODO
 		this.logStats = logStats;
 		calculate();
@@ -95,7 +106,8 @@ public class DataModel {
 		// Clear all derived data.
 		runStatus = Status.OK;
 
-		allSuiteData.clear();
+		suiteDataMap.clear();
+
 		ignoredSuites.clear();
 		pendingSuites.clear();
 		failingSuites.clear();
@@ -119,34 +131,58 @@ public class DataModel {
 			contentsMap.put(contents.getName(), contents);
 		}
 
-		for (String name : dataListenerInfo.getSuiteNames()) {
-			SuiteContents contents = contentsMap.get(name);
+		for (SuiteContents contents : suiteContents) {
+			String name = contents.getName();
 			SuiteResults result = resultsMap.get(name);
-			boolean ignored = dataListenerInfo.getIgnoredSuiteNames().contains(
-					name);
-			allSuiteData.add(new SuiteData(name, ignored, contents, result));
+			boolean ignored = ignoredTestList.isIgnored(name);
+			ProcessOutput failureMessages = dataListenerInfo
+					.getFailureMessages().get(name);
+			suiteDataMap.put(name, new SuiteData(name, ignored, contents,
+					result, failureMessages));
 		}
 
 		/*
 		 * Tallys of suites and tests.
 		 */
-		for (SuiteData sd : allSuiteData) {
-			SuiteContents contents = sd.getContents();
-			SuiteResults result = sd.getResults();
-			if (result != null) {
-				tallyTestResults(result);
-			} else if (contents != null) {
+		for (SuiteData sd : suiteDataMap.values()) {
+			switch (sd.getSuiteStatus()) {
+			case ERROR:
+				failingSuites.add(sd);
+				break;
+			case PENDING:
+				pendingSuites.add(sd);
+				break;
+			case WARN:
+				ignoredSuites.add(sd);
+				break;
+			default: // Status.OK
+				passingSuites.add(sd);
+				break;
+			}
+		}
+
+		for (SuiteData sd : suiteDataMap.values()) {
+			SuiteResults sResult = sd.getResults();
+			if (sResult != null) {
+				tallyTestResults(sResult);
+			} else if (sd.getContents() != null) {
 				tallyTestContents(sd);
 			}
-
-			if (sd.isIgnored()) {
-				ignoredSuites.add(sd);
-			} else if (result == null) {
-				pendingSuites.add(sd);
-			} else if (result.getStatus() == Status.ERROR) {
-				failingSuites.add(sd);
-			} else {
-				passingSuites.add(sd);
+		}
+		for (TestResults tResult : allTests) {
+			switch (tResult.getStatus()) {
+			case OK:
+				passingTests.add(tResult);
+				break;
+			case PENDING:
+				pendingTests.add(tResult);
+				break;
+			case WARN:
+				ignoredTests.add(tResult);
+				break;
+			default: // Status.ERROR
+				failingTests.add(tResult);
+				break;
 			}
 		}
 
@@ -175,41 +211,19 @@ public class DataModel {
 	private void tallyTestResults(SuiteResults sResult) {
 		for (TestResults tResult : sResult.getTests()) {
 			allTests.add(tResult);
-			switch (tResult.getStatus()) {
-			case OK:
-				passingTests.add(tResult);
-				break;
-			case PENDING:
-				pendingTests.add(tResult);
-				break;
-			case WARN:
-				ignoredTests.add(tResult);
-				break;
-			default: // Status.ERROR
-				failingTests.add(tResult);
-				break;
-			}
 		}
 	}
 
 	/**
-	 * Categorize all tests for which we have no results.
+	 * Populate {@link #allTests} with the tests for which we have no results.
 	 */
 	private void tallyTestContents(SuiteData suiteData) {
-		SuiteContents contents = suiteData.getContents();
+		Status suiteStatus = suiteData.getSuiteStatus();
 
-		for (String testName : contents.getTestNames()) {
-			if (suiteData.isIgnored()) {
-				TestResults t = new TestResults(testName, suiteData.getName(),
-						"", Status.WARN, "");
-				allTests.add(t);
-				ignoredTests.add(t);
-			} else {
-				TestResults t = new TestResults(testName, suiteData.getName(),
-						"", Status.PENDING, "");
-				allTests.add(t);
-				pendingTests.add(t);
-			}
+		for (String testName : suiteData.getContents().getTestNames()) {
+			TestResults t = new TestResults(testName, suiteData.getName(), "",
+					suiteStatus, "");
+			allTests.add(t);
 		}
 	}
 
@@ -250,7 +264,7 @@ public class DataModel {
 	}
 
 	public int getTotalSuiteCount() {
-		return allSuiteData.size();
+		return suiteDataMap.size();
 	}
 
 	public int getPassingSuiteCount() {
@@ -305,4 +319,30 @@ public class DataModel {
 		return Collections.unmodifiableCollection(ignoredTests);
 	}
 
+	public Collection<IgnoredTestInfo> getIgnoredTestInfo() {
+		return ignoredTestList.getList();
+	}
+
+	public String getOutputLink(String suiteName, String testName) {
+		SuiteData sd = suiteDataMap.get(suiteName);
+		if (sd != null) {
+			SuiteResults s = sd.getResults();
+			if (s != null) {
+				if (testName.equals("*")) {
+					return s.getOutputLink();
+				} else {
+					TestResults t = s.getTest(testName);
+					if (t != null) {
+						return t.getOutputLink();
+					}
+				}
+			}
+		}
+		return "";
+	}
+
+	public String getReasonForIgnoring(String suiteName, String testName) {
+		return ignoredTestList.getReasonForIgnoring(suiteName, testName);
+	}
+
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/SuiteData.java

@@ -2,22 +2,28 @@
 
 package edu.cornell.mannlib.vitro.utilities.testrunner.datamodel;
 
+import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
+import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
 import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults;
+import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults;
 
 /**
- * TODO
+ * What do we know about this suite, both before it runs and after it has run?
  */
 public class SuiteData {
 	private final String name;
 	private final boolean ignored;
 	private final SuiteContents contents;
 	private final SuiteResults results;
+	private final ProcessOutput failureMessages;
 
-	public SuiteData(String name, boolean ignored, SuiteContents contents, SuiteResults results) {
+	public SuiteData(String name, boolean ignored, SuiteContents contents,
+			SuiteResults results, ProcessOutput failureMessages) {
 		this.name = name;
 		this.ignored = ignored;
 		this.contents = contents;
 		this.results = results;
+		this.failureMessages = failureMessages;
 	}
 
 	public String getName() {
@@ -36,4 +42,25 @@ public class SuiteData {
 		return results;
 	}
 
+	public Status getSuiteStatus() {
+		if (ignored) {
+			return Status.WARN;
+		}
+		if (failureMessages != null) {
+			return Status.ERROR;
+		}
+		if (results == null) {
+			return Status.PENDING;
+		}
+
+		/*
+		 * If we have results and no failure messages, scan the results for the
+		 * worst status.
+		 */
+		Status status = Status.OK;
+		for (TestResults t : results.getTests()) {
+			status = Status.combine(status, t.getStatus());
+		}
+		return status;
+	}
 }

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputDataListener.java

@@ -4,10 +4,11 @@ package edu.cornell.mannlib.vitro.utilities.testrunner.output;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper;
 import edu.cornell.mannlib.vitro.utilities.testrunner.listener.Listener;
@@ -16,8 +17,9 @@ public class OutputDataListener implements Listener {
 	private boolean runCompleted;
 	private long startTime;
 	private long endTime;
-	private final List<String> suiteNames = new ArrayList<String>();
-	private final List<String> ignoredSuiteNames = new ArrayList<String>();
+
+	private ProcessOutput currentSuiteOutput = new ProcessOutput("");
+	private Map<String, ProcessOutput> failureMessages = new HashMap<String, ProcessOutput>();
 
 	// ----------------------------------------------------------------------
 	// Listener methods that affect the data model
@@ -40,14 +42,27 @@ public class OutputDataListener implements Listener {
 	}
 
 	@Override
-	public void suiteIgnored(File suite) {
-		suiteNames.add(FileHelper.baseName(suite));
-		ignoredSuiteNames.add(FileHelper.baseName(suite));
+	public void suiteStarted(File suiteDir) {
+		currentSuiteOutput = new ProcessOutput(FileHelper.baseName(suiteDir));
 	}
 
 	@Override
-	public void suiteAdded(File suite) {
-		suiteNames.add(FileHelper.baseName(suite));
+	public void suiteStopped(File suiteDir) {
+		if (currentSuiteOutput.isSuiteFailure()) {
+			failureMessages.put(currentSuiteOutput.suiteName,
+					currentSuiteOutput);
+		}
+		currentSuiteOutput = new ProcessOutput("");
+	}
+
+	@Override
+	public void subProcessStdout(String string) {
+		currentSuiteOutput.addStdout(string);
+	}
+
+	@Override
+	public void subProcessErrout(String string) {
+		currentSuiteOutput.addErrout(string);
 	}
 
 	// ----------------------------------------------------------------------
@@ -63,26 +78,22 @@ public class OutputDataListener implements Listener {
 		private final boolean runCompleted;
 		private final long startTime;
 		private final long endTime;
-		private final List<String> suiteNames;
-		private final List<String> ignoredSuiteNames;
+		private final Map<String, ProcessOutput> failureMessages;
 
 		Info() {
 			this.runCompleted = false;
 			this.startTime = 0;
 			this.endTime = 0;
-			this.suiteNames = Collections.emptyList();
-			this.ignoredSuiteNames = Collections.emptyList();
+			this.failureMessages = Collections.emptyMap();
 		}
 
 		Info(OutputDataListener parent) {
 			this.runCompleted = parent.runCompleted;
 			this.startTime = parent.startTime;
 			this.endTime = parent.endTime;
-			this.suiteNames = Collections
-					.unmodifiableList(new ArrayList<String>(parent.suiteNames));
-			this.ignoredSuiteNames = Collections
-					.unmodifiableList(new ArrayList<String>(
-							parent.ignoredSuiteNames));
+			this.failureMessages = Collections
+					.unmodifiableMap(new HashMap<String, ProcessOutput>(
+							parent.failureMessages));
 		}
 
 		public boolean isRunCompleted() {
@@ -105,13 +116,10 @@ public class OutputDataListener implements Listener {
 			}
 		}
 
-		public List<String> getSuiteNames() {
-			return suiteNames;
+		public Map<String, ProcessOutput> getFailureMessages() {
+			return failureMessages;
 		}
 
-		public List<String> getIgnoredSuiteNames() {
-			return ignoredSuiteNames;
-		}
 	}
 
 	/**
@@ -121,10 +129,61 @@ public class OutputDataListener implements Listener {
 		return new Info(this);
 	}
 
+	// ----------------------------------------------------------------------
+	// A class that summarized the sub-process output from a test suite.
+	// ----------------------------------------------------------------------
+
+	/**
+	 * The output from a subprocess that runs a test suite. It's only
+	 * interesting if it indicates a suite failure.
+	 */
+	public static class ProcessOutput {
+		private final String suiteName;
+		private final StringBuilder stdout = new StringBuilder();
+		private final StringBuilder errout = new StringBuilder();
+
+		public ProcessOutput(String suiteName) {
+			this.suiteName = suiteName;
+		}
+
+		public void addStdout(String string) {
+			stdout.append(string);
+		}
+
+		public void addErrout(String string) {
+			errout.append(string);
+		}
+
+		public String getSuiteName() {
+			return suiteName;
+		}
+
+		public String getStdout() {
+			return stdout.toString();
+		}
+
+		public String getErrout() {
+			return errout.toString();
+		}
+
+		public boolean isSuiteFailure() {
+			return errout.length() > 0;
+		}
+
+	}
+
 	// ----------------------------------------------------------------------
 	// Listener methods that don't affect the data model
 	// ----------------------------------------------------------------------
 
+	@Override
+	public void suiteAdded(File suite) {
+	}
+
+	@Override
+	public void suiteIgnored(File suite) {
+	}
+
 	@Override
 	public void runFailed(Exception e) {
 	}
@@ -205,22 +264,10 @@ public class OutputDataListener implements Listener {
 	public void subProcessStartInBackground(List<String> command) {
 	}
 
-	@Override
-	public void subProcessStdout(String string) {
-	}
-
-	@Override
-	public void subProcessErrout(String string) {
-	}
-
 	@Override
 	public void subProcessStop(List<String> command) {
 	}
 
-	@Override
-	public void suiteStarted(File suiteDir) {
-	}
-
 	@Override
 	public void suiteTestingStarted(File suiteDir) {
 	}
@@ -237,10 +284,6 @@ public class OutputDataListener implements Listener {
 	public void suiteTestingStopped(File suiteDir) {
 	}
 
-	@Override
-	public void suiteStopped(File suiteDir) {
-	}
-
 	@Override
 	public void cleanUploadStart(File uploadDirectory) {
 	}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputSummaryFormatter.java

@@ -13,6 +13,7 @@ import java.util.Collection;
 import java.util.Date;
 
 import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper;
+import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo;
 import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats;
 import edu.cornell.mannlib.vitro.utilities.testrunner.SeleniumRunnerParameters;
 import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
@@ -93,6 +94,8 @@ public class OutputSummaryFormatter {
 
 	private void writeHeader(PrintWriter writer) {
 		Status runStatus = dataModel.getRunStatus();
+		String statusString = (runStatus == Status.PENDING) ? "IN PROGRESS"
+				: runStatus.toString();
 		String startString = formatDateTime(dataModel.getStartTime());
 
 		writer.println("<html>");
@@ -107,7 +110,7 @@ public class OutputSummaryFormatter {
 		writer.println("  <div class=\"heading\">");
 		writer.println("    Acceptance test results: " + startString);
 		writer.println("    <div class=\"" + runStatus.getHtmlClass()
-				+ " one-word\">" + runStatus + "</div>");
+				+ " one-word\">" + statusString + "</div>");
 		writer.println("  </div>");
 	}
 
@@ -214,7 +217,8 @@ public class OutputSummaryFormatter {
 
 	private void writeIgnoreSection(PrintWriter writer) {
 		String warnClass = Status.WARN.getHtmlClass();
-		Collection<TestResults> ignoredTests = dataModel.getIgnoredTests();
+		Collection<IgnoredTestInfo> ignoredTests = dataModel
+				.getIgnoredTestInfo();
 
 		writer.println("  <div class=section>Ignored tests</div>");
 		writer.println();
@@ -225,13 +229,22 @@ public class OutputSummaryFormatter {
 			writer.println("    <tr><td colspan=\"3\">No tests ignored.</td>"
 					+ "</tr>");
 		} else {
-			for (TestResults t : ignoredTests) {
+			for (IgnoredTestInfo info : ignoredTests) {
+				String suiteName = info.suiteName;
+				String testName = info.testName;
+				String link = dataModel.getOutputLink(suiteName, testName);
+				String reason = dataModel.getReasonForIgnoring(suiteName,
+						testName);
+
 				writer.println("    <tr class=\"" + warnClass + "\">");
-				writer.println("      <td>" + t.getSuiteName() + "</td>");
-				writer.println("      <td><a href=\"" + t.getOutputLink()
-						+ "\">" + t.getTestName() + "</a></td>");
-				writer.println("      <td>" + t.getReasonForIgnoring()
-						+ "</td>");
+				writer.println("      <td>" + suiteName + "</td>");
+				if (link.isEmpty()) {
+					writer.println("      <td>" + testName + "</td>");
+				} else {
+					writer.println("      <td><a href=\"" + link + "\">"
+							+ testName + "</a></td>");
+				}
+				writer.println("      <td>" + reason + "</td>");
 				writer.println("    </tr>");
 			}
 		}

utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/SuiteResults.java

@@ -9,7 +9,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -120,15 +122,20 @@ public class SuiteResults {
 
 	private final String suiteName;
 	private final String outputLink;
-	private final List<TestResults> tests;
+	private final Map<String, TestResults> testMap;
 	private final Status status;
 
 	public SuiteResults(String suiteName, String outputLink,
 			List<TestResults> tests, Status status) {
 		this.suiteName = suiteName;
 		this.outputLink = outputLink;
-		this.tests = tests;
 		this.status = status;
+
+		Map<String, TestResults> map = new HashMap<String, TestResults>();
+		for (TestResults t : tests) {
+			map.put(t.getTestName(), t);
+		}
+		testMap = Collections.unmodifiableMap(map);
 	}
 
 	public String getName() {
@@ -144,7 +151,11 @@ public class SuiteResults {
 	}
 
 	public Collection<TestResults> getTests() {
-		return Collections.unmodifiableCollection(tests);
+		return Collections.unmodifiableCollection(testMap.values());
+	}
+
+	public TestResults getTest(String testName) {
+		return testMap.get(testName);
 	}
 
 	public static class TestResults {