diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java index afb9d2cc8..aa3989bb8 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java @@ -17,9 +17,19 @@ import java.util.regex.Pattern; * is logged with a warning, not an error. */ public class IgnoredTests { - private final File file; + public static final IgnoredTests EMPTY_LIST = new IgnoredTests(); + + private final String filePath; private final List tests; + /** + * Create an empty instance. + */ + private IgnoredTests() { + this.filePath = "NO FILE"; + this.tests = Collections.emptyList(); + } + /** *

* Parse the file of ignored tests. @@ -38,7 +48,8 @@ public class IgnoredTests { *

*/ public IgnoredTests(File file) { - this.file = file; + this.filePath = file.getAbsolutePath(); + List tests = new ArrayList(); BufferedReader reader = null; @@ -81,9 +92,9 @@ public class IgnoredTests { } /** - * Package access -- only used in unit tests. + * Get a copy of the whole list. */ - List getList() { + public List getList() { return new ArrayList(tests); } @@ -138,7 +149,7 @@ public class IgnoredTests { } public String toString() { - String s = " ignored tests from " + file.getPath() + "\n"; + String s = " ignored tests from " + this.filePath + "\n"; for (IgnoredTestInfo test : tests) { s += " " + test.suiteName + ", " + test.testName + "\n"; } @@ -146,12 +157,13 @@ public class IgnoredTests { } /** - * Package access so it can be used in unit tests. + * Encapsulates a line from the file with suite name, test name, and + * comment. */ - static class IgnoredTestInfo { - final String suiteName; - final String testName; - final String comment; + public static class IgnoredTestInfo { + public final String suiteName; + public final String testName; + public final String comment; public IgnoredTestInfo(String suiteName, String testName, String comment) { this.suiteName = suiteName.trim(); diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java index 045f8dbbd..3596460fb 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java @@ -36,8 +36,9 @@ public class SeleniumRunner { this.modelCleaner = new ModelCleaner(parms, this.tomcatController); this.suiteRunner = new SuiteRunner(parms); this.outputManager = new OutputManager(parms); - this.dataModel = new DataModel(); + this.dataModel = new DataModel(); + this.dataModel.setIgnoredTestList(parms.getIgnoredTests()); } /** diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/DataModel.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/DataModel.java index de9eac4ab..b8dbb6e36 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/DataModel.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/DataModel.java @@ -9,10 +9,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests; +import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo; import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats; import edu.cornell.mannlib.vitro.utilities.testrunner.Status; import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener; +import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput; import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults; import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults; @@ -26,12 +31,13 @@ public class DataModel { private Collection selectedSuites = Collections.emptyList(); private Collection suiteResults = Collections.emptyList(); private OutputDataListener.Info dataListenerInfo = OutputDataListener.Info.EMPTY_INFO; + private IgnoredTests ignoredTestList = IgnoredTests.EMPTY_LIST; private LogStats logStats = LogStats.EMPTY_LOG_STATS; // TODO /* derived data */ private Status runStatus = Status.PENDING; - private final List allSuiteData = new ArrayList(); + private final SortedMap suiteDataMap = new TreeMap(); private final List pendingSuites = new ArrayList(); private final List passingSuites = new ArrayList(); private final List failingSuites = new ArrayList(); @@ -79,6 +85,11 @@ public class DataModel { calculate(); } + public void setIgnoredTestList(IgnoredTests ignoredTestList) { + this.ignoredTestList = ignoredTestList; + calculate(); + } + public void setLogStats(LogStats logStats) { // TODO this.logStats = logStats; calculate(); @@ -95,7 +106,8 @@ public class DataModel { // Clear all derived data. runStatus = Status.OK; - allSuiteData.clear(); + suiteDataMap.clear(); + ignoredSuites.clear(); pendingSuites.clear(); failingSuites.clear(); @@ -119,34 +131,58 @@ public class DataModel { contentsMap.put(contents.getName(), contents); } - for (String name : dataListenerInfo.getSuiteNames()) { - SuiteContents contents = contentsMap.get(name); + for (SuiteContents contents : suiteContents) { + String name = contents.getName(); SuiteResults result = resultsMap.get(name); - boolean ignored = dataListenerInfo.getIgnoredSuiteNames().contains( - name); - allSuiteData.add(new SuiteData(name, ignored, contents, result)); + boolean ignored = ignoredTestList.isIgnored(name); + ProcessOutput failureMessages = dataListenerInfo + .getFailureMessages().get(name); + suiteDataMap.put(name, new SuiteData(name, ignored, contents, + result, failureMessages)); } /* * Tallys of suites and tests. */ - for (SuiteData sd : allSuiteData) { - SuiteContents contents = sd.getContents(); - SuiteResults result = sd.getResults(); - if (result != null) { - tallyTestResults(result); - } else if (contents != null) { + for (SuiteData sd : suiteDataMap.values()) { + switch (sd.getSuiteStatus()) { + case ERROR: + failingSuites.add(sd); + break; + case PENDING: + pendingSuites.add(sd); + break; + case WARN: + ignoredSuites.add(sd); + break; + default: // Status.OK + passingSuites.add(sd); + break; + } + } + + for (SuiteData sd : suiteDataMap.values()) { + SuiteResults sResult = sd.getResults(); + if (sResult != null) { + tallyTestResults(sResult); + } else if (sd.getContents() != null) { tallyTestContents(sd); } - - if (sd.isIgnored()) { - ignoredSuites.add(sd); - } else if (result == null) { - pendingSuites.add(sd); - } else if (result.getStatus() == Status.ERROR) { - failingSuites.add(sd); - } else { - passingSuites.add(sd); + } + for (TestResults tResult : allTests) { + switch (tResult.getStatus()) { + case OK: + passingTests.add(tResult); + break; + case PENDING: + pendingTests.add(tResult); + break; + case WARN: + ignoredTests.add(tResult); + break; + default: // Status.ERROR + failingTests.add(tResult); + break; } } @@ -175,41 +211,19 @@ public class DataModel { private void tallyTestResults(SuiteResults sResult) { for (TestResults tResult : sResult.getTests()) { allTests.add(tResult); - switch (tResult.getStatus()) { - case OK: - passingTests.add(tResult); - break; - case PENDING: - pendingTests.add(tResult); - break; - case WARN: - ignoredTests.add(tResult); - break; - default: // Status.ERROR - failingTests.add(tResult); - break; - } } } /** - * Categorize all tests for which we have no results. + * Populate {@link #allTests} with the tests for which we have no results. */ private void tallyTestContents(SuiteData suiteData) { - SuiteContents contents = suiteData.getContents(); + Status suiteStatus = suiteData.getSuiteStatus(); - for (String testName : contents.getTestNames()) { - if (suiteData.isIgnored()) { - TestResults t = new TestResults(testName, suiteData.getName(), - "", Status.WARN, ""); - allTests.add(t); - ignoredTests.add(t); - } else { - TestResults t = new TestResults(testName, suiteData.getName(), - "", Status.PENDING, ""); - allTests.add(t); - pendingTests.add(t); - } + for (String testName : suiteData.getContents().getTestNames()) { + TestResults t = new TestResults(testName, suiteData.getName(), "", + suiteStatus, ""); + allTests.add(t); } } @@ -250,7 +264,7 @@ public class DataModel { } public int getTotalSuiteCount() { - return allSuiteData.size(); + return suiteDataMap.size(); } public int getPassingSuiteCount() { @@ -305,4 +319,30 @@ public class DataModel { return Collections.unmodifiableCollection(ignoredTests); } + public Collection getIgnoredTestInfo() { + return ignoredTestList.getList(); + } + + public String getOutputLink(String suiteName, String testName) { + SuiteData sd = suiteDataMap.get(suiteName); + if (sd != null) { + SuiteResults s = sd.getResults(); + if (s != null) { + if (testName.equals("*")) { + return s.getOutputLink(); + } else { + TestResults t = s.getTest(testName); + if (t != null) { + return t.getOutputLink(); + } + } + } + } + return ""; + } + + public String getReasonForIgnoring(String suiteName, String testName) { + return ignoredTestList.getReasonForIgnoring(suiteName, testName); + } + } diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/SuiteData.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/SuiteData.java index cf21d8d42..6d4d1af52 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/SuiteData.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/datamodel/SuiteData.java @@ -2,22 +2,28 @@ package edu.cornell.mannlib.vitro.utilities.testrunner.datamodel; +import edu.cornell.mannlib.vitro.utilities.testrunner.Status; +import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput; import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults; +import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults; /** - * TODO + * What do we know about this suite, both before it runs and after it has run? */ public class SuiteData { private final String name; private final boolean ignored; private final SuiteContents contents; private final SuiteResults results; + private final ProcessOutput failureMessages; - public SuiteData(String name, boolean ignored, SuiteContents contents, SuiteResults results) { + public SuiteData(String name, boolean ignored, SuiteContents contents, + SuiteResults results, ProcessOutput failureMessages) { this.name = name; this.ignored = ignored; this.contents = contents; this.results = results; + this.failureMessages = failureMessages; } public String getName() { @@ -36,4 +42,25 @@ public class SuiteData { return results; } + public Status getSuiteStatus() { + if (ignored) { + return Status.WARN; + } + if (failureMessages != null) { + return Status.ERROR; + } + if (results == null) { + return Status.PENDING; + } + + /* + * If we have results and no failure messages, scan the results for the + * worst status. + */ + Status status = Status.OK; + for (TestResults t : results.getTests()) { + status = Status.combine(status, t.getStatus()); + } + return status; + } } diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputDataListener.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputDataListener.java index 92088b56a..e877301eb 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputDataListener.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputDataListener.java @@ -4,10 +4,11 @@ package edu.cornell.mannlib.vitro.utilities.testrunner.output; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper; import edu.cornell.mannlib.vitro.utilities.testrunner.listener.Listener; @@ -16,8 +17,9 @@ public class OutputDataListener implements Listener { private boolean runCompleted; private long startTime; private long endTime; - private final List suiteNames = new ArrayList(); - private final List ignoredSuiteNames = new ArrayList(); + + private ProcessOutput currentSuiteOutput = new ProcessOutput(""); + private Map failureMessages = new HashMap(); // ---------------------------------------------------------------------- // Listener methods that affect the data model @@ -40,14 +42,27 @@ public class OutputDataListener implements Listener { } @Override - public void suiteIgnored(File suite) { - suiteNames.add(FileHelper.baseName(suite)); - ignoredSuiteNames.add(FileHelper.baseName(suite)); + public void suiteStarted(File suiteDir) { + currentSuiteOutput = new ProcessOutput(FileHelper.baseName(suiteDir)); } @Override - public void suiteAdded(File suite) { - suiteNames.add(FileHelper.baseName(suite)); + public void suiteStopped(File suiteDir) { + if (currentSuiteOutput.isSuiteFailure()) { + failureMessages.put(currentSuiteOutput.suiteName, + currentSuiteOutput); + } + currentSuiteOutput = new ProcessOutput(""); + } + + @Override + public void subProcessStdout(String string) { + currentSuiteOutput.addStdout(string); + } + + @Override + public void subProcessErrout(String string) { + currentSuiteOutput.addErrout(string); } // ---------------------------------------------------------------------- @@ -63,26 +78,22 @@ public class OutputDataListener implements Listener { private final boolean runCompleted; private final long startTime; private final long endTime; - private final List suiteNames; - private final List ignoredSuiteNames; + private final Map failureMessages; Info() { this.runCompleted = false; this.startTime = 0; this.endTime = 0; - this.suiteNames = Collections.emptyList(); - this.ignoredSuiteNames = Collections.emptyList(); + this.failureMessages = Collections.emptyMap(); } Info(OutputDataListener parent) { this.runCompleted = parent.runCompleted; this.startTime = parent.startTime; this.endTime = parent.endTime; - this.suiteNames = Collections - .unmodifiableList(new ArrayList(parent.suiteNames)); - this.ignoredSuiteNames = Collections - .unmodifiableList(new ArrayList( - parent.ignoredSuiteNames)); + this.failureMessages = Collections + .unmodifiableMap(new HashMap( + parent.failureMessages)); } public boolean isRunCompleted() { @@ -105,13 +116,10 @@ public class OutputDataListener implements Listener { } } - public List getSuiteNames() { - return suiteNames; + public Map getFailureMessages() { + return failureMessages; } - public List getIgnoredSuiteNames() { - return ignoredSuiteNames; - } } /** @@ -121,10 +129,61 @@ public class OutputDataListener implements Listener { return new Info(this); } + // ---------------------------------------------------------------------- + // A class that summarized the sub-process output from a test suite. + // ---------------------------------------------------------------------- + + /** + * The output from a subprocess that runs a test suite. It's only + * interesting if it indicates a suite failure. + */ + public static class ProcessOutput { + private final String suiteName; + private final StringBuilder stdout = new StringBuilder(); + private final StringBuilder errout = new StringBuilder(); + + public ProcessOutput(String suiteName) { + this.suiteName = suiteName; + } + + public void addStdout(String string) { + stdout.append(string); + } + + public void addErrout(String string) { + errout.append(string); + } + + public String getSuiteName() { + return suiteName; + } + + public String getStdout() { + return stdout.toString(); + } + + public String getErrout() { + return errout.toString(); + } + + public boolean isSuiteFailure() { + return errout.length() > 0; + } + + } + // ---------------------------------------------------------------------- // Listener methods that don't affect the data model // ---------------------------------------------------------------------- + @Override + public void suiteAdded(File suite) { + } + + @Override + public void suiteIgnored(File suite) { + } + @Override public void runFailed(Exception e) { } @@ -205,22 +264,10 @@ public class OutputDataListener implements Listener { public void subProcessStartInBackground(List command) { } - @Override - public void subProcessStdout(String string) { - } - - @Override - public void subProcessErrout(String string) { - } - @Override public void subProcessStop(List command) { } - @Override - public void suiteStarted(File suiteDir) { - } - @Override public void suiteTestingStarted(File suiteDir) { } @@ -237,10 +284,6 @@ public class OutputDataListener implements Listener { public void suiteTestingStopped(File suiteDir) { } - @Override - public void suiteStopped(File suiteDir) { - } - @Override public void cleanUploadStart(File uploadDirectory) { } diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputSummaryFormatter.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputSummaryFormatter.java index d34429462..0a9f51e6b 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputSummaryFormatter.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/OutputSummaryFormatter.java @@ -13,6 +13,7 @@ import java.util.Collection; import java.util.Date; import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper; +import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo; import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats; import edu.cornell.mannlib.vitro.utilities.testrunner.SeleniumRunnerParameters; import edu.cornell.mannlib.vitro.utilities.testrunner.Status; @@ -93,6 +94,8 @@ public class OutputSummaryFormatter { private void writeHeader(PrintWriter writer) { Status runStatus = dataModel.getRunStatus(); + String statusString = (runStatus == Status.PENDING) ? "IN PROGRESS" + : runStatus.toString(); String startString = formatDateTime(dataModel.getStartTime()); writer.println(""); @@ -107,7 +110,7 @@ public class OutputSummaryFormatter { writer.println("
"); writer.println(" Acceptance test results: " + startString); writer.println("
" + runStatus + "
"); + + " one-word\">" + statusString + "
"); writer.println(" "); } @@ -214,7 +217,8 @@ public class OutputSummaryFormatter { private void writeIgnoreSection(PrintWriter writer) { String warnClass = Status.WARN.getHtmlClass(); - Collection ignoredTests = dataModel.getIgnoredTests(); + Collection ignoredTests = dataModel + .getIgnoredTestInfo(); writer.println("
Ignored tests
"); writer.println(); @@ -225,13 +229,22 @@ public class OutputSummaryFormatter { writer.println(" No tests ignored." + ""); } else { - for (TestResults t : ignoredTests) { + for (IgnoredTestInfo info : ignoredTests) { + String suiteName = info.suiteName; + String testName = info.testName; + String link = dataModel.getOutputLink(suiteName, testName); + String reason = dataModel.getReasonForIgnoring(suiteName, + testName); + writer.println(" "); - writer.println(" " + t.getSuiteName() + ""); - writer.println(" " + t.getTestName() + ""); - writer.println(" " + t.getReasonForIgnoring() - + ""); + writer.println(" " + suiteName + ""); + if (link.isEmpty()) { + writer.println(" " + testName + ""); + } else { + writer.println(" " + + testName + ""); + } + writer.println(" " + reason + ""); writer.println(" "); } } diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/SuiteResults.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/SuiteResults.java index 4ff1ea5a1..b519df934 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/SuiteResults.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/output/SuiteResults.java @@ -9,7 +9,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -120,15 +122,20 @@ public class SuiteResults { private final String suiteName; private final String outputLink; - private final List tests; + private final Map testMap; private final Status status; public SuiteResults(String suiteName, String outputLink, List tests, Status status) { this.suiteName = suiteName; this.outputLink = outputLink; - this.tests = tests; this.status = status; + + Map map = new HashMap(); + for (TestResults t : tests) { + map.put(t.getTestName(), t); + } + testMap = Collections.unmodifiableMap(map); } public String getName() { @@ -144,7 +151,11 @@ public class SuiteResults { } public Collection getTests() { - return Collections.unmodifiableCollection(tests); + return Collections.unmodifiableCollection(testMap.values()); + } + + public TestResults getTest(String testName) { + return testMap.get(testName); } public static class TestResults {