NIHVIVO-222 Consolidate pre-run and post-run data into SuiteData, backing away from SuiteContents and SuiteResults, except as data-gathering classes. Distinguish between INGORED and WARN as two valid statuses with distinct meanings.
This commit is contained in:
parent
ff1e62c0a9
commit
7a2c5691cf
10 changed files with 435 additions and 317 deletions
|
@ -29,6 +29,13 @@ public class ModelCleaner {
|
|||
this.tomcatController = tomcatController;
|
||||
|
||||
sanityCheck();
|
||||
try {
|
||||
tomcatController.stopTheWebapp();
|
||||
tomcatController.startTheWebapp();
|
||||
} catch (CommandRunnerException e) {
|
||||
throw new FatalException(
|
||||
"sanityCheck: Failed to stop and start Tomcat.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void sanityCheck() {
|
||||
|
|
|
@ -61,7 +61,7 @@ public class SeleniumRunner {
|
|||
|
||||
listener.runEndTime();
|
||||
outputManager.summarizeOutput(dataModel);
|
||||
success = (dataModel.getRunStatus() == Status.OK);
|
||||
success = Status.isSuccess(dataModel.getRunStatus());
|
||||
} catch (IOException e) {
|
||||
listener.runFailed(e);
|
||||
success = false;
|
||||
|
|
|
@ -11,7 +11,6 @@ import java.io.Reader;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
|
@ -411,7 +410,6 @@ public class SeleniumRunnerParameters {
|
|||
* recognize a suite directory because it contains a file named Suite.html.
|
||||
*/
|
||||
public Collection<File> findSuiteDirs(File parentDir) {
|
||||
System.out.println("parentDir: " + parentDir);
|
||||
return Arrays.asList(parentDir.listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
if (!pathname.isDirectory()) {
|
||||
|
|
|
@ -12,13 +12,13 @@ public enum Status {
|
|||
/**
|
||||
* One or more tests have not been run yet.
|
||||
*/
|
||||
PENDING(""),
|
||||
PENDING("pending"),
|
||||
|
||||
/**
|
||||
* Any test failure was ignored, and any messages were no worse than
|
||||
* warnings.
|
||||
* Will not run because it is ignored, or has run and failed but the failure
|
||||
* is ignored.
|
||||
*/
|
||||
WARN("fair"),
|
||||
IGNORED("fair"),
|
||||
|
||||
/**
|
||||
* A test failed and could not be ignored, or an error message was
|
||||
|
@ -44,4 +44,9 @@ public enum Status {
|
|||
return s2;
|
||||
}
|
||||
}
|
||||
|
||||
/** Anything except ERROR is considered to be a success. */
|
||||
public static boolean isSuccess(Status status) {
|
||||
return status != Status.ERROR;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import java.io.File;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -16,10 +17,10 @@ import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests;
|
|||
import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.SuiteData.TestData;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults;
|
||||
|
||||
/**
|
||||
* Collect all that we know about suites, tests, and their current status.
|
||||
|
@ -38,16 +39,12 @@ public class DataModel {
|
|||
private Status runStatus = Status.PENDING;
|
||||
|
||||
private final SortedMap<String, SuiteData> suiteDataMap = new TreeMap<String, SuiteData>();
|
||||
private final List<SuiteData> pendingSuites = new ArrayList<SuiteData>();
|
||||
private final List<SuiteData> passingSuites = new ArrayList<SuiteData>();
|
||||
private final List<SuiteData> failingSuites = new ArrayList<SuiteData>();
|
||||
private final List<SuiteData> ignoredSuites = new ArrayList<SuiteData>();
|
||||
private final EnumMap<Status, List<SuiteData>> suiteMapByStatus = new EnumMap<Status, List<SuiteData>>(
|
||||
Status.class);
|
||||
|
||||
private final List<TestResults> allTests = new ArrayList<TestResults>();
|
||||
private final List<TestResults> pendingTests = new ArrayList<TestResults>();
|
||||
private final List<TestResults> passingTests = new ArrayList<TestResults>();
|
||||
private final List<TestResults> failingTests = new ArrayList<TestResults>();
|
||||
private final List<TestResults> ignoredTests = new ArrayList<TestResults>();
|
||||
private final List<TestData> allTests = new ArrayList<TestData>();
|
||||
private final EnumMap<Status, List<TestData>> testMapByStatus = new EnumMap<Status, List<TestData>>(
|
||||
Status.class);
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Constructor
|
||||
|
@ -107,20 +104,19 @@ public class DataModel {
|
|||
runStatus = Status.OK;
|
||||
|
||||
suiteDataMap.clear();
|
||||
|
||||
ignoredSuites.clear();
|
||||
pendingSuites.clear();
|
||||
failingSuites.clear();
|
||||
passingSuites.clear();
|
||||
suiteMapByStatus.clear();
|
||||
for (Status s : Status.values()) {
|
||||
suiteMapByStatus.put(s, new ArrayList<SuiteData>());
|
||||
}
|
||||
|
||||
allTests.clear();
|
||||
ignoredTests.clear();
|
||||
pendingTests.clear();
|
||||
failingTests.clear();
|
||||
passingTests.clear();
|
||||
testMapByStatus.clear();
|
||||
for (Status s : Status.values()) {
|
||||
testMapByStatus.put(s, new ArrayList<TestData>());
|
||||
}
|
||||
|
||||
/*
|
||||
* Suite data.
|
||||
* Populate the Suite map with all Suites.
|
||||
*/
|
||||
Map<String, SuiteResults> resultsMap = new HashMap<String, SuiteResults>();
|
||||
for (SuiteResults result : suiteResults) {
|
||||
|
@ -142,90 +138,30 @@ public class DataModel {
|
|||
}
|
||||
|
||||
/*
|
||||
* Tallys of suites and tests.
|
||||
* Map the Suites by status.
|
||||
*/
|
||||
for (SuiteData sd : suiteDataMap.values()) {
|
||||
switch (sd.getSuiteStatus()) {
|
||||
case ERROR:
|
||||
failingSuites.add(sd);
|
||||
break;
|
||||
case PENDING:
|
||||
pendingSuites.add(sd);
|
||||
break;
|
||||
case WARN:
|
||||
ignoredSuites.add(sd);
|
||||
break;
|
||||
default: // Status.OK
|
||||
passingSuites.add(sd);
|
||||
break;
|
||||
for (SuiteData s : suiteDataMap.values()) {
|
||||
getSuites(s.getStatus()).add(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* Populate the Test map with all Tests, and map by status.
|
||||
*/
|
||||
for (SuiteData s : suiteDataMap.values()) {
|
||||
for (TestData t : s.getTestMap().values()) {
|
||||
allTests.add(t);
|
||||
getTests(t.getStatus()).add(t);
|
||||
}
|
||||
}
|
||||
|
||||
for (SuiteData sd : suiteDataMap.values()) {
|
||||
SuiteResults sResult = sd.getResults();
|
||||
if (sResult != null) {
|
||||
tallyTestResults(sResult);
|
||||
} else if (sd.getContents() != null) {
|
||||
tallyTestContents(sd);
|
||||
}
|
||||
}
|
||||
for (TestResults tResult : allTests) {
|
||||
switch (tResult.getStatus()) {
|
||||
case OK:
|
||||
passingTests.add(tResult);
|
||||
break;
|
||||
case PENDING:
|
||||
pendingTests.add(tResult);
|
||||
break;
|
||||
case WARN:
|
||||
ignoredTests.add(tResult);
|
||||
break;
|
||||
default: // Status.ERROR
|
||||
failingTests.add(tResult);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Overall status. Warnings in the log are scary, but ignored tests are
|
||||
* OK.
|
||||
*/
|
||||
if (logStats.hasErrors() || !failingSuites.isEmpty()) {
|
||||
if (logStats.hasErrors() || !getSuites(Status.ERROR).isEmpty()) {
|
||||
runStatus = Status.ERROR;
|
||||
} else {
|
||||
if (logStats.hasWarnings()) {
|
||||
runStatus = Status.WARN;
|
||||
} else {
|
||||
if (!pendingSuites.isEmpty()) {
|
||||
} else if (!getSuites(Status.PENDING).isEmpty()) {
|
||||
runStatus = Status.PENDING;
|
||||
} else {
|
||||
runStatus = Status.OK;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize all test results according to status.
|
||||
*/
|
||||
private void tallyTestResults(SuiteResults sResult) {
|
||||
for (TestResults tResult : sResult.getTests()) {
|
||||
allTests.add(tResult);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Populate {@link #allTests} with the tests for which we have no results.
|
||||
*/
|
||||
private void tallyTestContents(SuiteData suiteData) {
|
||||
Status suiteStatus = suiteData.getSuiteStatus();
|
||||
|
||||
for (String testName : suiteData.getContents().getTestNames()) {
|
||||
TestResults t = new TestResults(testName, suiteData.getName(), "",
|
||||
suiteStatus, "");
|
||||
allTests.add(t);
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Access the derived data.
|
||||
|
@ -248,19 +184,19 @@ public class DataModel {
|
|||
}
|
||||
|
||||
public boolean isAnyPasses() {
|
||||
return !(passingSuites.isEmpty() && passingTests.isEmpty());
|
||||
return !getTests(Status.OK).isEmpty();
|
||||
}
|
||||
|
||||
public boolean isAnyFailures() {
|
||||
return !(failingSuites.isEmpty() && failingTests.isEmpty());
|
||||
return !getTests(Status.ERROR).isEmpty();
|
||||
}
|
||||
|
||||
public boolean isAnyIgnores() {
|
||||
return !(ignoredSuites.isEmpty() && ignoredTests.isEmpty());
|
||||
return !getTests(Status.IGNORED).isEmpty();
|
||||
}
|
||||
|
||||
public boolean isAnyPending() {
|
||||
return !pendingSuites.isEmpty();
|
||||
return !getTests(Status.PENDING).isEmpty();
|
||||
}
|
||||
|
||||
public int getTotalSuiteCount() {
|
||||
|
@ -268,23 +204,33 @@ public class DataModel {
|
|||
}
|
||||
|
||||
public int getPassingSuiteCount() {
|
||||
return passingSuites.size();
|
||||
return getSuites(Status.OK).size();
|
||||
}
|
||||
|
||||
public int getFailingSuiteCount() {
|
||||
return failingSuites.size();
|
||||
return getSuites(Status.ERROR).size();
|
||||
}
|
||||
|
||||
public int getIgnoredSuiteCount() {
|
||||
return ignoredSuites.size();
|
||||
return getSuites(Status.IGNORED).size();
|
||||
}
|
||||
|
||||
public int getPendingSuitesCount() {
|
||||
return pendingSuites.size();
|
||||
public int getPendingSuiteCount() {
|
||||
return getSuites(Status.PENDING).size();
|
||||
}
|
||||
|
||||
public Collection<SuiteResults> getSuiteResults() {
|
||||
return Collections.unmodifiableCollection(suiteResults);
|
||||
public Collection<SuiteData> getAllSuites() {
|
||||
return suiteDataMap.values();
|
||||
}
|
||||
|
||||
public Map<String, SuiteData> getSuitesWithFailureMessages() {
|
||||
Map<String, SuiteData> map = new TreeMap<String, SuiteData>();
|
||||
for (SuiteData s : suiteDataMap.values()) {
|
||||
if (s.getFailureMessages() != null) {
|
||||
map.put(s.getName(), s);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public int getTotalTestCount() {
|
||||
|
@ -292,57 +238,57 @@ public class DataModel {
|
|||
}
|
||||
|
||||
public int getPassingTestCount() {
|
||||
return passingTests.size();
|
||||
return getTests(Status.OK).size();
|
||||
}
|
||||
|
||||
public int getFailingTestCount() {
|
||||
return failingTests.size();
|
||||
return getTests(Status.ERROR).size();
|
||||
}
|
||||
|
||||
public int getIgnoredTestCount() {
|
||||
return ignoredTests.size();
|
||||
return getTests(Status.IGNORED).size();
|
||||
}
|
||||
|
||||
public int getPendingTestsCount() {
|
||||
return pendingTests.size();
|
||||
public int getPendingTestCount() {
|
||||
return getTests(Status.PENDING).size();
|
||||
}
|
||||
|
||||
public Collection<TestResults> getAllTests() {
|
||||
public Collection<TestData> getAllTests() {
|
||||
return Collections.unmodifiableCollection(allTests);
|
||||
}
|
||||
|
||||
public Collection<TestResults> getFailingTests() {
|
||||
return Collections.unmodifiableCollection(failingTests);
|
||||
public Collection<TestData> getFailingTests() {
|
||||
return Collections.unmodifiableCollection(getTests(Status.ERROR));
|
||||
}
|
||||
|
||||
public Collection<TestResults> getIgnoredTests() {
|
||||
return Collections.unmodifiableCollection(ignoredTests);
|
||||
public Collection<TestData> getIgnoredTests() {
|
||||
return Collections.unmodifiableCollection(getTests(Status.IGNORED));
|
||||
}
|
||||
|
||||
public Collection<IgnoredTestInfo> getIgnoredTestInfo() {
|
||||
return ignoredTestList.getList();
|
||||
}
|
||||
|
||||
public String getOutputLink(String suiteName, String testName) {
|
||||
SuiteData sd = suiteDataMap.get(suiteName);
|
||||
if (sd != null) {
|
||||
SuiteResults s = sd.getResults();
|
||||
if (s != null) {
|
||||
if (testName.equals("*")) {
|
||||
return s.getOutputLink();
|
||||
} else {
|
||||
TestResults t = s.getTest(testName);
|
||||
if (t != null) {
|
||||
return t.getOutputLink();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public String getReasonForIgnoring(String suiteName, String testName) {
|
||||
return ignoredTestList.getReasonForIgnoring(suiteName, testName);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Helper methods
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Get the list of suites that have this status.
|
||||
*/
|
||||
private List<SuiteData> getSuites(Status st) {
|
||||
return suiteMapByStatus.get(st);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the list of tests that have this status.
|
||||
*/
|
||||
private List<TestData> getTests(Status st) {
|
||||
return testMapByStatus.get(st);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
package edu.cornell.mannlib.vitro.utilities.testrunner.datamodel;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults;
|
||||
|
@ -11,56 +14,168 @@ import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestRe
|
|||
* What do we know about this suite, both before it runs and after it has run?
|
||||
*/
|
||||
public class SuiteData {
|
||||
/**
|
||||
* If this suite has failure messages, the output link is to an anchor on
|
||||
* the same page.
|
||||
*/
|
||||
public static String failureMessageAnchor(SuiteData s) {
|
||||
return "suiteFailure_" + s.getName();
|
||||
}
|
||||
|
||||
private final String name;
|
||||
private final boolean ignored;
|
||||
private final SuiteContents contents;
|
||||
private final SuiteResults results;
|
||||
private final Status status;
|
||||
private final String outputLink;
|
||||
private final ProcessOutput failureMessages;
|
||||
|
||||
/**
|
||||
* This map iterates according to the order that the tests were specified in
|
||||
* the suite file.
|
||||
*/
|
||||
private final Map<String, TestData> testMap;
|
||||
|
||||
public SuiteData(String name, boolean ignored, SuiteContents contents,
|
||||
SuiteResults results, ProcessOutput failureMessages) {
|
||||
this.name = name;
|
||||
this.ignored = ignored;
|
||||
this.contents = contents;
|
||||
this.results = results;
|
||||
this.failureMessages = failureMessages;
|
||||
|
||||
if (ignored) {
|
||||
this.status = Status.IGNORED;
|
||||
this.outputLink = null;
|
||||
this.testMap = buildTestMap(contents, results);
|
||||
} else if (failureMessages != null) {
|
||||
this.status = Status.ERROR;
|
||||
this.outputLink = "#" + failureMessageAnchor(this);
|
||||
this.testMap = buildTestMap(contents, results);
|
||||
} else if (results != null) {
|
||||
this.testMap = buildTestMap(contents, results);
|
||||
this.status = buildStatusFromTestMap();
|
||||
this.outputLink = results.getOutputLink();
|
||||
} else {
|
||||
this.status = Status.PENDING;
|
||||
this.outputLink = null;
|
||||
this.testMap = buildTestMap(contents, results);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the test map. Do we have test results, or only the advance list of
|
||||
* tests?
|
||||
*/
|
||||
private Map<String, TestData> buildTestMap(SuiteContents contents,
|
||||
SuiteResults results) {
|
||||
if (results == null) {
|
||||
return buildTestMapFromContents(contents);
|
||||
} else {
|
||||
return buildTestMapFromResults(contents, results);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* All we have to build from is the contents of the Suite HTML file.
|
||||
*/
|
||||
private Map<String, TestData> buildTestMapFromContents(
|
||||
SuiteContents contents) {
|
||||
Map<String, TestData> map = new LinkedHashMap<String, TestData>();
|
||||
for (String testName : contents.getTestNames()) {
|
||||
map.put(testName, new TestData(testName, this.name, this.status,
|
||||
null));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* We can build from both the contents of the Suite HTML file and from the
|
||||
* test results output file.
|
||||
*/
|
||||
private Map<String, TestData> buildTestMapFromResults(
|
||||
SuiteContents contents, SuiteResults results) {
|
||||
Map<String, TestData> map = new LinkedHashMap<String, TestData>();
|
||||
for (String testName : contents.getTestNames()) {
|
||||
TestResults testResult = results.getTest(testName);
|
||||
if (testResult == null) {
|
||||
// This shouldn't happen. How do we show it?
|
||||
map.put(testName, new TestData(testName, this.name,
|
||||
Status.PENDING, null));
|
||||
} else {
|
||||
map.put(testName,
|
||||
new TestData(testName, this.name, testResult
|
||||
.getStatus(), testResult.getOutputLink()));
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* The suite ran to completion, so its status is the worst of the individual
|
||||
* test statuses.
|
||||
*/
|
||||
private Status buildStatusFromTestMap() {
|
||||
Status status = Status.OK;
|
||||
for (TestData t : this.testMap.values()) {
|
||||
status = Status.combine(status, t.getStatus());
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public boolean isIgnored() {
|
||||
return ignored;
|
||||
}
|
||||
|
||||
public SuiteContents getContents() {
|
||||
return contents;
|
||||
}
|
||||
|
||||
public SuiteResults getResults() {
|
||||
return results;
|
||||
}
|
||||
|
||||
public Status getSuiteStatus() {
|
||||
if (ignored) {
|
||||
return Status.WARN;
|
||||
}
|
||||
if (failureMessages != null) {
|
||||
return Status.ERROR;
|
||||
}
|
||||
if (results == null) {
|
||||
return Status.PENDING;
|
||||
}
|
||||
|
||||
/*
|
||||
* If we have results and no failure messages, scan the results for the
|
||||
* worst status.
|
||||
*/
|
||||
Status status = Status.OK;
|
||||
for (TestResults t : results.getTests()) {
|
||||
status = Status.combine(status, t.getStatus());
|
||||
}
|
||||
public Status getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public String getOutputLink() {
|
||||
return outputLink;
|
||||
}
|
||||
|
||||
public ProcessOutput getFailureMessages() {
|
||||
return failureMessages;
|
||||
}
|
||||
|
||||
public Map<String, TestData> getTestMap() {
|
||||
return testMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* What do we know about this test, both before it runs and after it has
|
||||
* run?
|
||||
*/
|
||||
public static class TestData {
|
||||
private final String testName;
|
||||
private final String suiteName;
|
||||
private final Status status;
|
||||
private final String outputLink;
|
||||
|
||||
public TestData(String testName, String suiteName, Status status,
|
||||
String outputLink) {
|
||||
this.testName = testName;
|
||||
this.suiteName = suiteName;
|
||||
this.status = status;
|
||||
this.outputLink = outputLink;
|
||||
}
|
||||
|
||||
public String getTestName() {
|
||||
return testName;
|
||||
}
|
||||
|
||||
public String getSuiteName() {
|
||||
return suiteName;
|
||||
}
|
||||
|
||||
public Status getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public String getOutputLink() {
|
||||
return outputLink;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TestData[testName=" + testName + ", suiteName=" + suiteName
|
||||
+ ", status=" + status + ", outputLink=" + outputLink + "]";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -9,6 +9,8 @@ import java.util.Date;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.listener.Listener;
|
||||
|
@ -138,6 +140,7 @@ public class OutputDataListener implements Listener {
|
|||
* interesting if it indicates a suite failure.
|
||||
*/
|
||||
public static class ProcessOutput {
|
||||
private static final String SUITE_FAILURE_PATTERN = "exception|error(?i)";
|
||||
private final String suiteName;
|
||||
private final StringBuilder stdout = new StringBuilder();
|
||||
private final StringBuilder errout = new StringBuilder();
|
||||
|
@ -167,7 +170,9 @@ public class OutputDataListener implements Listener {
|
|||
}
|
||||
|
||||
public boolean isSuiteFailure() {
|
||||
return errout.length() > 0;
|
||||
Pattern p = Pattern.compile(SUITE_FAILURE_PATTERN);
|
||||
Matcher m = p.matcher(errout);
|
||||
return m.find();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -11,6 +11,7 @@ import java.io.Reader;
|
|||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.FileHelper;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.IgnoredTests.IgnoredTestInfo;
|
||||
|
@ -18,7 +19,9 @@ import edu.cornell.mannlib.vitro.utilities.testrunner.LogStats;
|
|||
import edu.cornell.mannlib.vitro.utilities.testrunner.SeleniumRunnerParameters;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.Status;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.DataModel;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.SuiteResults.TestResults;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.SuiteData;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.datamodel.SuiteData.TestData;
|
||||
import edu.cornell.mannlib.vitro.utilities.testrunner.output.OutputDataListener.ProcessOutput;
|
||||
|
||||
/**
|
||||
* Creates the summary HTML file.
|
||||
|
@ -60,6 +63,7 @@ public class OutputSummaryFormatter {
|
|||
writeIgnoreSection(writer);
|
||||
writeSuitesSection(writer);
|
||||
writeAllTestsSection(writer);
|
||||
writeSuiteErrorMessagesSection(writer);
|
||||
writeFooter(writer);
|
||||
} catch (IOException e) {
|
||||
// There is no appeal for any problems here. Just report them.
|
||||
|
@ -92,207 +96,234 @@ public class OutputSummaryFormatter {
|
|||
}
|
||||
}
|
||||
|
||||
private void writeHeader(PrintWriter writer) {
|
||||
private void writeHeader(PrintWriter w) {
|
||||
Status runStatus = dataModel.getRunStatus();
|
||||
String statusString = (runStatus == Status.PENDING) ? "IN PROGRESS"
|
||||
: runStatus.toString();
|
||||
String startString = formatDateTime(dataModel.getStartTime());
|
||||
|
||||
writer.println("<html>");
|
||||
writer.println("<head>");
|
||||
writer.println(" <title>Summary of Acceptance Tests " + startString
|
||||
w.println("<html>");
|
||||
w.println("<head>");
|
||||
w.println(" <title>Summary of Acceptance Tests " + startString
|
||||
+ "</title>");
|
||||
writer.println(" <link rel=\"stylesheet\" type=\"text/css\" "
|
||||
w.println(" <link rel=\"stylesheet\" type=\"text/css\" "
|
||||
+ "href=\"summary.css\">");
|
||||
writer.println("</head>");
|
||||
writer.println("<body>");
|
||||
writer.println();
|
||||
writer.println(" <div class=\"heading\">");
|
||||
writer.println(" Acceptance test results: " + startString);
|
||||
writer.println(" <div class=\"" + runStatus.getHtmlClass()
|
||||
w.println("</head>");
|
||||
w.println("<body>");
|
||||
w.println();
|
||||
w.println(" <div class=\"heading\">");
|
||||
w.println(" Acceptance test results: " + startString);
|
||||
w.println(" <div class=\"" + runStatus.getHtmlClass()
|
||||
+ " one-word\">" + statusString + "</div>");
|
||||
writer.println(" </div>");
|
||||
w.println(" </div>");
|
||||
}
|
||||
|
||||
private void writeStatsSection(PrintWriter writer) {
|
||||
private void writeStatsSection(PrintWriter w) {
|
||||
String passClass = dataModel.isAnyPasses() ? Status.OK.getHtmlClass()
|
||||
: "";
|
||||
String failClass = dataModel.isAnyFailures() ? Status.ERROR
|
||||
.getHtmlClass() : "";
|
||||
String ignoreClass = dataModel.isAnyIgnores() ? Status.WARN
|
||||
String ignoreClass = dataModel.isAnyIgnores() ? Status.IGNORED
|
||||
.getHtmlClass() : "";
|
||||
|
||||
String start = formatDateTime(dataModel.getStartTime());
|
||||
String end = formatDateTime(dataModel.getEndTime());
|
||||
String elapsed = formatElapsedTime(dataModel.getElapsedTime());
|
||||
|
||||
writer.println(" <div class=\"section\">Summary</div>");
|
||||
writer.println();
|
||||
writer.println(" <table class=\"summary\" cellspacing=\"0\">");
|
||||
writer.println(" <tr>");
|
||||
writer.println(" <td>");
|
||||
writer.println(" <table cellspacing=\"0\">");
|
||||
writer.println(" <tr><td>Start time:</td><td>" + start
|
||||
w.println(" <div class=\"section\">Summary</div>");
|
||||
w.println();
|
||||
w.println(" <table class=\"summary\" cellspacing=\"0\">");
|
||||
w.println(" <tr>");
|
||||
w.println(" <td>");
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
w.println(" <tr><td>Start time:</td><td>" + start
|
||||
+ "</td></tr>");
|
||||
writer.println(" <tr><td>End time:</td><td>" + end
|
||||
w.println(" <tr><td>End time:</td><td>" + end + "</td></tr>");
|
||||
w.println(" <tr><td>Elapsed time</td><td>" + elapsed
|
||||
+ "</td></tr>");
|
||||
writer.println(" <tr><td>Elapsed time</td><td>" + elapsed
|
||||
+ "</td></tr>");
|
||||
writer.println(" </table>");
|
||||
writer.println(" </td>");
|
||||
writer.println(" <td>");
|
||||
writer.println(" <table class=\"tallys\" cellspacing=\"0\">");
|
||||
writer.println(" <tr><th> </th><th>Suites</th><th>Tests</th>");
|
||||
writer.println(" <tr class=\"" + passClass
|
||||
w.println(" </table>");
|
||||
w.println(" </td>");
|
||||
w.println(" <td>");
|
||||
w.println(" <table class=\"tallys\" cellspacing=\"0\">");
|
||||
w.println(" <tr><th> </th><th>Suites</th><th>Tests</th>");
|
||||
w.println(" <tr class=\"" + passClass
|
||||
+ "\"><td>Passed</td><td>" + dataModel.getPassingSuiteCount()
|
||||
+ "</td><td>" + dataModel.getPassingTestCount() + "</td>");
|
||||
writer.println(" <tr class=\"" + failClass
|
||||
w.println(" <tr class=\"" + failClass
|
||||
+ "\"><td>Failed</td><td>" + dataModel.getFailingSuiteCount()
|
||||
+ "</td><td>" + dataModel.getFailingTestCount() + "</td>");
|
||||
writer.println(" <tr class=\"" + ignoreClass
|
||||
w.println(" <tr class=\"" + ignoreClass
|
||||
+ "\"><td>Ignored</td><td>" + dataModel.getIgnoredSuiteCount()
|
||||
+ "</td><td>" + dataModel.getIgnoredTestCount() + "</td>");
|
||||
if (dataModel.isAnyPending()) {
|
||||
writer.println(" <tr><td>Pending</td><td>"
|
||||
+ dataModel.getPendingSuitesCount() + "</td><td>"
|
||||
+ dataModel.getPendingTestsCount() + "</td>");
|
||||
w.println(" <tr class=\"" + Status.PENDING.getHtmlClass()
|
||||
+ "\"><td>Pending</td><td>"
|
||||
+ dataModel.getPendingSuiteCount() + "</td><td>"
|
||||
+ dataModel.getPendingTestCount() + "</td>");
|
||||
}
|
||||
writer.println(" <tr><td class=\"total\">Total</td><td>"
|
||||
w.println(" <tr><td class=\"total\">Total</td><td>"
|
||||
+ dataModel.getTotalSuiteCount() + "</td><td>"
|
||||
+ dataModel.getTotalTestCount() + "</td>");
|
||||
writer.println(" </table>");
|
||||
writer.println(" </td>");
|
||||
writer.println(" </tr>");
|
||||
writer.println(" </table>");
|
||||
writer.println();
|
||||
w.println(" </table>");
|
||||
w.println(" </td>");
|
||||
w.println(" </tr>");
|
||||
w.println(" </table>");
|
||||
w.println();
|
||||
}
|
||||
|
||||
private void writeErrorMessagesSection(PrintWriter writer) {
|
||||
private void writeErrorMessagesSection(PrintWriter w) {
|
||||
String errorClass = Status.ERROR.getHtmlClass();
|
||||
String warnClass = Status.WARN.getHtmlClass();
|
||||
|
||||
writer.println(" <div class=section>Errors and warnings</div>");
|
||||
writer.println();
|
||||
writer.println(" <table cellspacing=\"0\">");
|
||||
w.println(" <div class=section>Errors and warnings</div>");
|
||||
w.println();
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
|
||||
if ((!logStats.hasErrors()) && (!logStats.hasWarnings())) {
|
||||
writer.println(" <tr><td colspan=\"2\">No errors or warnings</td></tr>");
|
||||
w.println(" <tr><td colspan=\"2\">No errors or warnings</td></tr>");
|
||||
} else {
|
||||
for (String e : logStats.getErrors()) {
|
||||
writer.println(" <tr class=\"" + errorClass
|
||||
w.println(" <tr class=\"" + errorClass
|
||||
+ "\"><td>ERROR</td><td>" + e + "</td></tr>");
|
||||
}
|
||||
for (String w : logStats.getWarnings()) {
|
||||
writer.println(" <tr class=\"" + warnClass
|
||||
+ "\"><td>ERROR</td><td>" + w + "</td></tr>");
|
||||
}
|
||||
}
|
||||
writer.println(" </table>");
|
||||
writer.println();
|
||||
w.println(" </table>");
|
||||
w.println();
|
||||
}
|
||||
|
||||
private void writeFailureSection(PrintWriter writer) {
|
||||
private void writeFailureSection(PrintWriter w) {
|
||||
String errorClass = Status.ERROR.getHtmlClass();
|
||||
Collection<TestResults> failingTests = dataModel.getFailingTests();
|
||||
Collection<TestData> failingTests = dataModel.getFailingTests();
|
||||
|
||||
writer.println(" <div class=section>Failing tests</div>");
|
||||
writer.println();
|
||||
writer.println(" <table cellspacing=\"0\">");
|
||||
writer.println(" <tr><th>Suite name</th><th>Test name</th></tr>\n");
|
||||
w.println(" <div class=section>Failures</div>");
|
||||
w.println();
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
w.println(" <tr><th>Suite name</th><th>Test name</th></tr>\n");
|
||||
if (failingTests.isEmpty()) {
|
||||
writer.println(" <tr><td colspan=\"2\">No tests failed.</td>"
|
||||
w.println(" <tr><td colspan=\"2\">No tests failed.</td>"
|
||||
+ "</tr>");
|
||||
} else {
|
||||
for (TestResults t : failingTests) {
|
||||
writer.println(" <tr class=\"" + errorClass + "\">");
|
||||
writer.println(" <td>" + t.getSuiteName() + "</td>");
|
||||
writer.println(" <td><a href=\"" + t.getOutputLink()
|
||||
+ "\">" + t.getTestName() + "</a></td>");
|
||||
writer.println(" </tr>");
|
||||
Map<String, SuiteData> failedSuiteMap = dataModel
|
||||
.getSuitesWithFailureMessages();
|
||||
for (SuiteData s : failedSuiteMap.values()) {
|
||||
w.println(" <tr class=\"" + errorClass + "\">");
|
||||
w.println(" <td>" + s.getName() + "</td>");
|
||||
w.println(" <td>" + outputLink(s) + "</td>");
|
||||
w.println(" </tr>");
|
||||
}
|
||||
for (TestData t : failingTests) {
|
||||
if (!failedSuiteMap.containsKey(t.getSuiteName())) {
|
||||
w.println(" <tr class=\"" + errorClass + "\">");
|
||||
w.println(" <td>" + t.getSuiteName() + "</td>");
|
||||
w.println(" <td>" + outputLink(t) + "</td>");
|
||||
w.println(" </tr>");
|
||||
}
|
||||
}
|
||||
writer.println(" </table>");
|
||||
writer.println();
|
||||
}
|
||||
w.println(" </table>");
|
||||
w.println();
|
||||
}
|
||||
|
||||
private void writeIgnoreSection(PrintWriter writer) {
|
||||
String warnClass = Status.WARN.getHtmlClass();
|
||||
private void writeIgnoreSection(PrintWriter w) {
|
||||
String warnClass = Status.IGNORED.getHtmlClass();
|
||||
Collection<IgnoredTestInfo> ignoredTests = dataModel
|
||||
.getIgnoredTestInfo();
|
||||
|
||||
writer.println(" <div class=section>Ignored tests</div>");
|
||||
writer.println();
|
||||
writer.println(" <table cellspacing=\"0\">");
|
||||
writer.println(" <tr><th>Suite name</th><th>Test name</th>"
|
||||
w.println(" <div class=section>Ignored</div>");
|
||||
w.println();
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
w.println(" <tr><th>Suite name</th><th>Test name</th>"
|
||||
+ "<th>Reason for ignoring</th></tr>\n");
|
||||
if (ignoredTests.isEmpty()) {
|
||||
writer.println(" <tr><td colspan=\"3\">No tests ignored.</td>"
|
||||
w.println(" <tr><td colspan=\"3\">No tests ignored.</td>"
|
||||
+ "</tr>");
|
||||
} else {
|
||||
for (IgnoredTestInfo info : ignoredTests) {
|
||||
String suiteName = info.suiteName;
|
||||
String testName = info.testName;
|
||||
String link = dataModel.getOutputLink(suiteName, testName);
|
||||
String reason = dataModel.getReasonForIgnoring(suiteName,
|
||||
testName);
|
||||
|
||||
writer.println(" <tr class=\"" + warnClass + "\">");
|
||||
writer.println(" <td>" + suiteName + "</td>");
|
||||
if (link.isEmpty()) {
|
||||
writer.println(" <td>" + testName + "</td>");
|
||||
} else {
|
||||
writer.println(" <td><a href=\"" + link + "\">"
|
||||
+ testName + "</a></td>");
|
||||
}
|
||||
writer.println(" <td>" + reason + "</td>");
|
||||
writer.println(" </tr>");
|
||||
w.println(" <tr class=\"" + warnClass + "\">");
|
||||
w.println(" <td>" + suiteName + "</td>");
|
||||
w.println(" <td>" + testName + "</td>");
|
||||
w.println(" <td>" + reason + "</td>");
|
||||
w.println(" </tr>");
|
||||
}
|
||||
}
|
||||
writer.println(" </table>");
|
||||
writer.println();
|
||||
w.println(" </table>");
|
||||
w.println();
|
||||
}
|
||||
|
||||
private void writeSuitesSection(PrintWriter writer) {
|
||||
writer.println(" <div class=section>Suites</div>");
|
||||
writer.println();
|
||||
writer.println(" <table cellspacing=\"0\">");
|
||||
private void writeSuitesSection(PrintWriter w) {
|
||||
w.println(" <div class=section>Suites Summary</div>");
|
||||
w.println();
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
|
||||
for (SuiteResults s : dataModel.getSuiteResults()) {
|
||||
writer.println(" <tr class=\"" + s.getStatus().getHtmlClass()
|
||||
for (SuiteData s : dataModel.getAllSuites()) {
|
||||
w.println(" <tr class=\"" + s.getStatus().getHtmlClass() + "\">");
|
||||
w.println(" <td>" + outputLink(s) + "</td>");
|
||||
w.println(" <td>" + s.getStatus() + "</td>");
|
||||
w.println(" </tr>");
|
||||
}
|
||||
|
||||
w.println(" </table>");
|
||||
w.println();
|
||||
}
|
||||
|
||||
private void writeAllTestsSection(PrintWriter w) {
|
||||
Collection<TestData> allTests = dataModel.getAllTests();
|
||||
|
||||
w.println(" <div class=section>All tests</div>");
|
||||
w.println();
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
|
||||
w.println(" <tr><th>Suite name</th><th>Test name</th><th> </th></tr>\n");
|
||||
for (TestData t : allTests) {
|
||||
w.println(" <tr class=\"" + t.getStatus().getHtmlClass() + "\">");
|
||||
w.println(" <td>" + t.getSuiteName() + "</td>");
|
||||
w.println(" <td>" + outputLink(t) + "</td>");
|
||||
w.println(" <td>" + t.getStatus() + "</td>");
|
||||
w.println(" </tr>");
|
||||
}
|
||||
|
||||
w.println(" </table>");
|
||||
w.println();
|
||||
}
|
||||
|
||||
private void writeSuiteErrorMessagesSection(PrintWriter w) {
|
||||
Map<String, SuiteData> failedSuiteMap = dataModel
|
||||
.getSuitesWithFailureMessages();
|
||||
if (failedSuiteMap.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
w.println(" <div class=section>All tests</div>");
|
||||
w.println();
|
||||
for (SuiteData s : failedSuiteMap.values()) {
|
||||
ProcessOutput output = s.getFailureMessages();
|
||||
|
||||
w.println(" <a name=\"" + SuiteData.failureMessageAnchor(s)
|
||||
+ "\">");
|
||||
writer.println(" <td><a href=\"" + s.getOutputLink() + "\">"
|
||||
+ s.getName() + "</a></td>");
|
||||
writer.println(" </tr>");
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
w.println(" <tr><th>Standard Output</th></tr>\n");
|
||||
w.println(" <tr><td><pre>" + output.getStdout()
|
||||
+ "</pre></td></tr>\n");
|
||||
w.println(" </table>");
|
||||
w.println("<br/> <br/>");
|
||||
|
||||
w.println(" <table cellspacing=\"0\">");
|
||||
w.println(" <tr><th>Error Output</th></tr>\n");
|
||||
w.println(" <tr><td><pre>" + output.getErrout()
|
||||
+ "</pre></td></tr>\n");
|
||||
w.println(" </table>");
|
||||
w.println("<br/> <br/>");
|
||||
w.println();
|
||||
}
|
||||
}
|
||||
|
||||
writer.println(" </table>");
|
||||
writer.println();
|
||||
}
|
||||
|
||||
private void writeAllTestsSection(PrintWriter writer) {
|
||||
Collection<TestResults> allTests = dataModel.getAllTests();
|
||||
|
||||
writer.println(" <div class=section>All tests</div>");
|
||||
writer.println();
|
||||
writer.println(" <table cellspacing=\"0\">");
|
||||
|
||||
writer.println(" <tr><th>Suite name</th><th>Test name</th></tr>\n");
|
||||
for (TestResults t : allTests) {
|
||||
writer.println(" <tr class=\"" + t.getStatus().getHtmlClass()
|
||||
+ "\">");
|
||||
writer.println(" <td>" + t.getSuiteName() + "</td>");
|
||||
writer.println(" <td><a href=\"" + t.getOutputLink() + "\">"
|
||||
+ t.getTestName() + "</a></td>");
|
||||
writer.println(" </tr>");
|
||||
}
|
||||
|
||||
writer.println(" </table>");
|
||||
writer.println();
|
||||
}
|
||||
|
||||
private void writeFooter(PrintWriter writer) {
|
||||
writer.println(" <div class=section>Log</div>");
|
||||
writer.println(" <pre>");
|
||||
private void writeFooter(PrintWriter w) {
|
||||
w.println(" <div class=section>Log</div>");
|
||||
w.println(" <pre>");
|
||||
|
||||
Reader reader = null;
|
||||
try {
|
||||
|
@ -300,7 +331,7 @@ public class OutputSummaryFormatter {
|
|||
char[] buffer = new char[4096];
|
||||
int howMany;
|
||||
while (-1 != (howMany = reader.read(buffer))) {
|
||||
writer.write(buffer, 0, howMany);
|
||||
w.write(buffer, 0, howMany);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
|
@ -314,9 +345,9 @@ public class OutputSummaryFormatter {
|
|||
}
|
||||
}
|
||||
|
||||
writer.println(" </pre>");
|
||||
writer.println("</body>");
|
||||
writer.println("</html>");
|
||||
w.println(" </pre>");
|
||||
w.println("</body>");
|
||||
w.println("</html>");
|
||||
}
|
||||
|
||||
private String formatElapsedTime(long elapsed) {
|
||||
|
@ -350,4 +381,21 @@ public class OutputSummaryFormatter {
|
|||
return dateFormat.format(new Date(dateTime));
|
||||
}
|
||||
|
||||
private String outputLink(SuiteData s) {
|
||||
if (s.getOutputLink() == null) {
|
||||
return s.getName();
|
||||
} else {
|
||||
return "<a href=\"" + s.getOutputLink() + "\">" + s.getName()
|
||||
+ "</a>";
|
||||
}
|
||||
}
|
||||
|
||||
private String outputLink(TestData t) {
|
||||
if (t.getOutputLink() == null) {
|
||||
return t.getTestName();
|
||||
} else {
|
||||
return "<a href=\"" + t.getOutputLink() + "\">" + t.getTestName()
|
||||
+ "</a>";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -76,21 +76,16 @@ public class SuiteResults {
|
|||
String testLink = outputLink + m.group(2);
|
||||
|
||||
Status testStatus;
|
||||
String reasonForIgnoring;
|
||||
if ("status_passed".equals(m.group(1))) {
|
||||
testStatus = Status.OK;
|
||||
reasonForIgnoring = "";
|
||||
} else if (ignoredTests.isIgnored(suiteName, testName)) {
|
||||
testStatus = Status.WARN;
|
||||
reasonForIgnoring = ignoredTests.getReasonForIgnoring(
|
||||
suiteName, testName);
|
||||
testStatus = Status.IGNORED;
|
||||
} else {
|
||||
testStatus = Status.ERROR;
|
||||
reasonForIgnoring = "";
|
||||
}
|
||||
|
||||
tests.add(new TestResults(testName, suiteName, testLink,
|
||||
testStatus, reasonForIgnoring));
|
||||
testStatus));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -163,15 +158,13 @@ public class SuiteResults {
|
|||
private final String suite;
|
||||
private final String outputLink;
|
||||
private final Status status;
|
||||
private final String reasonForIgnoring;
|
||||
|
||||
public TestResults(String name, String suite, String outputLink,
|
||||
Status status, String reasonForIgnoring) {
|
||||
Status status) {
|
||||
this.name = name;
|
||||
this.suite = suite;
|
||||
this.outputLink = outputLink;
|
||||
this.status = status;
|
||||
this.reasonForIgnoring = reasonForIgnoring;
|
||||
}
|
||||
|
||||
public Status getStatus() {
|
||||
|
@ -190,9 +183,6 @@ public class SuiteResults {
|
|||
return outputLink;
|
||||
}
|
||||
|
||||
public String getReasonForIgnoring() {
|
||||
return reasonForIgnoring;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -67,6 +67,10 @@ table.tallys td.total {
|
|||
background: rgb(100%, 100%, 60%);
|
||||
}
|
||||
|
||||
.pending {
|
||||
background: rgb(90%, 90%, 100%);
|
||||
}
|
||||
|
||||
.one-word {
|
||||
width: 20%;
|
||||
text-align: center;
|
||||
|
|
Loading…
Add table
Reference in a new issue