From eea97d50e96b1aab31b7910bc3899c42279bcc22 Mon Sep 17 00:00:00 2001 From: jeb228 Date: Wed, 19 May 2010 21:28:37 +0000 Subject: [PATCH] NIHVIVO-222 Complete the batch version. --- .../utilities/testrunner/FileHelper.java | 133 +++++++ .../utilities/testrunner/IgnoredTests.java | 130 +++++++ .../vitro/utilities/testrunner/Listener.java | 17 + .../vitro/utilities/testrunner/LogStats.java | 133 +++++++ .../utilities/testrunner/OutputManager.java | 83 ++++ .../testrunner/OutputSummaryFormatter.java | 356 ++++++++++++++++++ .../utilities/testrunner/SeleniumRunner.java | 49 ++- .../testrunner/SeleniumRunnerParameters.java | 47 ++- .../vitro/utilities/testrunner/Status.java | 33 ++ .../utilities/testrunner/SuiteRunner.java | 10 +- .../utilities/testrunner/SuiteStats.java | 186 +++++++++ .../testrunner/UploadAreaCleaner.java | 54 +-- 12 files changed, 1150 insertions(+), 81 deletions(-) create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/FileHelper.java create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/LogStats.java create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputManager.java create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputSummaryFormatter.java create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Status.java create mode 100644 utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteStats.java diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/FileHelper.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/FileHelper.java new file mode 100644 index 000000000..029a3788b --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/FileHelper.java @@ -0,0 +1,133 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Some utility methods for dealing with files and directories. + */ +public class FileHelper { + + /** + * Delete a file. If it can't be deleted, complain. + */ + public static void deleteFile(File file) throws IOException { + if (file.exists()) { + file.delete(); + } + if (!file.exists()) { + return; + } + + /* + * If we were unable to delete the file, is it because it's a non-empty + * directory? + */ + if (file.isDirectory()) { + final StringBuffer message = new StringBuffer( + "Can't delete directory '" + file.getPath() + "'\n"); + file.listFiles(new FileFilter() { + public boolean accept(File pathname) { + message.append(" contains file '" + pathname + "'\n"); + return true; + } + }); + throw new IOException(message.toString().trim()); + } else { + throw new IOException("Unable to delete file '" + file.getPath() + + "'"); + } + } + + /** + * Delete all of the files in a directory, any sub-directories, and the + * directory itself. + */ + public static void purgeDirectoryRecursively(File directory) + throws IOException { + File[] files = directory.listFiles(); + for (File file : files) { + if (file.isDirectory()) { + purgeDirectoryRecursively(file); + } else { + deleteFile(file); + } + } + deleteFile(directory); + } + + /** + * Confirm that this is an existing, readable file. + */ + public static void checkReadableFile(File file, String label) { + if (!file.exists()) { + throw new IllegalArgumentException(label + " does not exist."); + } + if (!file.isFile()) { + throw new IllegalArgumentException(label + " is not a file."); + } + if (!file.canRead()) { + throw new IllegalArgumentException(label + " is not readable."); + } + } + + /** + * Get the name of this file, without the extension. + */ + public static String baseName(File file) { + String name = file.getName(); + int periodHere = name.indexOf('.'); + if (periodHere == -1) { + return name; + } else { + return name.substring(0, periodHere); + } + } + + /** + * Copy the contents of a file to a new location. If the target file already + * exists, it will be over-written. + */ + public static void copy(File source, File target) throws IOException { + InputStream input = null; + OutputStream output = null; + + try { + input = new FileInputStream(source); + output = new FileOutputStream(target); + int howMany; + byte[] buffer = new byte[4096]; + while (-1 != (howMany = input.read(buffer))) { + output.write(buffer, 0, howMany); + } + }finally { + if (input != null) { + try { + input.close(); + } catch (IOException e1) { + e1.printStackTrace(); + } + } + if (output != null) { + try { + output.close(); + } catch (IOException e1) { + e1.printStackTrace(); + } + } + } + } + + /** No need to instantiate this, since all methods are static. */ + private FileHelper() { + // Nothing to initialize. + } + +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java new file mode 100644 index 000000000..8140cd200 --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/IgnoredTests.java @@ -0,0 +1,130 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A list of tests to be ignored - they are expected to fail, and their failure + * is logged with a warning, not an error. + */ +public class IgnoredTests { + private final File file; + private final List tests; + + /** + *

+ * Parse the file of ignored tests. + *

+ *

+ * Ignore any blank line, or any line starting with '#' or '!' + *

+ *

+ * Each other line describes an ignored test. The line contains the suite + * name, a comma (with optional space), the test name (with optional space) + * and optionally a comment, starting with a '#'. + *

+ */ + public IgnoredTests(File file) { + this.file = file; + List tests = new ArrayList(); + + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(file)); + String line; + while (null != (line = reader.readLine())) { + line = line.trim(); + if ((line.length() == 0) || (line.charAt(0) == '#') + || (line.charAt(0) == '!')) { + continue; + } + Pattern p = Pattern.compile("([^,#]+),([^,#]+)(#(.*))?"); + Matcher m = p.matcher(line); + if (m.matches()) { + tests.add(new IgnoredTestInfo(m.group(1), m.group(2), m + .group(4))); + } else { + throw new FatalException( + "Bad format on ignored test description: '" + line + + "', should be " + + ", [# comment]"); + } + } + } catch (IOException e) { + throw new FatalException( + "Failed to parse the list of ignored tests: '" + + file.getPath() + "'", e); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + this.tests = Collections.unmodifiableList(tests); + } + + /** + * Is this test ignored or not? + */ + public boolean isIgnored(String suiteName, String testName) { + for (IgnoredTestInfo test : tests) { + if (test.matches(suiteName, testName)) { + return true; + } + } + return false; + } + + /** + * If this test is ignored, what is the reason? If not, return an empty + * string. + */ + public String getReasonForIgnoring(String suiteName, String testName) { + for (IgnoredTestInfo test : tests) { + if (test.matches(suiteName, testName)) { + return test.comment; + } + } + return ""; + } + + public String toString() { + String s = " ignored tests from " + file.getPath() + "\n"; + for (IgnoredTestInfo test : tests) { + s += " " + test.suiteName + ", " + test.testName + "\n"; + } + return s; + } + + private static class IgnoredTestInfo { + final String suiteName; + final String testName; + final String comment; + + public IgnoredTestInfo(String suiteName, String testName, String comment) { + this.suiteName = suiteName.trim(); + this.testName = testName.trim(); + this.comment = (comment == null) ? "" : comment.trim(); + } + + public boolean matches(String suiteName, String testName) { + return this.suiteName.equals(suiteName) + && this.testName.equals(testName); + } + + } + +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Listener.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Listener.java index 54556f7f0..d1c8ce92e 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Listener.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Listener.java @@ -49,9 +49,26 @@ public class Listener { log(e); } + public void runEndTime() { + log("Testing complete."); + } + public void runStopped() { log("Run stopped."); } + + public void cleanOutputStart(File outputDirectory) { + log("Output area cleaning started: " + outputDirectory.getPath()); + } + + public void cleanOutputFailed(File outputDirectory, IOException e) { + log("Output area cleaning failed: " + outputDirectory.getPath()); + log(e); + } + + public void cleanOutputStop(File outputDirectory) { + log("Output area cleaning stopped: " + outputDirectory.getPath()); + } public void webappStopping(String tomcatStopCommand) { log("Stopping tomcat: " + tomcatStopCommand); diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/LogStats.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/LogStats.java new file mode 100644 index 000000000..63ba6c4fc --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/LogStats.java @@ -0,0 +1,133 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Extract any summary information from the log file. + */ +public class LogStats { + private static final Pattern START_TIME_PATTERN = Pattern + .compile("(.*) Run started."); + private static final Pattern END_TIME_PATTERN = Pattern + .compile("(.*) Testing complete."); + private static final Pattern SUITE_NAME_PATTERN = Pattern + .compile("Running suite (.*)"); + private static final Pattern ERROR_PATTERN = Pattern + .compile("ERROR\\s+(.*)"); + private static final Pattern WARNING_PATTERN = Pattern + .compile("WARN\\s+(.*)"); + private static final SimpleDateFormat dateParser = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss.SSS"); + + /** + * Factory method. + */ + public static LogStats parse(File logFile) { + return new LogStats(logFile); + } + + private long startTime; + private long endTime; + private final List suiteNames = new ArrayList(); + private final List errors = new ArrayList(); + private final List warnings = new ArrayList(); + + private LogStats(File logFile) { + + BufferedReader reader = null; + String line; + try { + reader = new BufferedReader(new FileReader(logFile)); + while (null != (line = reader.readLine())) { + Matcher m; + m = START_TIME_PATTERN.matcher(line); + if (m.matches()) { + startTime = parseTime(m.group(1)); + } else { + m = END_TIME_PATTERN.matcher(line); + if (m.matches()) { + endTime = parseTime(m.group(1)); + } else { + m = SUITE_NAME_PATTERN.matcher(line); + if (m.matches()) { + suiteNames.add(m.group(1)); + } else { + m = ERROR_PATTERN.matcher(line); + if (m.matches()) { + errors.add(m.group(1)); + } else { + m = WARNING_PATTERN.matcher(line); + if (m.matches()) { + warnings.add(m.group(1)); + } + } + } + } + } + } + + } catch (IOException e) { + // Can't give up - I need to create as much output as I can. + e.printStackTrace(); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + private long parseTime(String timeString) { + try { + return dateParser.parse(timeString).getTime(); + } catch (ParseException e) { + e.printStackTrace(); + return 0L; + } + } + + public boolean hasErrors() { + return !errors.isEmpty(); + } + + public Collection getErrors() { + return Collections.unmodifiableCollection(errors); + } + + public boolean hasWarnings() { + return !warnings.isEmpty(); + } + + public Collection getWarnings() { + return Collections.unmodifiableCollection(warnings); + } + + public long getStartTime() { + return startTime; + } + + public long getEndTime() { + return endTime; + } + + public long getElapsedTime() { + return Math.abs(endTime - startTime); + } + +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputManager.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputManager.java new file mode 100644 index 000000000..cfc95ef6e --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputManager.java @@ -0,0 +1,83 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Manages the contents of the output area. Removes old files prior to a run. + * Creates a unified summary of the test suite outputs. + */ +public class OutputManager { + private final SeleniumRunnerParameters parms; + private final Listener listener; + + public OutputManager(SeleniumRunnerParameters parms) { + this.parms = parms; + this.listener = parms.getListener(); + } + + /** + * Delete any output files from previous runs. + */ + public void cleanOutputDirectory() throws IOException { + File outputDirectory = parms.getOutputDirectory(); + listener.cleanOutputStart(outputDirectory); + + try { + for (File file : outputDirectory.listFiles()) { + // Skip the log file, since we are already over-writing it. + if (file.equals(parms.getLogFile())) { + continue; + } + // Skip any hidden files (like .svn) + if (file.getPath().startsWith(".")) { + continue; + } + // Delete all of the others. + if (file.isFile()) { + FileHelper.deleteFile(file); + } else { + FileHelper.purgeDirectoryRecursively(file); + } + } + } catch (IOException e) { + listener.cleanOutputFailed(outputDirectory, e); + throw e; + } finally { + listener.cleanOutputStop(outputDirectory); + } + } + + /** + * Parse each of the output files from the test suites, and create a unified + * output file. + */ + public void summarizeOutput() { + LogStats log = LogStats.parse(parms.getLogFile()); + + List suites = new ArrayList(); + for (File outputFile : parms.getOutputDirectory().listFiles( + new HtmlFileFilter())) { + SuiteStats suite = SuiteStats.parse(parms, outputFile); + if (suite != null) { + suites.add(suite); + } + } + + OutputSummaryFormatter formatter = new OutputSummaryFormatter(parms); + formatter.format(log, suites); + } + + private static class HtmlFileFilter implements FileFilter { + public boolean accept(File path) { + return path.getName().endsWith(".html") + || path.getName().endsWith(".htm"); + } + + } +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputSummaryFormatter.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputSummaryFormatter.java new file mode 100644 index 000000000..a4a02140c --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/OutputSummaryFormatter.java @@ -0,0 +1,356 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Reader; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import edu.cornell.mannlib.vitro.utilities.testrunner.SuiteStats.TestInfo; + +/** + * Creates the summary HTML file. + */ +public class OutputSummaryFormatter { + public static final String SUMMARY_HTML_FILENAME = "summary.html"; + public static final String SUMMARY_CSS_FILENAME = "summary.css"; + private final SimpleDateFormat dateFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + private final SeleniumRunnerParameters parms; + + private LogStats log; + private List suites; + private Status runStatus; + private List allTests = new ArrayList(); + private int passingTestCount; + private List failingTests = new ArrayList(); + private List ignoredTests = new ArrayList(); + + public OutputSummaryFormatter(SeleniumRunnerParameters parms) { + this.parms = parms; + } + + /** + * Create a summary HTML file from the info contained in this log file and + * these suite outputs. + */ + public void format(LogStats log, List suites) { + this.log = log; + this.suites = suites; + this.runStatus = figureOverallStatus(log, suites); + tallyTests(); + + PrintWriter writer = null; + try { + copyCssFile(); + + File outputFile = new File(parms.getOutputDirectory(), + SUMMARY_HTML_FILENAME); + writer = new PrintWriter(outputFile); + + writeHeader(writer); + writeStatsSection(writer); + writeErrorMessagesSection(writer); + writeFailureSection(writer); + writeIgnoreSection(writer); + writeSuitesSection(writer); + writeAllTestsSection(writer); + writeFooter(writer); + } catch (IOException e) { + // There is no appeal for any problems here. Just report them. + e.printStackTrace(); + } finally { + if (writer != null) { + writer.close(); + } + } + } + + /** + * Copy the CSS file into the output directory. + */ + private void copyCssFile() { + File cssSource = parms.getSummaryCssFile(); + File cssTarget = new File(parms.getOutputDirectory(), + SUMMARY_CSS_FILENAME); + try { + FileHelper.copy(cssSource, cssTarget); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * The overall status for the run is the worst status of any component. + */ + private Status figureOverallStatus(LogStats log, List suites) { + if (log.hasErrors()) { + return Status.ERROR; + } + boolean hasWarnings = log.hasWarnings(); + + for (SuiteStats s : suites) { + if (s.getStatus() == Status.ERROR) { + return Status.ERROR; + } else if (s.getStatus() == Status.WARN) { + hasWarnings = true; + } + } + + if (hasWarnings) { + return Status.WARN; + } else { + return Status.OK; + } + } + + private void tallyTests() { + for (SuiteStats s : suites) { + for (TestInfo t : s.getTests()) { + this.allTests.add(t); + if (t.getStatus() == Status.OK) { + this.passingTestCount++; + } else if (t.getStatus() == Status.WARN) { + this.ignoredTests.add(t); + } else { + this.failingTests.add(t); + } + } + } + } + + private void writeHeader(PrintWriter writer) { + String startString = formatDateTime(log.getStartTime()); + + writer.println(""); + writer.println(""); + writer.println(" Summary of Acceptance Tests " + startString + + ""); + writer.println(" "); + writer.println(""); + writer.println(""); + writer.println(); + writer.println("
"); + writer.println(" Acceptance test results: " + startString); + writer.println("
" + this.runStatus + "
"); + writer.println("
"); + } + + private void writeStatsSection(PrintWriter writer) { + String passClass = Status.OK.getHtmlClass(); + String failClass = this.failingTests.isEmpty() ? "" : Status.ERROR + .getHtmlClass(); + String ignoreClass = this.ignoredTests.isEmpty() ? "" : Status.WARN + .getHtmlClass(); + + String start = formatDateTime(log.getStartTime()); + String end = formatDateTime(log.getEndTime()); + String elapsed = formatElapsedTime(log.getElapsedTime()); + + writer.println("
Summary
"); + writer.println(); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println("
"); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println("
Start time:" + start + + "
End time:" + end + + "
Elapsed time" + elapsed + + "
"); + writer.println(" 		"); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println("
Suites" + this.suites.size() + + "
Total tests" + + this.allTests.size() + "
Passing tests" + this.passingTestCount + + "
Failing tests" + this.failingTests.size() + + "
Ignored tests" + this.ignoredTests.size() + + "
"); + writer.println("
"); + writer.println(); + } + + private void writeErrorMessagesSection(PrintWriter writer) { + String errorClass = Status.ERROR.getHtmlClass(); + String warnClass = Status.WARN.getHtmlClass(); + + writer.println("
Errors and warnings
"); + writer.println(); + writer.println(" "); + + if ((!log.hasErrors()) && (!log.hasWarnings())) { + writer + .println(" "); + } else { + for (String e : log.getErrors()) { + writer.println(" "); + } + for (String w : log.getWarnings()) { + writer.println(" "); + } + } + writer.println("
No errors or warnings
ERROR" + e + "
ERROR" + w + "
"); + writer.println(); + } + + private void writeFailureSection(PrintWriter writer) { + String errorClass = Status.ERROR.getHtmlClass(); + + writer.println("
Failing tests
"); + writer.println(); + writer.println(" "); + writer.println(" \n"); + if (failingTests.isEmpty()) { + writer.println(" " + + ""); + } else { + for (TestInfo t : failingTests) { + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + } + } + writer.println("
Suite nameTest name
No tests failed.
" + t.getSuiteName() + "" + t.getTestName() + "
"); + writer.println(); + } + + private void writeIgnoreSection(PrintWriter writer) { + String warnClass = Status.WARN.getHtmlClass(); + + writer.println("
Ignored tests
"); + writer.println(); + writer.println(" "); + writer.println(" " + + "\n"); + if (ignoredTests.isEmpty()) { + writer.println(" " + + ""); + } else { + for (TestInfo t : ignoredTests) { + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + } + } + writer.println("
Suite nameTest nameReason for ignoring
No tests ignored.
" + t.getSuiteName() + "" + t.getTestName() + "" + t.getReasonForIgnoring() + + "
"); + writer.println(); + } + + private void writeSuitesSection(PrintWriter writer) { + writer.println("
Suites
"); + writer.println(); + writer.println(" "); + + for (SuiteStats s : suites) { + writer.println(" "); + writer.println(" "); + writer.println(" "); + } + + writer.println("
" + + s.getName() + "
"); + writer.println(); + } + + private void writeAllTestsSection(PrintWriter writer) { + writer.println("
All tests
"); + writer.println(); + writer.println(" "); + + writer.println(" \n"); + for (TestInfo t : allTests) { + writer.println(" "); + writer.println(" "); + writer.println(" "); + writer.println(" "); + } + + writer.println("
Suite nameTest name
" + t.getSuiteName() + "" + + t.getTestName() + "
"); + writer.println(); + } + + private void writeFooter(PrintWriter writer) { + writer.println("
Log
"); + writer.println(" 
");
+
+		Reader reader = null;
+		try {
+			reader = new FileReader(parms.getLogFile());
+			char[] buffer = new char[4096];
+			int howMany;
+			while (-1 != (howMany = reader.read(buffer))) {
+				writer.write(buffer, 0, howMany);
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			if (reader != null) {
+				try {
+					reader.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		writer.println("
"); + writer.println(""); + writer.println(""); + } + + private String formatElapsedTime(long elapsed) { + long elapsedSeconds = elapsed / 1000L; + long seconds = elapsedSeconds % 60L; + long elapsedMinutes = elapsedSeconds / 60L; + long minutes = elapsedMinutes % 60L; + long hours = elapsedMinutes / 60L; + + String elapsedTime = ""; + if (hours > 0) { + elapsedTime += hours + "h "; + } + if (minutes > 0 || hours > 0) { + elapsedTime += minutes + "m "; + } + elapsedTime += seconds + "s"; + + return elapsedTime; + } + + private String formatDateTime(long dateTime) { + return dateFormat.format(new Date(dateTime)); + } + +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java index 6dcd52c62..b293cc014 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java @@ -19,6 +19,7 @@ public class SeleniumRunner { private final UploadAreaCleaner uploadCleaner; private final ModelCleaner modelCleaner; private final SuiteRunner suiteRunner; + private final OutputManager outputManager; public SeleniumRunner(SeleniumRunnerParameters parms) { this.parms = parms; @@ -26,34 +27,42 @@ public class SeleniumRunner { this.uploadCleaner = new UploadAreaCleaner(parms); this.modelCleaner = new ModelCleaner(parms); this.suiteRunner = new SuiteRunner(parms); + this.outputManager = new OutputManager(parms); } public void runSelectedSuites() { - listener.runStarted(); - for (File suiteDir : parms.getSelectedSuites()) { - listener.suiteStarted(suiteDir); - try { - if (parms.isCleanModel()) { - modelCleaner.clean(); + try { + listener.runStarted(); + outputManager.cleanOutputDirectory(); + for (File suiteDir : parms.getSelectedSuites()) { + listener.suiteStarted(suiteDir); + try { + if (parms.isCleanModel()) { + modelCleaner.clean(); + } + if (parms.isCleanUploads()) { + uploadCleaner.clean(); + } + suiteRunner.runSuite(suiteDir); + } catch (IOException e) { + listener.suiteFailed(suiteDir, e); + } catch (CommandRunnerException e) { + listener.suiteFailed(suiteDir, e); } - if (parms.isCleanUploads()) { - uploadCleaner.clean(); - } - suiteRunner.runSuite(suiteDir); - } catch (IOException e) { - listener.suiteFailed(suiteDir, e); - } catch (CommandRunnerException e) { - listener.suiteFailed(suiteDir, e); - } catch (FatalException e) { - listener.runFailed(e); - e.printStackTrace(); - break; + listener.suiteStopped(suiteDir); } - listener.suiteStopped(suiteDir); + listener.runEndTime(); + outputManager.summarizeOutput(); + } catch (IOException e) { + listener.runFailed(e); + e.printStackTrace(); + } catch (FatalException e) { + listener.runFailed(e); + e.printStackTrace(); } listener.runStopped(); } - + private static void selectAllSuites(SeleniumRunnerParameters parms) { List suites = new ArrayList(); for (File parentDir : parms.getSuiteParentDirectories()) { diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunnerParameters.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunnerParameters.java index 9f0f13eb9..024aad366 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunnerParameters.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunnerParameters.java @@ -27,6 +27,8 @@ public class SeleniumRunnerParameters { public static final String PROP_FIREFOX_PROFILE_PATH = "firefox_profile_template_path"; public static final String PROP_SUITE_TIMEOUT_LIMIT = "suite_timeout_limit"; public static final String PROP_SELENIUM_JAR_PATH = "selenium_jar_path"; + public static final String PROP_IGNORED_TESTS = "ignored_tests_file"; + public static final String PROP_SUMMARY_CSS = "summary_css_file"; public static final String LOGFILE_NAME = "log_file.txt"; @@ -40,6 +42,8 @@ public class SeleniumRunnerParameters { private final File logFile; private final Collection suiteParentDirectories; private final ModelCleanerProperties modelCleanerProperties; + private final IgnoredTests ignoredTests; + private final File summaryCssFile; private Collection selectedSuites = Collections.emptySet(); private boolean cleanModel = true; @@ -70,6 +74,8 @@ public class SeleniumRunnerParameters { this.uploadDirectory = checkReadWriteDirectory(props, PROP_UPLOAD_DIRECTORY); + this.summaryCssFile = checkSummaryCssFile(props); + this.outputDirectory = checkReadWriteDirectory(props, PROP_OUTPUT_DIRECTORY); this.logFile = new File(this.outputDirectory, LOGFILE_NAME); @@ -78,6 +84,14 @@ public class SeleniumRunnerParameters { this.suiteParentDirectories = checkSuiteParentDirectories(props); this.modelCleanerProperties = new ModelCleanerProperties(props); + + // Get the list of ignored tests. + String ignoredFilesPath = getRequiredProperty(props, + PROP_IGNORED_TESTS); + File ignoredFilesFile = new File(ignoredFilesPath); + FileHelper.checkReadableFile(ignoredFilesFile, "File '" + + ignoredFilesPath + "'"); + this.ignoredTests = new IgnoredTests(ignoredFilesFile); } finally { if (propsReader != null) { try { @@ -89,6 +103,17 @@ public class SeleniumRunnerParameters { } } + /** + * The CSS file must be specified, must exist, and must be readable. + */ + private File checkSummaryCssFile(Properties props) { + String summaryCssPath = getRequiredProperty(props, PROP_SUMMARY_CSS); + File cssFile = new File(summaryCssPath); + FileHelper.checkReadableFile(cssFile, "File '" + summaryCssPath + + "'"); + return cssFile; + } + /** * If there is a parameter for this key, it should point to a readable * directory. @@ -120,8 +145,8 @@ public class SeleniumRunnerParameters { } /** - * Check that there is a property for the output directory, and that it - * points to a valid directory. + * Check that there is a property for the required directory path, and that + * it points to a valid directory. */ private File checkReadWriteDirectory(Properties props, String key) { String value = getRequiredProperty(props, key); @@ -265,6 +290,14 @@ public class SeleniumRunnerParameters { return outputDirectory; } + public File getLogFile() { + return logFile; + } + + public File getSummaryCssFile() { + return summaryCssFile; + } + public Collection getSuiteParentDirectories() { return suiteParentDirectories; } @@ -273,6 +306,10 @@ public class SeleniumRunnerParameters { return modelCleanerProperties; } + public IgnoredTests getIgnoredTests() { + return ignoredTests; + } + public void setSelectedSuites(Collection selectedSuites) { this.selectedSuites = selectedSuites; } @@ -307,9 +344,9 @@ public class SeleniumRunnerParameters { + "\n outputDirectory: " + outputDirectory.getPath() + "\n suiteParentDirectories: " + suiteParentDirectories + "\n modelCleanerProperties: " + modelCleanerProperties - + "\n\n selectedSuites: " + showSelectedSuites() - + "\n cleanModel: " + cleanModel + "\n cleanUploads: " - + cleanUploads; + + "\n" + ignoredTests + "\n\n selectedSuites: " + + showSelectedSuites() + "\n cleanModel: " + cleanModel + + "\n cleanUploads: " + cleanUploads; } private String showSelectedSuites() { diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Status.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Status.java new file mode 100644 index 000000000..ce03f3557 --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/Status.java @@ -0,0 +1,33 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +/** + * Status for each test, each suite, and the entire run. + */ +public enum Status { + /** All tests passed, and there were no warnings or errors. */ + OK("good"), + + /** + * Any test failure was ignored, and any messages were no worse than + * warnings. + */ + WARN("fair"), + + /** + * A test failed and could not be ignored, or an error message was + * generated. + */ + ERROR("bad"); + + private final String htmlClass; + + private Status(String htmlClass) { + this.htmlClass = htmlClass; + } + + public String getHtmlClass() { + return this.htmlClass; + } +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteRunner.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteRunner.java index 9f5febf6c..4052412d2 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteRunner.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteRunner.java @@ -37,10 +37,12 @@ public class SuiteRunner { cmd.add("-userExtensions"); cmd.add(parms.getUserExtensionsFile().getPath()); - if (parms.hasFirefoxProfileDir()) { - cmd.add("-firefoxProfileTemplate"); - cmd.add(parms.getFirefoxProfileDir().getPath()); - } + // TODO - figure out why the use of a template means running the test + // twice in simultaneous tabs. + // if (parms.hasFirefoxProfileDir()) { + // cmd.add("-firefoxProfileTemplate"); + // cmd.add(parms.getFirefoxProfileDir().getPath()); + // } String suiteName = suiteDir.getName(); File outputFile = new File(parms.getOutputDirectory(), suiteName diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteStats.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteStats.java new file mode 100644 index 000000000..98642e8d6 --- /dev/null +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SuiteStats.java @@ -0,0 +1,186 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.testrunner; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Extract any summary information from an HTML output file, produced by a test + * suite. + */ +public class SuiteStats { + /** + * If the file doesn't contain a line that includes this pattern, it is not + * a suite output file. + */ + private static final Pattern TITLE_LINE_PATTERN = Pattern + .compile("Test suite results"); + + /** + * A test line looks something like this example: + */ + public static final String EXAMPLE_TEST_LINE = "" + + "
MyTest
"; + + /** + * So here is the pattern to match it: + */ + private static final Pattern TEST_LINE_PATTERN = Pattern + .compile("([^<]*)"); + + /** + * Parse the fields from this file and attempt to produce a + * {@link SuiteStats} object. If this is not an appropriate file, just + * return null. + */ + public static SuiteStats parse(SeleniumRunnerParameters parms, + File outputFile) { + IgnoredTests ignoredTests = parms.getIgnoredTests(); + + boolean isSuiteOutputFile = false; + Status status = Status.ERROR; + + List tests = new ArrayList(); + String suiteName = FileHelper.baseName(outputFile); + String outputLink = outputFile.getName(); + + BufferedReader reader = null; + String line; + try { + reader = new BufferedReader(new FileReader(outputFile)); + while (null != (line = reader.readLine())) { + if (TITLE_LINE_PATTERN.matcher(line).find()) { + isSuiteOutputFile = true; + } + + Matcher m; + m = TEST_LINE_PATTERN.matcher(line); + if (m.matches()) { + String testName = m.group(3); + String testLink = outputLink + m.group(2); + + Status testStatus; + String reasonForIgnoring; + if ("status_passed".equals(m.group(1))) { + testStatus = Status.OK; + reasonForIgnoring = ""; + } else if (ignoredTests.isIgnored(suiteName, testName)) { + testStatus = Status.WARN; + reasonForIgnoring = ignoredTests.getReasonForIgnoring( + suiteName, testName); + } else { + testStatus = Status.ERROR; + reasonForIgnoring = ""; + } + + tests.add(new TestInfo(testName, suiteName, testLink, + testStatus, reasonForIgnoring)); + } + } + + status = Status.OK; + for (TestInfo t : tests) { + if (t.status == Status.ERROR) { + status = Status.ERROR; + } else if ((t.status == Status.WARN) && (status == Status.OK)) { + status = Status.WARN; + } + } + + if (isSuiteOutputFile) { + return new SuiteStats(suiteName, outputLink, tests, status); + } else { + return null; + } + + } catch (IOException e) { + // Can't give up - I need to create as much output as I can. + e.printStackTrace(); + return null; + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + private final String suiteName; + private final String outputLink; + private final List tests; + private final Status status; + + public SuiteStats(String suiteName, String outputLink, + List tests, Status status) { + this.suiteName = suiteName; + this.outputLink = outputLink; + this.tests = tests; + this.status = status; + } + + public String getName() { + return suiteName; + } + + public Status getStatus() { + return status; + } + + public String getOutputLink() { + return outputLink; + } + + public Collection getTests() { + return Collections.unmodifiableCollection(tests); + } + + public static class TestInfo { + private final String name; + private final String suite; + private final String outputLink; + private final Status status; + private final String reasonForIgnoring; + + public TestInfo(String name, String suite, String outputLink, + Status status, String reasonForIgnoring) { + this.name = name; + this.suite = suite; + this.outputLink = outputLink; + this.status = status; + this.reasonForIgnoring = reasonForIgnoring; + } + + public Status getStatus() { + return status; + } + + public String getSuiteName() { + return suite; + } + + public String getTestName() { + return name; + } + + public String getOutputLink() { + return outputLink; + } + + public String getReasonForIgnoring() { + return reasonForIgnoring; + } + } + +} diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/UploadAreaCleaner.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/UploadAreaCleaner.java index 921715b8a..9ddb714f4 100644 --- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/UploadAreaCleaner.java +++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/UploadAreaCleaner.java @@ -3,7 +3,6 @@ package edu.cornell.mannlib.vitro.utilities.testrunner; import java.io.File; -import java.io.FileFilter; import java.io.IOException; /** @@ -34,9 +33,9 @@ public class UploadAreaCleaner { try { for (File file : uploadDirectory.listFiles()) { if (file.isFile()) { - deleteFile(file); + FileHelper.deleteFile(file); } else { - purgeDirectoryRecursively(uploadDirectory); + FileHelper.purgeDirectoryRecursively(file); } } } catch (IOException e) { @@ -47,53 +46,4 @@ public class UploadAreaCleaner { } } - /** - * Delete all of the files in a directory, any sub-directories, and the - * directory itself. - */ - protected static void purgeDirectoryRecursively(File directory) - throws IOException { - File[] files = directory.listFiles(); - for (File file : files) { - if (file.isDirectory()) { - purgeDirectoryRecursively(file); - } else { - deleteFile(file); - } - } - deleteFile(directory); - } - - /** - * Delete a file, either before or after the test. If it can't be deleted, - * complain. - */ - protected static void deleteFile(File file) throws IOException { - if (file.exists()) { - file.delete(); - } - if (!file.exists()) { - return; - } - - /* - * If we were unable to delete the file, is it because it's a non-empty - * directory? - */ - if (!file.isDirectory()) { - final StringBuffer message = new StringBuffer( - "Can't delete directory '" + file.getPath() + "'\n"); - file.listFiles(new FileFilter() { - public boolean accept(File pathname) { - message.append(" contains file '" + pathname + "'\n"); - return true; - } - }); - throw new IOException(message.toString().trim()); - } else { - throw new IOException("Unable to delete file '" + file.getPath() - + "'"); - } - } - }