diff --git a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java
index 6b8640218..8b332a837 100644
--- a/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java
+++ b/utilities/testrunner/src/edu/cornell/mannlib/vitro/utilities/testrunner/SeleniumRunner.java
@@ -53,7 +53,8 @@ public class SeleniumRunner {
 				listener.suiteStopped(suiteDir);
 			}
 			listener.runEndTime();
-			success = true;
+			Status status = outputManager.summarizeOutput();
+			success = (status == Status.OK);
 		} catch (IOException e) {
 			listener.runFailed(e);
 			success = false;