--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import junit.framework.TestCase;
+import dalvik.annotation.AndroidOnly;
+import dalvik.annotation.BrokenTest;
+import dalvik.annotation.KnownFailure;
+import dalvik.annotation.SideEffect;
+
+/**
+ * A dummy test for testing our CoreTestRunner.
+ */
+public class CoreTestDummy extends TestCase {
+
+ @AndroidOnly("")
+ public void testAndroidOnlyPass() {
+ }
+
+ @AndroidOnly("")
+ public void testAndroidOnlyFail() {
+ fail("Oops!");
+ }
+
+ @BrokenTest("")
+ public void testBrokenTestPass() {
+ }
+
+ @BrokenTest("")
+ public void testBrokenTestFail() {
+ fail("Oops!");
+ }
+
+ @KnownFailure("")
+ public void testKnownFailurePass() {
+ }
+
+ @KnownFailure("")
+ public void testKnownFailureFail() {
+ fail("Oops!");
+ }
+
+ @SideEffect("")
+ public void testSideEffectPass() {
+ }
+
+ @SideEffect("")
+ public void testSideEffectFail() {
+ fail("Oops!");
+ }
+
+ public void testNormalPass() {
+ }
+
+ public void testNormalFail() {
+ fail("Oops!");
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import junit.framework.Test;
+import junit.framework.TestFailure;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+import junit.textui.TestRunner;
+
+/**
+ * A minimalistic TestRunner implementation that silently executes a single test
+ * method and writes a possible stack trace to a temporary file. Used for
+ * isolating tests.
+ */
+public class CoreTestIsolator extends TestRunner {
+
+ /**
+ * Creates a new CoreTestIsolator. The superclass needs to be able to build
+ * a proper ResultPrinter, so we pass it a null device for printing stuff.
+ */
+ public CoreTestIsolator() {
+ super(new PrintStream(new OutputStream() {
+ @Override
+ public void write(int oneByte) throws IOException {
+ // Null device
+ }
+ }));
+ }
+
+ @Override
+ protected TestResult createTestResult() {
+ return new TestResult();
+ }
+
+ /**
+ * Provides the main entry point. First and second argument are class and
+ * method name, respectively. Third argument is the temporary file name for
+ * the result. Exits with one of the usual JUnit exit values.
+ */
+ public static void main(String args[]) {
+ Logger.global.setLevel(Level.OFF);
+
+ CoreTestIsolator testRunner = new CoreTestIsolator();
+ try {
+ TestResult r = testRunner.start(args);
+
+ if (!r.wasSuccessful()) {
+ // Store failure or error - we know there must be one
+ Throwable failure = r.failureCount() != 0 ?
+ ((TestFailure)r.failures().nextElement()).
+ thrownException() :
+ ((TestFailure)r.errors().nextElement()).
+ thrownException();
+
+ saveStackTrace(failure, args[2]);
+
+ System.exit(FAILURE_EXIT);
+ } else {
+ // Nothing to see here, please get along
+ System.exit(SUCCESS_EXIT);
+ }
+ } catch(Exception e) {
+ // Let main TestRunner know about execution problem
+ saveStackTrace(e, args[2]);
+ System.exit(EXCEPTION_EXIT);
+ }
+
+ }
+
+ /**
+ * Saves a given stack trace to a given file.
+ */
+ private static void saveStackTrace(Throwable throwable, String fileName) {
+ try {
+ FileOutputStream fos = new FileOutputStream(fileName);
+ ObjectOutputStream oos = new ObjectOutputStream(fos);
+
+ oos.writeObject(throwable);
+
+ oos.flush();
+ oos.close();
+ } catch (IOException ex) {
+ // Ignore
+ }
+ }
+
+ @Override
+ protected TestResult start(String args[]) {
+ try {
+ Test suite = TestSuite.createTest(Class.forName(args[0]), args[1]);
+ return doRun(suite);
+ }
+ catch(Exception e) {
+ throw new RuntimeException("Unable to launch test", e);
+ }
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.io.PrintStream;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import junit.textui.ResultPrinter;
+
+/**
+ * A special ResultPrinter implementation that displays additional statistics
+ * about the test that have been executed.
+ */
+public class CoreTestPrinter extends ResultPrinter {
+
+ /**
+ * The last test class we executed.
+ */
+ private Class<?> fLastClass;
+
+ /**
+ * The current output column for dots.
+ */
+ private int fColumn;
+
+ /**
+ * The time it took to execute the tests.
+ */
+ private int fRunTime;
+
+ /**
+ * The flags the user specified.
+ */
+ private int fFlags;
+
+ /**
+ * Creates a new CoreTestPrinter for the given parameters.
+ */
+ public CoreTestPrinter(PrintStream writer, int flags) {
+ super(writer);
+ fFlags = flags;
+ }
+
+ @Override
+ protected void printHeader(long runTime) {
+ fRunTime = (int)(runTime / 1000);
+
+ if (fColumn != 0) {
+ getWriter().println();
+ }
+
+ getWriter().println();
+ }
+
+ @Override
+ protected void printFooter(TestResult result) {
+ CoreTestResult coreResult = (CoreTestResult)result;
+
+ PrintStream printer = getWriter();
+
+ if (fColumn != 0) {
+ printer.println();
+ }
+
+ printer.println();
+ printer.println("Total tests : " + coreResult.fTotalTestCount);
+ printer.println("Tests run : " + coreResult.runCount());
+ printer.println("Tests ignored : " + coreResult.fIgnoredCount);
+
+ printer.println();
+ printer.println("Normal tests : " + coreResult.fNormalTestCount);
+ printer.println("Android-only : " + coreResult.fAndroidOnlyCount);
+ printer.println("Broken tests : " + coreResult.fBrokenTestCount);
+ printer.println("Known failures: " + coreResult.fKnownFailureCount);
+ printer.println("Side-effects : " + coreResult.fSideEffectCount);
+
+ printMemory();
+
+ int seconds = fRunTime;
+
+ int hours = seconds / 3600;
+ seconds = seconds % 3600;
+
+ int minutes = seconds / 60;
+ seconds = seconds % 60;
+
+ String text = String.format("%02d:%02d:%02d", hours, minutes, seconds);
+
+ printer.println();
+ printer.println("Time taken : " + text);
+
+ super.printFooter(result);
+ }
+
+ /**
+ * Dumps some memory info.
+ */
+ private void printMemory() {
+ PrintStream printer = getWriter();
+ Runtime runtime = Runtime.getRuntime();
+
+ long total = runtime.totalMemory();
+ long free = runtime.freeMemory();
+ long used = total - free;
+
+ printer.println();
+ printer.println("Total memory : " + total);
+ printer.println("Used memory : " + used);
+ printer.println("Free memory : " + free);
+ }
+
+ @Override
+ public void startTest(Test test) {
+ TestCase caze = (TestCase)test;
+
+ if (fLastClass == null ||
+ caze.getClass().getPackage() != fLastClass.getPackage()) {
+
+ if (fColumn != 0) {
+ getWriter().println();
+ fColumn = 0;
+ }
+
+ getWriter().println();
+ Package pack = caze.getClass().getPackage();
+ getWriter().println(pack == null ? "Default package" :
+ pack.getName());
+ getWriter().println();
+
+ }
+
+ if ((fFlags & CoreTestSuite.VERBOSE) != 0) {
+ if (caze.getClass() != fLastClass) {
+ if (fColumn != 0) {
+ getWriter().println();
+ fColumn = 0;
+ }
+
+ String name = caze.getClass().getSimpleName().toString();
+
+ printMemory();
+ getWriter().println("Now executing : " + name);
+ getWriter().println();
+ }
+ }
+
+ getWriter().print(".");
+ if (fColumn++ >= 40) {
+ getWriter().println();
+ fColumn= 0;
+ }
+
+ fLastClass = caze.getClass();
+ }
+
+ @Override
+ public void addError(Test test, Throwable t) {
+ if (t instanceof CoreTestTimeout) {
+ getWriter().print("T");
+ } else {
+ super.addError(test, t);
+ }
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.lang.reflect.Method;
+
+import junit.framework.Protectable;
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import dalvik.annotation.KnownFailure;
+import dalvik.annotation.SideEffect;
+
+/**
+ * A special TestResult implementation that is able to filter out annotated
+ * tests and handles our known failures properly (expects them to fail).
+ * Handy when running the Core Libraries tests on Android, the bare-metal
+ * Dalvik VM, or the RI.
+ */
+public class CoreTestResult extends TestResult {
+
+ /**
+ * The flags the user specified for this test run.
+ */
+ protected int fFlags;
+
+ /**
+ * The timeout the user specified for this test run.
+ */
+ protected int fTimeout;
+
+ /**
+ * The total number of tests in the original suite.
+ */
+ protected int fTotalTestCount;
+
+ /**
+ * The number of Android-only tests in the original suite.
+ */
+ protected int fAndroidOnlyCount;
+
+ /**
+ * The number of broken tests in the original suite.
+ */
+ protected int fBrokenTestCount;
+
+ /**
+ * The number of known failures in the original suite.
+ */
+ protected int fKnownFailureCount;
+
+ /**
+ * The number of side-effective tests in the original suite.
+ */
+ protected int fSideEffectCount;
+
+ /**
+ * The number of normal (non-annotated) tests in the original suite.
+ */
+ protected int fNormalTestCount;
+
+ /**
+ * The number of ignored tests, that is, the number of tests that were
+ * excluded from this suite due to their annotations.
+ */
+ protected int fIgnoredCount;
+
+ /**
+ * Creates a new CoreTestResult with the given flags and timeout.
+ */
+ public CoreTestResult(int flags, int timeout) {
+ super();
+
+ fFlags = flags;
+ fTimeout = timeout;
+ }
+
+ /**
+ * Checks whether the given TestCase method has the given annotation.
+ */
+ @SuppressWarnings("unchecked")
+ boolean hasAnnotation(TestCase test, Class clazz) {
+ try {
+ Method method = test.getClass().getMethod(test.getName());
+ return method.getAnnotation(clazz) != null;
+ } catch (Exception e) {
+ // Ignore
+ }
+
+ return false;
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void runProtected(final Test test, Protectable p) {
+ if ((fFlags & CoreTestSuite.DRY_RUN) == 0) {
+ if (test instanceof TestCase) {
+ TestCase testCase = (TestCase)test;
+
+ // Check whether we need to invert the test result (known failures)
+ boolean invert = hasAnnotation(testCase, KnownFailure.class) &&
+ (fFlags & CoreTestSuite.INVERT_KNOWN_FAILURES) != 0;
+
+ // Check whether we need to isolate the test (side effects)
+ boolean isolate = hasAnnotation(testCase, SideEffect.class) &&
+ (fFlags & CoreTestSuite.ISOLATE_NONE) == 0 ||
+ (fFlags & CoreTestSuite.ISOLATE_ALL) != 0;
+
+ CoreTestRunnable runnable = new CoreTestRunnable(
+ testCase, this, p, invert, isolate);
+
+ if (fTimeout > 0) {
+ Thread thread = new Thread(runnable);
+ thread.start();
+ try {
+ thread.join(fTimeout * 1000);
+ } catch (InterruptedException ex) {
+ // Ignored
+ }
+ if (thread.isAlive()) {
+ runnable.stop();
+ thread.stop();
+ try {
+ thread.join(fTimeout * 1000);
+ } catch (InterruptedException ex) {
+ // Ignored
+ }
+
+ addError(test, new CoreTestTimeout("Test timed out"));
+ }
+ } else {
+ runnable.run();
+ }
+ }
+ }
+ }
+
+ /**
+ * Updates the statistics in this TestResult. Called from the TestSuite,
+ * since, once the original suite has been filtered, we don't actually see
+ * these tests here anymore.
+ */
+ void updateStats(int total, int androidOnly, int broken, int knownFailure,
+ int normal, int ignored, int sideEffect) {
+
+ this.fTotalTestCount += total;
+ this.fAndroidOnlyCount += androidOnly;
+ this.fBrokenTestCount += broken;
+ this.fKnownFailureCount += knownFailure;
+ this.fNormalTestCount += normal;
+ this.fIgnoredCount += ignored;
+ this.fSideEffectCount += sideEffect;
+ }
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.ObjectInputStream;
+
+import junit.framework.AssertionFailedError;
+import junit.framework.Protectable;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import junit.textui.TestRunner;
+
+/**
+ * A wrapper around a single test that allows to execute the test either in the
+ * same thread, in a separate thread, or even in a different process.
+ */
+public class CoreTestRunnable implements Runnable {
+
+ private static boolean IS_DALVIK = "Dalvik".equals(
+ System.getProperty("java.vm.name"));
+
+ /**
+ * The test case we are supposed to run.
+ */
+ private TestCase fTest;
+
+ /**
+ * The TestResult we need to update after the run.
+ */
+ private TestResult fResult;
+
+ /**
+ * The Protectable that JUnit has created for us.
+ */
+ private Protectable fProtectable;
+
+ /**
+ * Reflects whether we need to invert the test result, which is used for
+ * treating known failures.
+ */
+ private boolean fInvert;
+
+ /**
+ * Reflects whether we need to isolate the test, which means we run it in
+ * a separate process.
+ */
+ private boolean fIsolate;
+
+ /**
+ * If we are isolating the test case, this holds the process that is running
+ * it.
+ */
+ private Process fProcess;
+
+ /**
+ * Creates a new CoreTestRunnable for the given parameters.
+ */
+ public CoreTestRunnable(TestCase test, TestResult result,
+ Protectable protectable, boolean invert, boolean isolate) {
+
+ this.fTest = test;
+ this.fProtectable = protectable;
+ this.fResult = result;
+ this.fInvert = invert;
+ this.fIsolate = isolate;
+ }
+
+ /**
+ * Executes the test and stores the results. May be run from a secondary
+ * Thread.
+ */
+ public void run() {
+ try {
+ if (fIsolate) {
+ runExternally();
+ } else {
+ runInternally();
+ }
+
+ if (fInvert) {
+ fInvert = false;
+ throw new AssertionFailedError(
+ "@KnownFailure expected to fail, but succeeded");
+ }
+ } catch (AssertionFailedError e) {
+ if (!fInvert) {
+ fResult.addFailure(fTest, e);
+ }
+ } catch (ThreadDeath e) { // don't catch ThreadDeath by accident
+ throw e;
+ } catch (Throwable e) {
+ if (!fInvert) {
+ fResult.addError(fTest, e);
+ }
+ }
+ }
+
+ /**
+ * Tells the test case to stop. Only used with isolation. We need to kill
+ * the external process in this case.
+ */
+ public void stop() {
+ if (fProcess != null) {
+ fProcess.destroy();
+ }
+ }
+
+ /**
+ * Runs the test case in the same process. This is basically what a
+ * run-of-the-mill JUnit does, except we might also do it in a secondary
+ * thread.
+ */
+ private void runInternally() throws Throwable {
+ fProtectable.protect();
+ }
+
+ /**
+ * Runs the test case in a different process. This is what we do for
+ * isolating test cases that have side effects or do suffer from them.
+ */
+ private void runExternally() throws Throwable {
+ Throwable throwable = null;
+
+ File file = File.createTempFile("isolation", ".tmp");
+
+ fProcess = Runtime.getRuntime().exec(
+ (IS_DALVIK ? "dalvikvm" : "java") +
+ " -classpath " + System.getProperty("java.class.path") +
+ " -Djava.home=" + System.getProperty("java.home") +
+ " -Duser.home=" + System.getProperty("user.home") +
+ " -Djava.io.tmpdir=" + System.getProperty("user.home") +
+ " com.google.coretests.CoreTestIsolator" +
+ " " + fTest.getClass().getName() +
+ " " + fTest.getName() +
+ " " + file.getAbsolutePath());
+
+ int result = fProcess.waitFor();
+
+ if (result != TestRunner.SUCCESS_EXIT) {
+ try {
+ FileInputStream fis = new FileInputStream(file);
+ ObjectInputStream ois = new ObjectInputStream(fis);
+ throwable = (Throwable)ois.readObject();
+ ois.close();
+ } catch (Exception ex) {
+ throwable = new RuntimeException("Error isolating test", ex);
+ }
+ }
+
+ file.delete();
+
+ if (throwable != null) {
+ throw throwable;
+ }
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import junit.framework.Test;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+import junit.textui.ResultPrinter;
+import junit.textui.TestRunner;
+
+/**
+ * A special TestRunner implementation that is able to filter out annotated
+ * tests and handles our known failures properly (expects them to fail).
+ * Handy when running the Core Libraries tests on Android, the bare-metal
+ * Dalvik VM, or the RI.
+ */
+public class CoreTestRunner extends TestRunner {
+
+ /**
+ * Reflects our environment.
+ */
+ private static boolean IS_DALVIK = "Dalvik".equals(
+ System.getProperty("java.vm.name"));
+
+ /**
+ * Defines the default flags for running on Dalvik.
+ */
+ private static final int DEFAULT_FLAGS_DALVIK =
+ CoreTestSuite.RUN_ANDROID_ONLY |
+ CoreTestSuite.RUN_NORMAL_TESTS |
+ CoreTestSuite.RUN_KNOWN_FAILURES |
+ CoreTestSuite.RUN_SIDE_EFFECTS |
+ CoreTestSuite.INVERT_KNOWN_FAILURES;
+
+ /**
+ * Defines the default flags for running on an RI.
+ */
+ private static final int DEFAULT_FLAGS_NON_DALVIK =
+ CoreTestSuite.RUN_NORMAL_TESTS |
+ CoreTestSuite.RUN_KNOWN_FAILURES |
+ CoreTestSuite.RUN_SIDE_EFFECTS;
+
+ /**
+ * Holds the flags specified by the user on the command line.
+ */
+ private int fFlags;
+
+ /**
+ * Holds the timeout value specified by the user on the command line.
+ */
+ private int fTimeout;
+
+ private int fStep = 1;
+
+ /**
+ * Creates a new instance of our CoreTestRunner.
+ */
+ public CoreTestRunner() {
+ super();
+ }
+
+ @Override
+ protected TestResult createTestResult() {
+ return new CoreTestResult(fFlags, fTimeout);
+ }
+
+ protected ResultPrinter createPrinter() {
+ return new CoreTestPrinter(System.out, fFlags);
+ }
+
+ /**
+ * Provides our main entry point.
+ */
+ public static void main(String args[]) {
+ Logger.global.setLevel(Level.OFF);
+
+ System.out.println(
+ "--------------------------------------------------");
+ System.out.println("Android Core Libraries Test Suite");
+ System.out.println("Version 1.0");
+ System.out.println(
+ "Copyright (c) 2009 The Android Open Source Project");
+ System.out.println("");
+
+ CoreTestRunner testRunner = new CoreTestRunner();
+ try {
+ TestResult r = testRunner.start(args);
+
+ System.out.println(
+ "--------------------------------------------------");
+
+ if (!r.wasSuccessful()) {
+ System.exit(FAILURE_EXIT);
+ } else {
+ System.exit(SUCCESS_EXIT);
+ }
+ } catch(Exception e) {
+ System.err.println(e.getMessage());
+ System.exit(EXCEPTION_EXIT);
+ }
+
+ }
+
+ @Override
+ public TestResult doRun(Test suite, boolean wait) {
+ setPrinter(createPrinter());
+
+ /*
+ * Make sure the original suite is unreachable after we have
+ * created the new one, so GC can dispose terminated tests.
+ */
+ suite = new CoreTestSuite(suite, fFlags, fStep, null);
+
+ return super.doRun(suite, wait);
+ }
+
+ /**
+ * Prints a help screen on the console.
+ */
+ private void showHelp() {
+ System.out.println("Usage: run-core-tests {<param>} <test>");
+ System.out.println();
+ System.out.println("Where <test> is a class name, optionally followed");
+ System.out.println("by \"#\" and a method name, and <param> is one of");
+ System.out.println("the following:");
+ System.out.println();
+ System.out.println(" --include-all");
+ System.out.println(" --exclude-all");
+ System.out.println(" --include-android-only");
+ System.out.println(" --exclude-android-only");
+ System.out.println(" --include-broken-tests");
+ System.out.println(" --exclude-broken-tests");
+ System.out.println(" --include-known-failures");
+ System.out.println(" --exclude-known-failures");
+ System.out.println(" --include-normal-tests");
+ System.out.println(" --exclude-normal-tests");
+ System.out.println(" --include-side-effects");
+ System.out.println(" --exclude-side-effects");
+ System.out.println();
+ System.out.println(" --known-failures-must-fail");
+ System.out.println(" --known-failures-must-pass");
+ System.out.println(" --timeout <seconds>");
+ // System.out.println(" --find-side-effect <test>");
+ System.out.println(" --isolate-all");
+ System.out.println(" --isolate-none");
+ System.out.println(" --verbose");
+ System.out.println(" --help");
+ System.out.println();
+ System.out.println("Default parameters are:");
+ System.out.println();
+
+ if (IS_DALVIK) {
+ System.out.println(" --include-android-only");
+ System.out.println(" --exclude-broken-tests");
+ System.out.println(" --include-known-failures");
+ System.out.println(" --include-normal-tests");
+ System.out.println(" --include-side-effects");
+ System.out.println(" --known-failures-must-fail");
+ } else {
+ System.out.println(" --exclude-android-only");
+ System.out.println(" --exclude-broken-tests");
+ System.out.println(" --include-known-failures");
+ System.out.println(" --include-normal-tests");
+ System.out.println(" --include-side-effects");
+ System.out.println(" --known-failures-must-pass");
+ }
+
+ System.out.println();
+ }
+
+ /**
+ * Tries to create a Test instance from a given string. The string might
+ * either specify a class only or a class plus a method name, separated by
+ * a "#".
+ */
+ private Test createTest(String testCase) throws Exception {
+ int p = testCase.indexOf("#");
+ if (p != -1) {
+ String testName = testCase.substring(p + 1);
+ testCase = testCase.substring(0, p);
+
+ return TestSuite.createTest(Class.forName(testCase), testName);
+ } else {
+ return getTest(testCase);
+ }
+
+ }
+
+ @Override
+ protected TestResult start(String args[]) throws Exception {
+ String testName = null;
+ // String victimName = null;
+
+ boolean wait = false;
+
+ if (IS_DALVIK) {
+ fFlags = DEFAULT_FLAGS_DALVIK;
+ } else {
+ fFlags = DEFAULT_FLAGS_NON_DALVIK;
+ }
+
+ for (int i= 0; i < args.length; i++) {
+ if (args[i].startsWith("--")) {
+ if (args[i].equals("--wait")) {
+ wait = true;
+ } else if (args[i].equals("--include-all")) {
+ fFlags = fFlags | CoreTestSuite.RUN_ALL_TESTS;
+ } else if (args[i].equals("--exclude-all")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_ALL_TESTS;
+ } else if (args[i].equals("--include-android-only")) {
+ fFlags = fFlags | CoreTestSuite.RUN_ANDROID_ONLY;
+ } else if (args[i].equals("--exclude-android-only")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_ANDROID_ONLY;
+ } else if (args[i].equals("--include-broken-tests")) {
+ fFlags = fFlags | CoreTestSuite.RUN_BROKEN_TESTS;
+ } else if (args[i].equals("--exclude-broken-tests")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_BROKEN_TESTS;
+ } else if (args[i].equals("--include-known-failures")) {
+ fFlags = fFlags | CoreTestSuite.RUN_KNOWN_FAILURES;
+ } else if (args[i].equals("--exclude-known-failures")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_KNOWN_FAILURES;
+ } else if (args[i].equals("--include-normal-tests")) {
+ fFlags = fFlags | CoreTestSuite.RUN_NORMAL_TESTS;
+ } else if (args[i].equals("--exclude-normal-tests")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_NORMAL_TESTS;
+ } else if (args[i].equals("--include-side-effects")) {
+ fFlags = fFlags | CoreTestSuite.RUN_SIDE_EFFECTS;
+ } else if (args[i].equals("--exclude-side-effects")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_SIDE_EFFECTS;
+ } else if (args[i].equals("--known-failures-must-fail")) {
+ fFlags = fFlags | CoreTestSuite.INVERT_KNOWN_FAILURES;
+ } else if (args[i].equals("--known-failures-must-pass")) {
+ fFlags = fFlags & ~CoreTestSuite.INVERT_KNOWN_FAILURES;
+ } else if (args[i].equals("--timeout")) {
+ fTimeout = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("--reverse")) {
+ fFlags = fFlags | CoreTestSuite.REVERSE;
+ } else if (args[i].equals("--step")) {
+ fStep = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("--isolate-all")) {
+ fFlags = (fFlags | CoreTestSuite.ISOLATE_ALL) &
+ ~CoreTestSuite.ISOLATE_NONE;
+ } else if (args[i].equals("--isolate-none")) {
+ fFlags = (fFlags | CoreTestSuite.ISOLATE_NONE) &
+ ~CoreTestSuite.ISOLATE_ALL;
+ } else if (args[i].equals("--verbose")) {
+ fFlags = fFlags | CoreTestSuite.VERBOSE;
+ // } else if (args[i].equals("--find-side-effect")) {
+ // victimName = args[++i];
+ } else if (args[i].equals("--dry-run")) {
+ fFlags = fFlags | CoreTestSuite.DRY_RUN;
+ } else if (args[i].equals("--help")) {
+ showHelp();
+ System.exit(1);
+ } else {
+ System.err.println("Unknown argument " + args[i] +
+ ", try --help");
+ System.exit(1);
+ }
+ } else {
+ testName = args[i];
+ }
+ }
+
+ if (IS_DALVIK) {
+ System.out.println("Using Dalvik VM version " +
+ System.getProperty("java.vm.version"));
+ } else {
+ System.out.println("Using Java VM version " +
+ System.getProperty("java.version"));
+ }
+ System.out.println();
+
+ try {
+ return doRun(createTest(testName), wait);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ throw new Exception("Could not create and run test suite: " + e);
+ }
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Enumeration;
+import java.util.Vector;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestFailure;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+import dalvik.annotation.AndroidOnly;
+import dalvik.annotation.BrokenTest;
+import dalvik.annotation.KnownFailure;
+import dalvik.annotation.SideEffect;
+
+/**
+ * A special TestSuite implementation that flattens the hierarchy of a given
+ * JUnit TestSuite and removes tests after executing them. This is so the core
+ * tests actually have a chance to succeed, since they do consume quite some
+ * memory and many tests do not (have a chance to) cleanup properly after
+ * themselves. The class also implements our filtering mechanism for tests, so
+ * it becomes easy to only include or exclude tests based on their annotations
+ * (like, say, execute all Android-only tests that are not known to be broken).
+ */
+public class CoreTestSuite implements Test {
+
+ /**
+ * Include all normal tests in the suite.
+ */
+ public static final int RUN_NORMAL_TESTS = 1;
+
+ /**
+ * Include all broken tests in the suite.
+ */
+ public static final int RUN_BROKEN_TESTS = 2;
+
+ /**
+ * Include all known failures in the suite.
+ */
+ public static final int RUN_KNOWN_FAILURES = 4;
+
+ /**
+ * Include all Android-only tests in the suite.
+ */
+ public static final int RUN_ANDROID_ONLY = 8;
+
+ /**
+ * Include side-effective tests in the suite.
+ */
+ public static final int RUN_SIDE_EFFECTS = 16;
+
+ /**
+ * Include all tests in the suite.
+ */
+ public static final int RUN_ALL_TESTS =
+ RUN_NORMAL_TESTS | RUN_BROKEN_TESTS |
+ RUN_KNOWN_FAILURES | RUN_SIDE_EFFECTS | RUN_ANDROID_ONLY;
+
+ /**
+ * Special treatment for known failures: they are expected to fail, so we
+ * throw an Exception if they succeed and accept them failing.
+ */
+ public static final int INVERT_KNOWN_FAILURES = 32;
+
+ /**
+ * Run each test in its own VM.
+ */
+ public static final int ISOLATE_ALL = 64;
+
+ /**
+ * Run no test in its own VM.
+ */
+ public static final int ISOLATE_NONE = 128;
+
+ /**
+ * Be verbose.
+ */
+ public static final int VERBOSE = 256;
+
+ public static final int REVERSE = 512;
+
+ public static final int DRY_RUN = 1024;
+
+ /**
+ * The total number of tests in the original suite.
+ */
+ protected int fTotalCount;
+
+ /**
+ * The number of Android-only tests in the original suite.
+ */
+ protected int fAndroidOnlyCount;
+
+ /**
+ * The number of broken tests in the original suite.
+ */
+ protected int fBrokenCount;
+
+ /**
+ * The number of known failures in the original suite.
+ */
+ protected int fKnownFailureCount;
+
+ /**
+ * The number of side-effective tests in the original suite.
+ */
+ protected int fSideEffectCount;
+
+ /**
+ * The number of normal (non-annotated) tests in the original suite.
+ */
+ protected int fNormalCount;
+
+ /**
+ * The number of ignored tests, that is, the number of tests that were
+ * excluded from this suite due to their annotations.
+ */
+ protected int fIgnoredCount;
+
+ /**
+ * Contains the actual test cases in a reverse-ordered, flat list.
+ */
+ private Vector<Test> fTests = new Vector<Test>();
+
+ private TestCase fVictim;
+
+ private int fStep;
+
+ private int fFlags;
+
+ /**
+ * Creates a new CoreTestSuite for the given ordinary JUnit Test (which may
+ * be a TestCase or TestSuite). The CoreTestSuite will be a flattened and
+ * potentially filtered subset of the original JUnit Test. The flags
+ * determine the way we filter.
+ */
+ public CoreTestSuite(Test suite, int flags, int step, TestCase victim) {
+ super();
+
+ fStep = step;
+ addAndFlatten(suite, flags);
+ fVictim = victim;
+ fFlags = flags;
+ }
+
+ /**
+ * Adds the given ordinary JUnit Test (which may be a TestCase or TestSuite)
+ * to this CoreTestSuite. Note we are storing the tests in reverse order,
+ * so it's easier to remove a finished test from the end of the list.
+ */
+ private void addAndFlatten(Test test, int flags) {
+ if (test instanceof TestSuite) {
+ TestSuite suite = (TestSuite)test;
+
+ if ((flags & REVERSE) != 0) {
+ for (int i = suite.testCount() - 1; i >= 0; i--) {
+ addAndFlatten(suite.testAt(i), flags);
+ }
+ } else {
+ for (int i = 0; i < suite.testCount(); i++) {
+ addAndFlatten(suite.testAt(i), flags);
+ }
+ }
+ } else if (test instanceof TestCase) {
+ TestCase caze = (TestCase)test;
+ boolean ignoreMe = false;
+
+ boolean isAndroidOnly = hasAnnotation(caze,
+ AndroidOnly.class);
+ boolean isBrokenTest = hasAnnotation(caze,
+ BrokenTest.class);
+ boolean isKnownFailure = hasAnnotation(caze,
+ KnownFailure.class);
+ boolean isSideEffect = hasAnnotation(caze,
+ SideEffect.class);
+ boolean isNormalTest =
+ !(isAndroidOnly || isBrokenTest || isKnownFailure ||
+ isSideEffect);
+
+ if (isAndroidOnly) {
+ fAndroidOnlyCount++;
+ }
+
+ if (isBrokenTest) {
+ fBrokenCount++;
+ }
+
+ if (isKnownFailure) {
+ fKnownFailureCount++;
+ }
+
+ if (isNormalTest) {
+ fNormalCount++;
+ }
+
+ if (isSideEffect) {
+ fSideEffectCount++;
+ }
+
+ if ((flags & RUN_ANDROID_ONLY) == 0 && isAndroidOnly) {
+ ignoreMe = true;
+ }
+
+ if ((flags & RUN_BROKEN_TESTS) == 0 && isBrokenTest) {
+ ignoreMe = true;
+ }
+
+ if ((flags & RUN_KNOWN_FAILURES) == 0 && isKnownFailure) {
+ ignoreMe = true;
+ }
+
+ if (((flags & RUN_NORMAL_TESTS) == 0) && isNormalTest) {
+ ignoreMe = true;
+ }
+
+ if (((flags & RUN_SIDE_EFFECTS) == 0) && isSideEffect) {
+ ignoreMe = true;
+ }
+
+ this.fTotalCount++;
+
+ if (!ignoreMe) {
+ fTests.add(test);
+ } else {
+ this.fIgnoredCount++;
+ }
+ } else {
+ System.out.println("Warning: Don't know how to handle " +
+ test.getClass().getName() + " " + test.toString());
+ }
+ }
+
+ /**
+ * Checks whether the given TestCase class has the given annotation.
+ */
+ @SuppressWarnings("unchecked")
+ private boolean hasAnnotation(TestCase test, Class clazz) {
+ try {
+ Method method = test.getClass().getMethod(test.getName());
+ return method.getAnnotation(clazz) != null;
+ } catch (Exception e) {
+ // Ignore
+ }
+
+ return false;
+ }
+
+ /**
+ * Runs the tests and collects their result in a TestResult.
+ */
+ public void run(TestResult result) {
+ // Run tests
+ int i = 0;
+
+ while (fTests.size() != 0 && !result.shouldStop()) {
+ TestCase test = (TestCase)fTests.elementAt(i);
+
+ Thread.currentThread().setContextClassLoader(
+ test.getClass().getClassLoader());
+
+ test.run(result);
+
+ /*
+ if (fVictim != null) {
+ TestResult dummy = fVictim.run();
+
+ if (dummy.failureCount() != 0) {
+ result.addError(fTests.elementAt(i), new RuntimeException(
+ "Probable side effect",
+ ((TestFailure)dummy.failures().nextElement()).
+ thrownException()));
+ } else if (dummy.errorCount() != 0) {
+ result.addError(fTests.elementAt(i), new RuntimeException(
+ "Probable side effect",
+ ((TestFailure)dummy.errors().nextElement()).
+ thrownException()));
+ }
+ }
+ */
+
+ fTests.remove(i);
+
+ if (fTests.size() != 0) {
+ i = (i + fStep - 1) % fTests.size();
+ }
+
+ }
+
+ // Forward overall stats to TestResult, so ResultPrinter can see it.
+ if (result instanceof CoreTestResult) {
+ ((CoreTestResult)result).updateStats(
+ fTotalCount, fAndroidOnlyCount, fBrokenCount,
+ fKnownFailureCount, fNormalCount, fIgnoredCount,
+ fSideEffectCount);
+ }
+ }
+
+ /**
+ * Nulls all reference fields in the given test object. This method helps
+ * us with those test classes that don't have an explicit tearDown()
+ * method. Normally the garbage collector should take care of everything,
+ * but it can't hurt to support it a bit.
+ */
+ private void cleanup(TestCase test) {
+ Field[] fields = test.getClass().getDeclaredFields();
+ for (int i = 0; i < fields.length; i++) {
+ Field f = fields[i];
+ if (!f.getType().isPrimitive() &&
+ (f.getModifiers() & Modifier.STATIC) == 0) {
+ try {
+ f.setAccessible(true);
+ f.set(test, null);
+ } catch (Exception ex) {
+ // Nothing we can do about it.
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the tests as an enumeration. Note this is empty once the tests
+ * have been executed.
+ */
+ @SuppressWarnings("unchecked")
+ public Enumeration tests() {
+ return fTests.elements();
+ }
+
+ /**
+ * Returns the number of tests in the suite. Note this is zero once the
+ * tests have been executed.
+ */
+ public int countTestCases() {
+ return fTests.size();
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+/**
+ * A special exception for timeouts during tests. The CoreResultPrinter knows
+ * how to handle this.
+ */
+@SuppressWarnings("serial")
+public class CoreTestTimeout extends RuntimeException {
+
+ /**
+ * Creates a new instance with the given message.
+ */
+ public CoreTestTimeout(String message) {
+ super(message);
+ }
+
+}
}
} else {
System.out.println("Running selected tests...");
- TestRunner.main(args);
+ CoreTestRunner.main(args);
}
Runtime.getRuntime().halt(0);