tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java

   1 /*
   2  * Copyright (C) 2019 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package com.android.tests.rollback.host;
  18
  19 import static com.android.tests.rollback.host.WatchdogEventLogger.watchdogEventOccurred;
  20
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertNull;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25 import static org.junit.Assume.assumeTrue;
  26
  27 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
  28 import com.android.tradefed.device.DeviceNotAvailableException;
  29 import com.android.tradefed.device.IFileEntry;
  30 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
  31 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test;
  32 import com.android.tradefed.util.CommandResult;
  33 import com.android.tradefed.util.CommandStatus;
  34
  35 import org.junit.After;
  36 import org.junit.Before;
  37 import org.junit.Test;
  38 import org.junit.runner.RunWith;
  39
  40 import java.io.File;
  41 import java.time.Instant;
  42 import java.util.Collections;
  43 import java.util.Date;
  44 import java.util.List;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.stream.Collectors;
  47
  48 /**
  49  * Runs the staged rollback tests.
  50  *
  51  * TODO(gavincorkery): Support the verification of logging parents in Watchdog metrics.
  52  */
  53 @RunWith(DeviceJUnit4ClassRunner.class)
  54 public class StagedRollbackTest extends BaseHostJUnit4Test {
  55     private static final int NATIVE_CRASHES_THRESHOLD = 5;
  56
  57     /**
  58      * Runs the given phase of a test by calling into the device.
  59      * Throws an exception if the test phase fails.
  60      * <p>
  61      * For example, <code>runPhase("testApkOnlyEnableRollback");</code>
  62      */
  63     private void runPhase(String phase) throws Exception {
  64         assertTrue(runDeviceTests("com.android.tests.rollback",
  65                     "com.android.tests.rollback.StagedRollbackTest",
  66                     phase));
  67     }
  68
  69     private static final String APK_IN_APEX_TESTAPEX_NAME = "com.android.apex.apkrollback.test";
  70     private static final String TESTAPP_A = "com.android.cts.install.lib.testapp.A";
  71
  72     private static final String TEST_SUBDIR = "/subdir/";
  73
  74     private static final String TEST_FILENAME_1 = "test_file.txt";
  75     private static final String TEST_STRING_1 = "hello this is a test";
  76     private static final String TEST_FILENAME_2 = "another_file.txt";
  77     private static final String TEST_STRING_2 = "this is a different file";
  78     private static final String TEST_FILENAME_3 = "also.xyz";
  79     private static final String TEST_STRING_3 = "also\n a\n test\n string";
  80     private static final String TEST_FILENAME_4 = "one_more.test";
  81     private static final String TEST_STRING_4 = "once more unto the test";
  82
  83     private static final String REASON_APP_CRASH = "REASON_APP_CRASH";
  84     private static final String REASON_NATIVE_CRASH = "REASON_NATIVE_CRASH";
  85
  86     private static final String ROLLBACK_INITIATE = "ROLLBACK_INITIATE";
  87     private static final String ROLLBACK_BOOT_TRIGGERED = "ROLLBACK_BOOT_TRIGGERED";
  88     private static final String ROLLBACK_SUCCESS = "ROLLBACK_SUCCESS";
  89
  90     private WatchdogEventLogger mLogger = new WatchdogEventLogger();
  91
  92     @Before
  93     public void setUp() throws Exception {
  94         deleteFiles("/system/apex/" + APK_IN_APEX_TESTAPEX_NAME + "*.apex",
  95                 "/data/apex/active/" + APK_IN_APEX_TESTAPEX_NAME + "*.apex");
  96         runPhase("testCleanUp");
  97         mLogger.start(getDevice());
  98     }
  99
 100     @After
 101     public void tearDown() throws Exception {
 102         mLogger.stop();
 103         runPhase("testCleanUp");
 104         deleteFiles("/system/apex/" + APK_IN_APEX_TESTAPEX_NAME + "*.apex",
 105                 "/data/apex/active/" + APK_IN_APEX_TESTAPEX_NAME + "*.apex",
 106                 apexDataDirDeSys(APK_IN_APEX_TESTAPEX_NAME) + "*",
 107                 apexDataDirCe(APK_IN_APEX_TESTAPEX_NAME, 0) + "*");
 108     }
 109
 110     /**
 111      * Deletes files and reboots the device if necessary.
 112      * @param files the paths of files which might contain wildcards
 113      */
 114     private void deleteFiles(String... files) throws Exception {
 115         boolean found = false;
 116         for (String file : files) {
 117             CommandResult result = getDevice().executeShellV2Command("ls " + file);
 118             if (result.getStatus() == CommandStatus.SUCCESS) {
 119                 found = true;
 120                 break;
 121             }
 122         }
 123
 124         if (found) {
 125             if (!getDevice().isAdbRoot()) {
 126                 getDevice().enableAdbRoot();
 127             }
 128             getDevice().remountSystemWritable();
 129             for (String file : files) {
 130                 getDevice().executeShellCommand("rm -rf " + file);
 131             }
 132             getDevice().reboot();
 133         }
 134     }
 135
 136     /**
 137      * Tests watchdog triggered staged rollbacks involving only apks.
 138      */
 139     @Test
 140     public void testBadApkOnly() throws Exception {
 141         runPhase("testBadApkOnly_Phase1");
 142         getDevice().reboot();
 143         runPhase("testBadApkOnly_Phase2");
 144
 145         // Trigger rollback and wait for reboot to happen
 146         runPhase("testBadApkOnly_Phase3");
 147         assertTrue(getDevice().waitForDeviceNotAvailable(TimeUnit.MINUTES.toMillis(2)));
 148
 149         getDevice().waitForDeviceAvailable();
 150
 151         runPhase("testBadApkOnly_Phase4");
 152
 153         List<String> watchdogEvents = mLogger.getWatchdogLoggingEvents();
 154         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_INITIATE, null,
 155                 REASON_APP_CRASH, TESTAPP_A));
 156         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_BOOT_TRIGGERED, null,
 157                 null, null));
 158         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_SUCCESS, null, null, null));
 159     }
 160
 161     @Test
 162     public void testNativeWatchdogTriggersRollback() throws Exception {
 163         runPhase("testNativeWatchdogTriggersRollback_Phase1");
 164
 165         // Reboot device to activate staged package
 166         getDevice().reboot();
 167
 168         runPhase("testNativeWatchdogTriggersRollback_Phase2");
 169
 170         // crash system_server enough times to trigger a rollback
 171         crashProcess("system_server", NATIVE_CRASHES_THRESHOLD);
 172
 173         // Rollback should be committed automatically now.
 174         // Give time for rollback to be committed. This could take a while,
 175         // because we need all of the following to happen:
 176         // 1. system_server comes back up and boot completes.
 177         // 2. Rollback health observer detects updatable crashing signal.
 178         // 3. Staged rollback session becomes ready.
 179         // 4. Device actually reboots.
 180         // So we give a generous timeout here.
 181         assertTrue(getDevice().waitForDeviceNotAvailable(TimeUnit.MINUTES.toMillis(5)));
 182         getDevice().waitForDeviceAvailable();
 183
 184         // verify rollback committed
 185         runPhase("testNativeWatchdogTriggersRollback_Phase3");
 186
 187         List<String> watchdogEvents = mLogger.getWatchdogLoggingEvents();
 188         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_INITIATE, null,
 189                         REASON_NATIVE_CRASH, null));
 190         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_BOOT_TRIGGERED, null,
 191                 null, null));
 192         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_SUCCESS, null, null, null));
 193     }
 194
 195     @Test
 196     public void testNativeWatchdogTriggersRollbackForAll() throws Exception {
 197         // This test requires committing multiple staged rollbacks
 198         assumeTrue(isCheckpointSupported());
 199
 200         // Install a package with rollback enabled.
 201         runPhase("testNativeWatchdogTriggersRollbackForAll_Phase1");
 202         getDevice().reboot();
 203
 204         // Once previous staged install is applied, install another package
 205         runPhase("testNativeWatchdogTriggersRollbackForAll_Phase2");
 206         getDevice().reboot();
 207
 208         // Verify the new staged install has also been applied successfully.
 209         runPhase("testNativeWatchdogTriggersRollbackForAll_Phase3");
 210
 211         // crash system_server enough times to trigger a rollback
 212         crashProcess("system_server", NATIVE_CRASHES_THRESHOLD);
 213
 214         // Rollback should be committed automatically now.
 215         // Give time for rollback to be committed. This could take a while,
 216         // because we need all of the following to happen:
 217         // 1. system_server comes back up and boot completes.
 218         // 2. Rollback health observer detects updatable crashing signal.
 219         // 3. Staged rollback session becomes ready.
 220         // 4. Device actually reboots.
 221         // So we give a generous timeout here.
 222         assertTrue(getDevice().waitForDeviceNotAvailable(TimeUnit.MINUTES.toMillis(5)));
 223         getDevice().waitForDeviceAvailable();
 224
 225         // verify all available rollbacks have been committed
 226         runPhase("testNativeWatchdogTriggersRollbackForAll_Phase4");
 227
 228         List<String> watchdogEvents = mLogger.getWatchdogLoggingEvents();
 229         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_INITIATE, null,
 230                         REASON_NATIVE_CRASH, null));
 231         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_BOOT_TRIGGERED, null,
 232                 null, null));
 233         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_SUCCESS, null, null, null));
 234     }
 235
 236     /**
 237      * Tests rolling back user data where there are multiple rollbacks for that package.
 238      */
 239     @Test
 240     public void testPreviouslyAbandonedRollbacks() throws Exception {
 241         runPhase("testPreviouslyAbandonedRollbacks_Phase1");
 242         getDevice().reboot();
 243         runPhase("testPreviouslyAbandonedRollbacks_Phase2");
 244         getDevice().reboot();
 245         runPhase("testPreviouslyAbandonedRollbacks_Phase3");
 246     }
 247
 248     /**
 249      * Tests we can enable rollback for a whitelisted app.
 250      */
 251     @Test
 252     public void testRollbackWhitelistedApp() throws Exception {
 253         assumeTrue(hasMainlineModule());
 254         runPhase("testRollbackWhitelistedApp_Phase1");
 255         getDevice().reboot();
 256         runPhase("testRollbackWhitelistedApp_Phase2");
 257     }
 258
 259     @Test
 260     public void testRollbackDataPolicy() throws Exception {
 261         runPhase("testRollbackDataPolicy_Phase1");
 262         getDevice().reboot();
 263         runPhase("testRollbackDataPolicy_Phase2");
 264         getDevice().reboot();
 265         runPhase("testRollbackDataPolicy_Phase3");
 266     }
 267
 268     /**
 269      * Tests that userdata of apk-in-apex is restored when apex is rolled back.
 270      */
 271     @Test
 272     public void testRollbackApexWithApk() throws Exception {
 273         getDevice().uninstallPackage("com.android.cts.install.lib.testapp.A");
 274         pushTestApex();
 275         runPhase("testRollbackApexWithApk_Phase1");
 276         getDevice().reboot();
 277         runPhase("testRollbackApexWithApk_Phase2");
 278         getDevice().reboot();
 279         runPhase("testRollbackApexWithApk_Phase3");
 280     }
 281
 282     /**
 283      * Tests that RollbackPackageHealthObserver is observing apk-in-apex.
 284      */
 285     @Test
 286     public void testRollbackApexWithApkCrashing() throws Exception {
 287         getDevice().uninstallPackage("com.android.cts.install.lib.testapp.A");
 288         pushTestApex();
 289
 290         // Install an apex with apk that crashes
 291         runPhase("testRollbackApexWithApkCrashing_Phase1");
 292         getDevice().reboot();
 293         // Verify apex was installed and then crash the apk
 294         runPhase("testRollbackApexWithApkCrashing_Phase2");
 295         // Wait for crash to trigger rollback
 296         assertTrue(getDevice().waitForDeviceNotAvailable(TimeUnit.MINUTES.toMillis(5)));
 297         getDevice().waitForDeviceAvailable();
 298         // Verify rollback occurred due to crash of apk-in-apex
 299         runPhase("testRollbackApexWithApkCrashing_Phase3");
 300
 301         List<String> watchdogEvents = mLogger.getWatchdogLoggingEvents();
 302         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_INITIATE, null,
 303                 REASON_APP_CRASH, TESTAPP_A));
 304         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_BOOT_TRIGGERED, null,
 305                 null, null));
 306         assertTrue(watchdogEventOccurred(watchdogEvents, ROLLBACK_SUCCESS, null, null, null));
 307     }
 308
 309     /**
 310      * Tests that data in DE_sys apex data directory is restored when apex is rolled back.
 311      */
 312     @Test
 313     public void testRollbackApexDataDirectories_DeSys() throws Exception {
 314         List<String> before = getSnapshotDirectories("/data/misc/apexrollback");
 315         pushTestApex();
 316
 317         // Push files to apex data directory
 318         String oldFilePath1 = apexDataDirDeSys(APK_IN_APEX_TESTAPEX_NAME) + "/" + TEST_FILENAME_1;
 319         String oldFilePath2 =
 320                 apexDataDirDeSys(APK_IN_APEX_TESTAPEX_NAME) + TEST_SUBDIR + TEST_FILENAME_2;
 321         assertTrue(getDevice().pushString(TEST_STRING_1, oldFilePath1));
 322         assertTrue(getDevice().pushString(TEST_STRING_2, oldFilePath2));
 323
 324         // Install new version of the APEX with rollback enabled
 325         runPhase("testRollbackApexDataDirectories_Phase1");
 326         getDevice().reboot();
 327
 328         // Replace files in data directory
 329         getDevice().deleteFile(oldFilePath1);
 330         getDevice().deleteFile(oldFilePath2);
 331         String newFilePath3 = apexDataDirDeSys(APK_IN_APEX_TESTAPEX_NAME) + "/" + TEST_FILENAME_3;
 332         String newFilePath4 =
 333                 apexDataDirDeSys(APK_IN_APEX_TESTAPEX_NAME) + TEST_SUBDIR + TEST_FILENAME_4;
 334         assertTrue(getDevice().pushString(TEST_STRING_3, newFilePath3));
 335         assertTrue(getDevice().pushString(TEST_STRING_4, newFilePath4));
 336
 337         // Roll back the APEX
 338         runPhase("testRollbackApexDataDirectories_Phase2");
 339         getDevice().reboot();
 340
 341         // Verify that old files have been restored and new files are gone
 342         assertEquals(TEST_STRING_1, getDevice().pullFileContents(oldFilePath1));
 343         assertEquals(TEST_STRING_2, getDevice().pullFileContents(oldFilePath2));
 344         assertNull(getDevice().pullFile(newFilePath3));
 345         assertNull(getDevice().pullFile(newFilePath4));
 346
 347         // Verify snapshots are deleted after restoration
 348         List<String> after = getSnapshotDirectories("/data/misc/apexrollback");
 349         // Only check directories newly created during the test
 350         after.removeAll(before);
 351         after.forEach(dir -> assertDirectoryIsEmpty(dir));
 352     }
 353
 354     /**
 355      * Tests that data in DE (user) apex data directory is restored when apex is rolled back.
 356      */
 357     @Test
 358     public void testRollbackApexDataDirectories_DeUser() throws Exception {
 359         List<String> before = getSnapshotDirectories("/data/misc_de/0/apexrollback");
 360         pushTestApex();
 361
 362         // Push files to apex data directory
 363         String oldFilePath1 = apexDataDirDeUser(
 364                 APK_IN_APEX_TESTAPEX_NAME, 0) + "/" + TEST_FILENAME_1;
 365         String oldFilePath2 =
 366                 apexDataDirDeUser(APK_IN_APEX_TESTAPEX_NAME, 0) + TEST_SUBDIR + TEST_FILENAME_2;
 367         assertTrue(getDevice().pushString(TEST_STRING_1, oldFilePath1));
 368         assertTrue(getDevice().pushString(TEST_STRING_2, oldFilePath2));
 369
 370         // Install new version of the APEX with rollback enabled
 371         runPhase("testRollbackApexDataDirectories_Phase1");
 372         getDevice().reboot();
 373
 374         // Replace files in data directory
 375         getDevice().deleteFile(oldFilePath1);
 376         getDevice().deleteFile(oldFilePath2);
 377         String newFilePath3 =
 378                 apexDataDirDeUser(APK_IN_APEX_TESTAPEX_NAME, 0) + "/" + TEST_FILENAME_3;
 379         String newFilePath4 =
 380                 apexDataDirDeUser(APK_IN_APEX_TESTAPEX_NAME, 0) + TEST_SUBDIR + TEST_FILENAME_4;
 381         assertTrue(getDevice().pushString(TEST_STRING_3, newFilePath3));
 382         assertTrue(getDevice().pushString(TEST_STRING_4, newFilePath4));
 383
 384         // Roll back the APEX
 385         runPhase("testRollbackApexDataDirectories_Phase2");
 386         getDevice().reboot();
 387
 388         // Verify that old files have been restored and new files are gone
 389         assertEquals(TEST_STRING_1, getDevice().pullFileContents(oldFilePath1));
 390         assertEquals(TEST_STRING_2, getDevice().pullFileContents(oldFilePath2));
 391         assertNull(getDevice().pullFile(newFilePath3));
 392         assertNull(getDevice().pullFile(newFilePath4));
 393
 394         // Verify snapshots are deleted after restoration
 395         List<String> after = getSnapshotDirectories("/data/misc_de/0/apexrollback");
 396         // Only check directories newly created during the test
 397         after.removeAll(before);
 398         after.forEach(dir -> assertDirectoryIsEmpty(dir));
 399     }
 400
 401     /**
 402      * Tests that data in CE apex data directory is restored when apex is rolled back.
 403      */
 404     @Test
 405     public void testRollbackApexDataDirectories_Ce() throws Exception {
 406         List<String> before = getSnapshotDirectories("/data/misc_ce/0/apexrollback");
 407         pushTestApex();
 408
 409         // Push files to apex data directory
 410         String oldFilePath1 = apexDataDirCe(APK_IN_APEX_TESTAPEX_NAME, 0) + "/" + TEST_FILENAME_1;
 411         String oldFilePath2 =
 412                 apexDataDirCe(APK_IN_APEX_TESTAPEX_NAME, 0) + TEST_SUBDIR + TEST_FILENAME_2;
 413         assertTrue(getDevice().pushString(TEST_STRING_1, oldFilePath1));
 414         assertTrue(getDevice().pushString(TEST_STRING_2, oldFilePath2));
 415
 416         // Install new version of the APEX with rollback enabled
 417         runPhase("testRollbackApexDataDirectories_Phase1");
 418         getDevice().reboot();
 419
 420         // Replace files in data directory
 421         getDevice().deleteFile(oldFilePath1);
 422         getDevice().deleteFile(oldFilePath2);
 423         String newFilePath3 = apexDataDirCe(APK_IN_APEX_TESTAPEX_NAME, 0) + "/" + TEST_FILENAME_3;
 424         String newFilePath4 =
 425                 apexDataDirCe(APK_IN_APEX_TESTAPEX_NAME, 0) + TEST_SUBDIR + TEST_FILENAME_4;
 426         assertTrue(getDevice().pushString(TEST_STRING_3, newFilePath3));
 427         assertTrue(getDevice().pushString(TEST_STRING_4, newFilePath4));
 428
 429         // Roll back the APEX
 430         runPhase("testRollbackApexDataDirectories_Phase2");
 431         getDevice().reboot();
 432
 433         // Verify that old files have been restored and new files are gone
 434         assertEquals(TEST_STRING_1, getDevice().pullFileContents(oldFilePath1));
 435         assertEquals(TEST_STRING_2, getDevice().pullFileContents(oldFilePath2));
 436         assertNull(getDevice().pullFile(newFilePath3));
 437         assertNull(getDevice().pullFile(newFilePath4));
 438
 439         // Verify snapshots are deleted after restoration
 440         List<String> after = getSnapshotDirectories("/data/misc_ce/0/apexrollback");
 441         // Only check directories newly created during the test
 442         after.removeAll(before);
 443         after.forEach(dir -> assertDirectoryIsEmpty(dir));
 444     }
 445
 446     /**
 447      * Tests an available rollback shouldn't be deleted when its session expires.
 448      */
 449     @Test
 450     public void testExpireSession() throws Exception {
 451         runPhase("testExpireSession_Phase1_Install");
 452         getDevice().reboot();
 453         runPhase("testExpireSession_Phase2_VerifyInstall");
 454
 455         // Advance system clock by 7 days to expire the staged session
 456         Instant t1 = Instant.ofEpochMilli(getDevice().getDeviceDate());
 457         Instant t2 = t1.plusMillis(TimeUnit.DAYS.toMillis(7));
 458         runAsRoot(() -> getDevice().setDate(Date.from(t2)));
 459
 460         // Somehow we need to wait for a while before reboot. Otherwise the change to the
 461         // system clock will be reset after reboot.
 462         Thread.sleep(3000);
 463         getDevice().reboot();
 464         runPhase("testExpireSession_Phase3_VerifyRollback");
 465     }
 466
 467     private void pushTestApex() throws Exception {
 468         CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(getBuild());
 469         final String fileName = APK_IN_APEX_TESTAPEX_NAME + "_v1.apex";
 470         final File apex = buildHelper.getTestFile(fileName);
 471         if (!getDevice().isAdbRoot()) {
 472             getDevice().enableAdbRoot();
 473         }
 474         getDevice().remountSystemWritable();
 475         assertTrue(getDevice().pushFile(apex, "/system/apex/" + fileName));
 476         getDevice().reboot();
 477     }
 478
 479     private static String apexDataDirDeSys(String apexName) {
 480         return String.format("/data/misc/apexdata/%s", apexName);
 481     }
 482
 483     private static String apexDataDirDeUser(String apexName, int userId) {
 484         return String.format("/data/misc_de/%d/apexdata/%s", userId, apexName);
 485     }
 486
 487     private static String apexDataDirCe(String apexName, int userId) {
 488         return String.format("/data/misc_ce/%d/apexdata/%s", userId, apexName);
 489     }
 490
 491     private List<String> getSnapshotDirectories(String baseDir) {
 492         try {
 493             return getDevice().getFileEntry(baseDir).getChildren(false)
 494                     .stream().filter(entry -> entry.getName().matches("\\d+(-prerestore)?"))
 495                     .map(entry -> entry.getFullPath())
 496                     .collect(Collectors.toList());
 497         } catch (Exception e) {
 498             // Return an empty list if any error
 499             return Collections.EMPTY_LIST;
 500         }
 501     }
 502
 503     private void assertDirectoryIsEmpty(String path) {
 504         try {
 505             IFileEntry file = getDevice().getFileEntry(path);
 506             assertTrue("Not a directory: " + path, file.isDirectory());
 507             assertTrue("Directory not empty: " + path, file.getChildren(false).isEmpty());
 508         } catch (DeviceNotAvailableException e) {
 509             fail("Can't access directory: " + path);
 510         }
 511     }
 512
 513     private void crashProcess(String processName, int numberOfCrashes) throws Exception {
 514         String pid = "";
 515         String lastPid = "invalid";
 516         for (int i = 0; i < numberOfCrashes; ++i) {
 517             // This condition makes sure before we kill the process, the process is running AND
 518             // the last crash was finished.
 519             while ("".equals(pid) || lastPid.equals(pid)) {
 520                 pid = getDevice().executeShellCommand("pidof " + processName);
 521             }
 522             getDevice().executeShellCommand("kill " + pid);
 523             lastPid = pid;
 524         }
 525     }
 526
 527     private boolean isCheckpointSupported() throws Exception {
 528         try {
 529             runPhase("isCheckpointSupported");
 530             return true;
 531         } catch (AssertionError ignore) {
 532             return false;
 533         }
 534     }
 535
 536     /**
 537      * True if this build has mainline modules installed.
 538      */
 539     private boolean hasMainlineModule() throws Exception {
 540         try {
 541             runPhase("hasMainlineModule");
 542             return true;
 543         } catch (AssertionError ignore) {
 544             return false;
 545         }
 546     }
 547
 548     @FunctionalInterface
 549     private interface ExceptionalRunnable {
 550         void run() throws Exception;
 551     }
 552
 553     private void runAsRoot(ExceptionalRunnable runnable) throws Exception {
 554         try {
 555             getDevice().enableAdbRoot();
 556             runnable.run();
 557         } finally {
 558             getDevice().disableAdbRoot();
 559         }
 560     }
 561 }