OSDN Git Service

Cert: make test setup more robust to binary failures
authorJack He <siyuanh@google.com>
Fri, 6 Mar 2020 08:19:48 +0000 (00:19 -0800)
committerJack He <siyuanh@google.com>
Fri, 6 Mar 2020 08:25:19 +0000 (00:25 -0800)
* Add checks to see if a backing process or root-canal is alive
  immediately after launching them to avoid making the whole
  test stuck on a response from these dead processes
* Add methods to disable verity on test devices and push test files
  to required directories on the Android device

Bug: 150321998
Test: source gd/cert/set_up_and_run_device_cert.sh
Change-Id: I44f2e73aa88e7924c7e594faad020f8199c76994

gd/cert/gd_base_test_facade_only.py
gd/cert/gd_device_base.py

index cdda85a..1412e64 100644 (file)
@@ -14,6 +14,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+from acts import asserts
 from acts.base_test import BaseTestClass
 from facade import rootservice_pb2 as facade_rootservice
 
@@ -24,6 +25,14 @@ import signal
 import subprocess
 
 
+def is_subprocess_alive(process, timeout_seconds=1):
+    try:
+        process.wait(timeout=timeout_seconds)
+        return False
+    except subprocess.TimeoutExpired as exp:
+        return True
+
+
 class GdFacadeOnlyBaseTestClass(BaseTestClass):
 
     def setup_class(self, dut_module, cert_module):
@@ -54,9 +63,14 @@ class GdFacadeOnlyBaseTestClass(BaseTestClass):
                 env=os.environ.copy(),
                 stdout=self.rootcanal_logs,
                 stderr=self.rootcanal_logs)
+            asserts.assert_true(
+                self.rootcanal_process,
+                msg="Cannot start root-canal at " + str(rootcanal))
+            asserts.assert_true(
+                is_subprocess_alive(self.rootcanal_process),
+                msg="root-canal stopped immediately after running")
             for gd_device in gd_devices:
                 gd_device["rootcanal_port"] = rootcanal_hci_port
-
         self.register_controller(
             importlib.import_module('cert.gd_device'), builtin=True)
 
index e18adaa..7c00cfc 100644 (file)
@@ -23,18 +23,21 @@ import socket
 import subprocess
 import time
 
+from acts import asserts
 from acts import context
-from acts.controllers.adb import AdbProxy
+from acts.controllers.adb import AdbProxy, AdbError
 
 import grpc
 
 from cert.environment_provider import PRODUCT_DEVICE
+from cert.gd_base_test_facade_only import is_subprocess_alive
 
 ANDROID_PRODUCT_OUT = os.path.join(
     os.getcwd(), "out/dist/bluetooth_cert_test/out/target/product",
     PRODUCT_DEVICE)
 
 WAIT_CHANNEL_READY_TIMEOUT = 10
+WAIT_FOR_DEVICE_TIMEOUT = 180
 
 
 def replace_vars(string, config):
@@ -77,26 +80,31 @@ class GdDeviceBase:
 
         self.serial_number = serial_number
         if self.serial_number:
-            self.ad = AdbProxy(serial_number)
-            self.ad.shell("date " + time.strftime("%m%d%H%M%Y.%S"))
-            self.ad.tcp_forward(int(grpc_port), int(grpc_port))
-            self.ad.tcp_forward(
+            self.adb = AdbProxy(self.serial_number)
+            self.ensure_verity_disabled()
+            asserts.assert_true(
+                self.adb.ensure_root(),
+                msg="device %s cannot run as root after enabling verity" %
+                self.serial_number)
+            self.adb.shell("date " + time.strftime("%m%d%H%M%Y.%S"))
+            self.adb.tcp_forward(int(grpc_port), int(grpc_port))
+            self.adb.tcp_forward(
                 int(grpc_root_server_port), int(grpc_root_server_port))
-            self.ad.reverse("tcp:%s tcp:%s" % (signal_port, signal_port))
-            self.ad.push(
+            self.adb.reverse("tcp:%s tcp:%s" % (signal_port, signal_port))
+            self.push_or_die(
                 os.path.join(ANDROID_PRODUCT_OUT,
                              "system/bin/bluetooth_stack_with_facade"),
                 "system/bin")
-            self.ad.push(
+            self.push_or_die(
                 os.path.join(ANDROID_PRODUCT_OUT,
                              "system/lib64/libbluetooth_gd.so"), "system/lib64")
-            self.ad.push(
+            self.push_or_die(
                 os.path.join(ANDROID_PRODUCT_OUT,
                              "system/lib64/libgrpc++_unsecure.so"),
                 "system/lib64")
-            self.ad.shell("logcat -c")
-            self.ad.shell("rm /data/misc/bluetooth/logs/btsnoop_hci.log")
-            self.ad.shell("svc bluetooth disable")
+            self.ensure_no_output(self.adb.shell("logcat -c"))
+            self.adb.shell("rm /data/misc/bluetooth/logs/btsnoop_hci.log")
+            self.ensure_no_output(self.adb.shell("svc bluetooth disable"))
 
         tester_signal_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         tester_signal_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
@@ -111,6 +119,13 @@ class GdDeviceBase:
             env=os.environ.copy(),
             stdout=self.backing_process_logs,
             stderr=self.backing_process_logs)
+        asserts.assert_true(
+            self.backing_process,
+            msg="Cannot start backing_process at " + " ".join(cmd))
+        asserts.assert_true(
+            is_subprocess_alive(self.backing_process),
+            msg="backing_process stopped immediately after running " +
+            " ".join(cmd))
         tester_signal_socket.accept()
         tester_signal_socket.close()
 
@@ -131,11 +146,11 @@ class GdDeviceBase:
                           (self.label, backing_process_return_code))
 
         if self.serial_number:
-            self.ad.shell("logcat -d -f /data/misc/bluetooth/logs/system_log")
-            self.ad.pull(
+            self.adb.shell("logcat -d -f /data/misc/bluetooth/logs/system_log")
+            self.adb.pull(
                 "/data/misc/bluetooth/logs/btsnoop_hci.log %s" % os.path.join(
                     self.log_path_base, "%s_btsnoop_hci.log" % self.label))
-            self.ad.pull(
+            self.adb.pull(
                 "/data/misc/bluetooth/logs/system_log %s" % os.path.join(
                     self.log_path_base, "%s_system_log" % self.label))
 
@@ -145,3 +160,97 @@ class GdDeviceBase:
             future.result(timeout=WAIT_CHANNEL_READY_TIMEOUT)
         except grpc.FutureTimeoutError:
             logging.error("wait channel ready timeout")
+
+    def ensure_no_output(self, result):
+        """
+        Ensure a command has not output
+        """
+        asserts.assert_true(
+            result is None or len(result) == 0,
+            msg="command returned something when it shouldn't: %s" % result)
+
+    def push_or_die(self, src_file_path, dst_file_path, push_timeout=300):
+        """Pushes a file to the Android device
+
+        Args:
+            src_file_path: The path to the file to install.
+            dst_file_path: The destination of the file.
+            push_timeout: How long to wait for the push to finish in seconds
+        """
+        try:
+            self.adb.ensure_root()
+            self.ensure_verity_disabled()
+            out = self.adb.push(
+                '%s %s' % (src_file_path, dst_file_path), timeout=push_timeout)
+            if 'error' in out:
+                asserts.fail('Unable to push file %s to %s due to %s' %
+                             (src_file_path, dst_file_path, out))
+        except Exception as e:
+            asserts.fail(
+                msg='Unable to push file %s to %s due to %s' %
+                (src_file_path, dst_file_path, e),
+                extras=e)
+
+    def ensure_verity_disabled(self):
+        """Ensures that verity is enabled.
+
+        If verity is not enabled, this call will reboot the phone. Note that
+        this only works on debuggable builds.
+        """
+        logging.debug("Disabling verity and remount for %s", self.serial_number)
+        asserts.assert_true(self.adb.ensure_root(),
+                            "device %s cannot run as root", self.serial_number)
+        # The below properties will only exist if verity has been enabled.
+        system_verity = self.adb.getprop('partition.system.verified')
+        vendor_verity = self.adb.getprop('partition.vendor.verified')
+        if system_verity or vendor_verity:
+            self.adb.disable_verity()
+            self.reboot()
+        self.adb.remount()
+        self.adb.wait_for_device(timeout=WAIT_FOR_DEVICE_TIMEOUT)
+
+    def reboot(self, timeout_minutes=15.0):
+        """Reboots the device.
+
+        Reboot the device, wait for device to complete booting.
+        """
+        logging.debug("Rebooting %s", self.serial_number)
+        self.adb.reboot()
+
+        timeout_start = time.time()
+        timeout = timeout_minutes * 60
+        # Android sometimes return early after `adb reboot` is called. This
+        # means subsequent calls may make it to the device before the reboot
+        # goes through, return false positives for getprops such as
+        # sys.boot_completed.
+        while time.time() < timeout_start + timeout:
+            try:
+                self.adb.get_state()
+                time.sleep(.1)
+            except AdbError:
+                # get_state will raise an error if the device is not found. We
+                # want the device to be missing to prove the device has kicked
+                # off the reboot.
+                break
+        minutes_left = timeout_minutes - (time.time() - timeout_start) / 60.0
+        self.wait_for_boot_completion(timeout_minutes=minutes_left)
+
+    def wait_for_boot_completion(self, timeout_minutes=15.0):
+        """Waits for Android framework to broadcast ACTION_BOOT_COMPLETED.
+        """
+        timeout_start = time.time()
+        timeout = timeout_minutes * 60
+
+        self.adb.wait_for_device(timeout=WAIT_FOR_DEVICE_TIMEOUT)
+        while time.time() < timeout_start + timeout:
+            try:
+                completed = self.adb.getprop("sys.boot_completed")
+                if completed == '1':
+                    return
+            except AdbError:
+                # adb shell calls may fail during certain period of booting
+                # process, which is normal. Ignoring these errors.
+                pass
+            time.sleep(5)
+        asserts.fail(msg='Device %s booting process timed out.' %
+                     self.serial_number)