Add a retry mechanism to potentially allow recovery from HIDL failures.

Retry a few times to see if HIDL failures persist.
Add some logging to track failures.

Bug: 36088202
Test: Ensure sensors stream normally after change.
Change-Id: I194eaffd455ba782fff041f03ef89da384e3a901

diff --git a/services/sensorservice/SensorDevice.cpp b/services/sensorservice/SensorDevice.cpp
index 7bd495f..2a17a7f 100644 (file)
--- a/services/sensorservice/SensorDevice.cpp
+++ b/services/sensorservice/SensorDevice.cpp
@@ -51,7 +51,7 @@ static status_t StatusFromResult(Result result) {
     }
 }
 
-SensorDevice::SensorDevice() {
+SensorDevice::SensorDevice() : mHidlTransportErrors(20) {
     // SensorDevice may wait upto 100ms * 10 = 1s for hidl service.
     constexpr auto RETRY_DELAY = std::chrono::milliseconds(100);
     size_t retry = 10;
@@ -118,7 +118,15 @@ std::string SensorDevice::dump() const {
     if (mSensors == NULL) return "HAL not initialized\n";
 
     String8 result;
-    checkReturn(mSensors->getSensorsList([&](const auto &list) {
+
+    result.appendFormat("Saw %d hidlTransport Errors\n", mTotalHidlTransportErrors);
+    for (auto it = mHidlTransportErrors.begin() ; it != mHidlTransportErrors.end(); it++ ) {
+        result += "\t";
+        result += it->toString();
+        result += "\n";
+    }
+
+    checkReturn(mSensors->getSensorsList([&](const auto &list){
             const size_t count = list.size();
 
             result.appendFormat(
@@ -182,19 +190,44 @@ ssize_t SensorDevice::poll(sensors_event_t* buffer, size_t count) {
     if (mSensors == NULL) return NO_INIT;
 
     ssize_t err;
+    int numHidlTransportErrors = 0;
+    bool hidlTransportError = false;
 
-    checkReturn(mSensors->poll(
-            count,
-            [&](auto result,
-                const auto &events,
-                const auto &dynamicSensorsAdded) {
-                if (result == Result::OK) {
-                    convertToSensorEvents(events, dynamicSensorsAdded, buffer);
-                    err = (ssize_t)events.size();
-                } else {
-                    err = StatusFromResult(result);
-                }
-            }));
+    do {
+        auto ret = mSensors->poll(
+                count,
+                [&](auto result,
+                    const auto &events,
+                    const auto &dynamicSensorsAdded) {
+                    if (result == Result::OK) {
+                        convertToSensorEvents(events, dynamicSensorsAdded, buffer);
+                        err = (ssize_t)events.size();
+                    } else {
+                        err = StatusFromResult(result);
+                    }
+                });
+
+        if (ret.isOk())  {
+            hidlTransportError = false;
+        } else {
+            hidlTransportError = true;
+            numHidlTransportErrors++;
+            if (numHidlTransportErrors > 50) {
+                // Log error and bail
+                ALOGE("Max Hidl transport errors this cycle : %d", numHidlTransportErrors);
+                handleHidlDeath(ret.description());
+            } else {
+                std::this_thread::sleep_for(std::chrono::milliseconds(10));
+            }
+        }
+    } while (hidlTransportError);
+
+    if(numHidlTransportErrors > 0) {
+        ALOGE("Saw %d Hidl transport failures", numHidlTransportErrors);
+        HidlTransportErrorLog errLog(time(NULL), numHidlTransportErrors);
+        mHidlTransportErrors.add(errLog);
+        mTotalHidlTransportErrors++;
+    }
 
     return err;
 }

diff --git a/services/sensorservice/SensorDevice.h b/services/sensorservice/SensorDevice.h
index 03552f6..410531b 100644 (file)
--- a/services/sensorservice/SensorDevice.h
+++ b/services/sensorservice/SensorDevice.h
@@ -31,6 +31,8 @@
 
 #include "android/hardware/sensors/1.0/ISensors.h"
 
+#include "RingBuffer.h"
+
 // ---------------------------------------------------------------------------
 
 namespace android {
@@ -41,6 +43,33 @@ using hardware::Return;
 
 class SensorDevice : public Singleton<SensorDevice>, public Dumpable {
 public:
+
+    class HidlTransportErrorLog {
+     public:
+
+        HidlTransportErrorLog() {
+            mTs = 0;
+            mCount = 0;
+        }
+
+        HidlTransportErrorLog(time_t ts, int count) {
+            mTs = ts;
+            mCount = count;
+        }
+
+        String8 toString() const {
+            String8 result;
+            struct tm *timeInfo = localtime(&mTs);
+            result.appendFormat("%02d:%02d:%02d :: %d", timeInfo->tm_hour, timeInfo->tm_min,
+                                timeInfo->tm_sec, mCount);
+            return result;
+        }
+
+    private:
+        time_t mTs; // timestamp of the error
+        int mCount;   // number of transport errors observed
+    };
+
     ssize_t getSensorList(sensor_t const** list);
 
     void handleDynamicSensorConnection(int handle, bool connected);
@@ -125,6 +154,10 @@ private:
     };
     DefaultKeyedVector<int, Info> mActivationCount;
 
+    // Keep track of any hidl transport failures
+    SensorServiceUtil::RingBuffer<HidlTransportErrorLog> mHidlTransportErrors;
+    int mTotalHidlTransportErrors;
+
     // Use this vector to determine which client is activated or deactivated.
     SortedVector<void *> mDisabledClients;
     SensorDevice();