From 0897ab3aad7d062c101892fd40a96271714f1613 Mon Sep 17 00:00:00 2001
From: Michael Butler <butlermichael@google.com>
Date: Wed, 4 Oct 2017 02:38:42 -0700
Subject: [PATCH] Only verify output of models a vendor nn service fully
 supports

Some service drivers won't be able to support all operations. In those
cases, run the test as far as it can go, but ignore the output data.

For example, the hvx nn service does not support svdf, so it should
attempt to prepare and execute the model, but the test should not fail
if the hvx driver reports an error. However, hvx indicates it is able to
run MAX_POOL_2D, so the test should fail if that output is incorrect.

Bug: 63905942
Test: mm
Test: run vts binary on hvx service
Change-Id: I3ebc05836c8e070b66daa2e38b107f96637ca812
---
 .../1.0/vts/functional/GeneratedTestHarness.cpp    | 32 +++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 366bfc19..8c9a0a36 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -21,6 +21,7 @@
 #include <android-base/logging.h>
 #include <android/hidl/memory/1.0/IMemory.h>
 #include <hidlmemory/mapping.h>
+#include <iostream>
 
 namespace android {
 namespace hardware {
@@ -70,6 +71,19 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
     const uint32_t OUTPUT = 1;
     Model model = create_model();
 
+    // see if service can handle model
+    ErrorStatus supportedStatus;
+    bool fullySupportsModel = false;
+    Return<void> supportedCall = device->getSupportedOperations(
+        model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
+            supportedStatus = status;
+            ASSERT_NE(0ul, supported.size());
+            fullySupportsModel =
+                std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
+        });
+    ASSERT_TRUE(supportedCall.isOk());
+    ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
+
     // launch prepare model
     sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
     ASSERT_NE(nullptr, preparedModelCallback.get());
@@ -79,8 +93,13 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
     // retrieve prepared model
     preparedModelCallback->wait();
     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
-    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+    if (fullySupportsModel) {
+        EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+    } else {
+        EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
+                    prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
+    }
     ASSERT_NE(nullptr, preparedModel.get());
 
     int example_no = 1;
@@ -173,6 +192,17 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
         // retrieve execution status
         executionCallback->wait();
         ErrorStatus executionReturnStatus = executionCallback->getStatus();
+        if (!fullySupportsModel &&
+            static_cast<ErrorStatus>(executionReturnStatus) == ErrorStatus::GENERAL_FAILURE) {
+            LOG(INFO) << "Ignoring execution results from model that is not supported by the "
+                         "vendor service driver";
+            std::cout << "[          ]   Ignoring execution results from model that is not "
+                         "supported by the vendor service driver"
+                      << std::endl;
+            continue;
+        }
+        LOG(INFO) << "CONTINUING TO CHECK VALUE";
+        std::cout << "[          ]   CONTINUING TO CHECK VALUE" << std::endl;
         EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
 
         // validate results
-- 
2.11.0