From f4d922b2d9c2011bcdc0543b5cdf2dd53ca04b25 Mon Sep 17 00:00:00 2001 From: Sandeep Siddhartha Date: Thu, 15 Aug 2013 16:08:42 -0700 Subject: [PATCH] Add hotword detection in insecure keyguard - This talks to a service that's implemented by the Search app - The AIDL interface may be moved to the framework in a later CL Change-Id: I26553e46f7d17ba4ac7a952c871b28b261cba975 --- packages/Keyguard/AndroidManifest.xml | 3 + .../com/android/keyguard/HotwordServiceClient.java | 208 +++++++++++++++++++++ .../com/android/keyguard/KeyguardSelectorView.java | 126 ++++++++++++- .../android/search/service/IHotwordService.aidl | 35 ++++ .../search/service/IHotwordServiceCallback.aidl | 34 ++++ 5 files changed, 403 insertions(+), 3 deletions(-) create mode 100644 packages/Keyguard/src/com/android/keyguard/HotwordServiceClient.java create mode 100644 packages/Keyguard/src/com/google/android/search/service/IHotwordService.aidl create mode 100644 packages/Keyguard/src/com/google/android/search/service/IHotwordServiceCallback.aidl diff --git a/packages/Keyguard/AndroidManifest.xml b/packages/Keyguard/AndroidManifest.xml index 7d77c48b5636..f3106da665d1 100644 --- a/packages/Keyguard/AndroidManifest.xml +++ b/packages/Keyguard/AndroidManifest.xml @@ -38,6 +38,9 @@ + + + diff --git a/packages/Keyguard/src/com/android/keyguard/HotwordServiceClient.java b/packages/Keyguard/src/com/android/keyguard/HotwordServiceClient.java new file mode 100644 index 000000000000..94733d4f6ac1 --- /dev/null +++ b/packages/Keyguard/src/com/android/keyguard/HotwordServiceClient.java @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.keyguard; + +import android.app.SearchManager; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.content.ServiceConnection; +import android.os.Handler; +import android.os.IBinder; +import android.os.Message; +import android.os.RemoteException; +import android.os.UserHandle; +import android.util.Log; + +import com.google.android.search.service.IHotwordService; +import com.google.android.search.service.IHotwordServiceCallback; + +/** + * Utility class with its callbacks to simplify usage of {@link IHotwordService}. + * + * The client is meant to be used for a single hotword detection in a session. + * start() -> stop(); client is asked to stop & disconnect from the service. + * start() -> onHotwordDetected(); client disconnects from the service automatically. + */ +public class HotwordServiceClient implements Handler.Callback { + private static final String TAG = "HotwordServiceClient"; + private static final boolean DBG = true; + private static final String ACTION_HOTWORD = + "com.google.android.search.service.IHotwordService"; + + private static final int MSG_SERVICE_CONNECTED = 0; + private static final int MSG_SERVICE_DISCONNECTED = 1; + private static final int MSG_HOTWORD_STARTED = 2; + private static final int MSG_HOTWORD_STOPPED = 3; + private static final int MSG_HOTWORD_DETECTED = 4; + + private final Context mContext; + private final Callback mClientCallback; + private final Handler mHandler; + + private IHotwordService mService; + + public HotwordServiceClient(Context context, Callback callback) { + mContext = context; + mClientCallback = callback; + mHandler = new Handler(this); + } + + public interface Callback { + void onServiceConnected(); + void onServiceDisconnected(); + void onHotwordDetectionStarted(); + void onHotwordDetectionStopped(); + void onHotwordDetected(String action); + } + + /** + * Binds to the {@link IHotwordService} and starts hotword detection + * when the service is connected. + * + * @return false if the service can't be bound to. + */ + public synchronized boolean start() { + if (mService != null) { + if (DBG) Log.d(TAG, "Multiple call to start(), service was already bound"); + return true; + } else { + // TODO: The hotword service is currently hosted within the search app + // so the component handling the assist intent should handle hotwording + // as well. + final Intent intent = + ((SearchManager) mContext.getSystemService(Context.SEARCH_SERVICE)) + .getAssistIntent(mContext, true, UserHandle.USER_CURRENT); + if (intent == null) { + return false; + } + + Intent hotwordIntent = new Intent(ACTION_HOTWORD); + hotwordIntent.fillIn(intent, Intent.FILL_IN_PACKAGE); + return mContext.bindService( + hotwordIntent, + mConnection, + Context.BIND_AUTO_CREATE); + } + } + + /** + * Unbinds from the the {@link IHotwordService}. + */ + public synchronized void stop() { + if (mService != null) { + mContext.unbindService(mConnection); + mService = null; + } + } + + @Override + public boolean handleMessage(Message msg) { + switch (msg.what) { + case MSG_SERVICE_CONNECTED: + handleServiceConnected(); + break; + case MSG_SERVICE_DISCONNECTED: + handleServiceDisconnected(); + break; + case MSG_HOTWORD_STARTED: + handleHotwordDetectionStarted(); + break; + case MSG_HOTWORD_STOPPED: + handleHotwordDetectionStopped(); + break; + case MSG_HOTWORD_DETECTED: + handleHotwordDetected((String) msg.obj); + break; + default: + if (DBG) Log.e(TAG, "Unhandled message"); + return false; + } + return true; + } + + private void handleServiceConnected() { + if (DBG) Log.d(TAG, "handleServiceConnected()"); + if (mClientCallback != null) mClientCallback.onServiceConnected(); + try { + mService.requestHotwordDetection(mServiceCallback); + } catch (RemoteException e) { + Log.e(TAG, "Exception while registering callback", e); + mHandler.sendEmptyMessage(MSG_SERVICE_DISCONNECTED); + } + } + + private void handleServiceDisconnected() { + if (DBG) Log.d(TAG, "handleServiceDisconnected()"); + mService = null; + if (mClientCallback != null) mClientCallback.onServiceDisconnected(); + } + + private void handleHotwordDetectionStarted() { + if (DBG) Log.d(TAG, "handleHotwordDetectionStarted()"); + if (mClientCallback != null) mClientCallback.onHotwordDetectionStarted(); + } + + private void handleHotwordDetectionStopped() { + if (DBG) Log.d(TAG, "handleHotwordDetectionStopped()"); + if (mClientCallback != null) mClientCallback.onHotwordDetectionStopped(); + } + + void handleHotwordDetected(final String action) { + if (DBG) Log.d(TAG, "handleHotwordDetected()"); + if (mClientCallback != null) mClientCallback.onHotwordDetected(action); + stop(); + } + + /** + * Implements service connection methods. + */ + private ServiceConnection mConnection = new ServiceConnection() { + /** + * Called when the service connects after calling bind(). + */ + public void onServiceConnected(ComponentName className, IBinder iservice) { + mService = IHotwordService.Stub.asInterface(iservice); + mHandler.sendEmptyMessage(MSG_SERVICE_CONNECTED); + } + + /** + * Called if the service unexpectedly disconnects. This indicates an error. + */ + public void onServiceDisconnected(ComponentName className) { + mService = null; + mHandler.sendEmptyMessage(MSG_SERVICE_DISCONNECTED); + } + }; + + /** + * Implements the AIDL IHotwordServiceCallback interface. + */ + private final IHotwordServiceCallback mServiceCallback = new IHotwordServiceCallback.Stub() { + + public void onHotwordDetectionStarted() { + mHandler.sendEmptyMessage(MSG_HOTWORD_STARTED); + } + + public void onHotwordDetectionStopped() { + mHandler.sendEmptyMessage(MSG_HOTWORD_STOPPED); + } + + public void onHotwordDetected(String action) { + mHandler.obtainMessage(MSG_HOTWORD_DETECTED, action).sendToTarget(); + } + }; +} diff --git a/packages/Keyguard/src/com/android/keyguard/KeyguardSelectorView.java b/packages/Keyguard/src/com/android/keyguard/KeyguardSelectorView.java index 4d891beacec5..1c658e3f6460 100644 --- a/packages/Keyguard/src/com/android/keyguard/KeyguardSelectorView.java +++ b/packages/Keyguard/src/com/android/keyguard/KeyguardSelectorView.java @@ -22,8 +22,11 @@ import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.graphics.drawable.Drawable; +import android.os.Build; +import android.os.PowerManager; import android.os.UserHandle; import android.provider.Settings; +import android.telephony.TelephonyManager; import android.util.AttributeSet; import android.util.Log; import android.util.Slog; @@ -34,12 +37,15 @@ import com.android.internal.telephony.IccCardConstants.State; import com.android.internal.widget.LockPatternUtils; import com.android.internal.widget.multiwaveview.GlowPadView; import com.android.internal.widget.multiwaveview.GlowPadView.OnTriggerListener; +import com.android.keyguard.KeyguardHostView.OnDismissAction; public class KeyguardSelectorView extends LinearLayout implements KeyguardSecurityView { private static final boolean DEBUG = KeyguardHostView.DEBUG; private static final String TAG = "SecuritySelectorView"; private static final String ASSIST_ICON_METADATA_NAME = "com.android.systemui.action_assist_icon"; + // Flag to enable/disable hotword detection on lock screen. + private static final boolean FLAG_HOTWORD = true; private KeyguardSecurityCallback mCallback; private GlowPadView mGlowPadView; @@ -51,11 +57,15 @@ public class KeyguardSelectorView extends LinearLayout implements KeyguardSecuri private LockPatternUtils mLockPatternUtils; private SecurityMessageDisplay mSecurityMessageDisplay; private Drawable mBouncerFrame; + private HotwordServiceClient mHotwordClient; OnTriggerListener mOnTriggerListener = new OnTriggerListener() { public void onTrigger(View v, int target) { final int resId = mGlowPadView.getResourceIdForTarget(target); + if (FLAG_HOTWORD) { + maybeStopHotwordDetector(); + } switch (resId) { case R.drawable.ic_action_assist_generic: Intent assistIntent = @@ -103,7 +113,7 @@ public class KeyguardSelectorView extends LinearLayout implements KeyguardSecuri }; - KeyguardUpdateMonitorCallback mInfoCallback = new KeyguardUpdateMonitorCallback() { + KeyguardUpdateMonitorCallback mUpdateCallback = new KeyguardUpdateMonitorCallback() { @Override public void onDevicePolicyManagerStateChanged() { @@ -114,6 +124,24 @@ public class KeyguardSelectorView extends LinearLayout implements KeyguardSecuri public void onSimStateChanged(State simState) { updateTargets(); } + + @Override + public void onPhoneStateChanged(int phoneState) { + if (FLAG_HOTWORD) { + // We need to stop the hotwording when a phone call comes in + // TODO(sansid): This is not really needed if onPause triggers + // when we navigate away from the keyguard + if (phoneState == TelephonyManager.CALL_STATE_RINGING) { + if (DEBUG) Log.d(TAG, "Stopping due to CALL_STATE_RINGING"); + maybeStopHotwordDetector(); + } + } + } + + @Override + public void onUserSwitching(int userId) { + maybeStopHotwordDetector(); + } }; private final KeyguardActivityLauncher mActivityLauncher = new KeyguardActivityLauncher() { @@ -152,6 +180,9 @@ public class KeyguardSelectorView extends LinearLayout implements KeyguardSecuri mSecurityMessageDisplay = new KeyguardMessageArea.Helper(this); View bouncerFrameView = findViewById(R.id.keyguard_selector_view_frame); mBouncerFrame = bouncerFrameView.getBackground(); + if (FLAG_HOTWORD) { + mHotwordClient = new HotwordServiceClient(getContext(), mHotwordCallback); + } } public void setCarrierArea(View carrierArea) { @@ -254,12 +285,22 @@ public class KeyguardSelectorView extends LinearLayout implements KeyguardSecuri @Override public void onPause() { - KeyguardUpdateMonitor.getInstance(getContext()).removeCallback(mInfoCallback); + KeyguardUpdateMonitor.getInstance(getContext()).removeCallback(mUpdateCallback); } @Override public void onResume(int reason) { - KeyguardUpdateMonitor.getInstance(getContext()).registerCallback(mInfoCallback); + KeyguardUpdateMonitor.getInstance(getContext()).registerCallback(mUpdateCallback); + // TODO: Figure out if there's a better way to do it. + // Right now we don't get onPause at all, and onResume gets called + // multiple times (even when the screen is turned off with VIEW_REVEALED) + if (reason == SCREEN_ON) { + if (!KeyguardUpdateMonitor.getInstance(getContext()).isSwitchingUser()) { + maybeStartHotwordDetector(); + } + } else { + maybeStopHotwordDetector(); + } } @Override @@ -280,4 +321,83 @@ public class KeyguardSelectorView extends LinearLayout implements KeyguardSecuri KeyguardSecurityViewHelper. hideBouncer(mSecurityMessageDisplay, mFadeView, mBouncerFrame, duration); } + + /** + * Start the hotword detector if: + *
  • HOTWORDING_ENABLED is true and + *
  • HotwordUnlock is initialized and + *
  • TelephonyManager is in CALL_STATE_IDLE + * + * If this method is called when the screen is off, + * it attempts to stop hotwording if it's running. + */ + private void maybeStartHotwordDetector() { + if (FLAG_HOTWORD) { + if (DEBUG) Log.d(TAG, "maybeStartHotwordDetector()"); + // Don't start it if the screen is off or not showing + PowerManager powerManager = (PowerManager) getContext().getSystemService( + Context.POWER_SERVICE); + if (!powerManager.isScreenOn()) { + if (DEBUG) Log.d(TAG, "screen was off, not starting"); + return; + } + + KeyguardUpdateMonitor monitor = KeyguardUpdateMonitor.getInstance(getContext()); + if (monitor.getPhoneState() != TelephonyManager.CALL_STATE_IDLE) { + if (DEBUG) Log.d(TAG, "Call underway, not starting"); + return; + } + if (!mHotwordClient.start()) { + Log.w(TAG, "Failed to start the hotword detector"); + } + } + } + + /** + * Stop hotword detector if HOTWORDING_ENABLED is true. + */ + private void maybeStopHotwordDetector() { + if (FLAG_HOTWORD) { + if (DEBUG) Log.d(TAG, "maybeStopHotwordDetector()"); + mHotwordClient.stop(); + } + } + + private final HotwordServiceClient.Callback mHotwordCallback = + new HotwordServiceClient.Callback() { + private static final String TAG = "HotwordServiceClient.Callback"; + + @Override + public void onServiceConnected() { + if (DEBUG) Log.d(TAG, "onServiceConnected()"); + } + + @Override + public void onServiceDisconnected() { + if (DEBUG) Log.d(TAG, "onServiceDisconnected()"); + } + + @Override + public void onHotwordDetectionStarted() { + if (DEBUG) Log.d(TAG, "onHotwordDetectionStarted()"); + // TODO: Change the usage of SecurityMessageDisplay to a better visual indication. + mSecurityMessageDisplay.setMessage("\"Ok Google...\"", true); + } + + @Override + public void onHotwordDetectionStopped() { + if (DEBUG) Log.d(TAG, "onHotwordDetectionStopped()"); + // TODO: Change the usage of SecurityMessageDisplay to a better visual indication. + } + + @Override + public void onHotwordDetected(String action) { + if (DEBUG) Log.d(TAG, "onHotwordDetected(" + action + ")"); + if (action != null) { + Intent intent = new Intent(action); + mActivityLauncher.launchActivity(intent, true, true, null, null); + } + mCallback.userActivity(0); + } + }; } diff --git a/packages/Keyguard/src/com/google/android/search/service/IHotwordService.aidl b/packages/Keyguard/src/com/google/android/search/service/IHotwordService.aidl new file mode 100644 index 000000000000..e053d7dd9a42 --- /dev/null +++ b/packages/Keyguard/src/com/google/android/search/service/IHotwordService.aidl @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.search.service; + +import com.google.android.search.service.IHotwordServiceCallback; + +/** + * Interface exposing hotword detector as a service. + */ +oneway interface IHotwordService { + + /** + * Indicates a desire to start hotword detection. + * It's best-effort and the client should rely on + * the callbacks to figure out if hotwording was actually + * started or not. + * + * @param a callback to notify of hotword events. + */ + void requestHotwordDetection(in IHotwordServiceCallback callback); +} diff --git a/packages/Keyguard/src/com/google/android/search/service/IHotwordServiceCallback.aidl b/packages/Keyguard/src/com/google/android/search/service/IHotwordServiceCallback.aidl new file mode 100644 index 000000000000..7b3765fac530 --- /dev/null +++ b/packages/Keyguard/src/com/google/android/search/service/IHotwordServiceCallback.aidl @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.search.service; + +/** + * Interface implemented by users of Hotword service to get callbacks + * for hotword events. + */ +oneway interface IHotwordServiceCallback { + + /** Hotword detection start/stop callbacks */ + void onHotwordDetectionStarted(); + void onHotwordDetectionStopped(); + + /** + * Called back when hotword is detected. + * The action tells the client what action to take, post hotword-detection. + */ + void onHotwordDetected(in String action); +} -- 2.11.0