From ba08b7945a66434a76840062b2dac1275adc4844 Mon Sep 17 00:00:00 2001 From: Chris Thornton Date: Thu, 8 Jun 2017 22:34:37 -0700 Subject: [PATCH] Add SoundTriggerManager APIs to use a PendingIntent to get callbacks. Test: APIs exercised using a special test app not in this CL. Change-Id: I99425d1e67a778513e6c75e7d595c072032aa2ab --- .../android/internal/app/ISoundTriggerService.aidl | 17 +- .../media/soundtrigger/SoundTriggerManager.java | 146 +++++++++ .../server/soundtrigger/SoundTriggerHelper.java | 7 + .../server/soundtrigger/SoundTriggerService.java | 364 +++++++++++++++++++++ 4 files changed, 530 insertions(+), 4 deletions(-) diff --git a/core/java/com/android/internal/app/ISoundTriggerService.aidl b/core/java/com/android/internal/app/ISoundTriggerService.aidl index f4c18c300513..1bee6924454d 100644 --- a/core/java/com/android/internal/app/ISoundTriggerService.aidl +++ b/core/java/com/android/internal/app/ISoundTriggerService.aidl @@ -16,6 +16,7 @@ package com.android.internal.app; +import android.app.PendingIntent; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; import android.os.ParcelUuid; @@ -26,7 +27,6 @@ import android.os.ParcelUuid; */ interface ISoundTriggerService { - SoundTrigger.GenericSoundModel getSoundModel(in ParcelUuid soundModelId); void updateSoundModel(in SoundTrigger.GenericSoundModel soundModel); @@ -36,8 +36,17 @@ interface ISoundTriggerService { int startRecognition(in ParcelUuid soundModelId, in IRecognitionStatusCallback callback, in SoundTrigger.RecognitionConfig config); - /** - * Stops recognition. - */ int stopRecognition(in ParcelUuid soundModelId, in IRecognitionStatusCallback callback); + + int loadGenericSoundModel(in SoundTrigger.GenericSoundModel soundModel); + int loadKeyphraseSoundModel(in SoundTrigger.KeyphraseSoundModel soundModel); + + int startRecognitionForIntent(in ParcelUuid soundModelId, in PendingIntent callbackIntent, + in SoundTrigger.RecognitionConfig config); + + int stopRecognitionForIntent(in ParcelUuid soundModelId); + + int unloadSoundModel(in ParcelUuid soundModelId); + + boolean isRecognitionActive(in ParcelUuid parcelUuid); } diff --git a/media/java/android/media/soundtrigger/SoundTriggerManager.java b/media/java/android/media/soundtrigger/SoundTriggerManager.java index 7f8140adaed2..92ffae0fe560 100644 --- a/media/java/android/media/soundtrigger/SoundTriggerManager.java +++ b/media/java/android/media/soundtrigger/SoundTriggerManager.java @@ -15,7 +15,9 @@ */ package android.media.soundtrigger; +import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR; +import android.app.PendingIntent; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; @@ -23,6 +25,10 @@ import android.annotation.SystemApi; import android.annotation.SystemService; import android.content.Context; import android.hardware.soundtrigger.SoundTrigger; +import android.hardware.soundtrigger.SoundTrigger.SoundModel; +import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; +import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; +import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; import android.os.Handler; import android.os.ParcelUuid; import android.os.RemoteException; @@ -178,4 +184,144 @@ public final class SoundTriggerManager { return mGenericSoundModel; } } + + + /** + * Default message type. + * @hide + */ + public static final int FLAG_MESSAGE_TYPE_UNKNOWN = -1; + /** + * Contents of EXTRA_MESSAGE_TYPE extra for a RecognitionEvent. + * @hide + */ + public static final int FLAG_MESSAGE_TYPE_RECOGNITION_EVENT = 0; + /** + * Contents of EXTRA_MESSAGE_TYPE extra for recognition error events. + * @hide + */ + public static final int FLAG_MESSAGE_TYPE_RECOGNITION_ERROR = 1; + /** + * Contents of EXTRA_MESSAGE_TYPE extra for a recognition paused events. + * @hide + */ + public static final int FLAG_MESSAGE_TYPE_RECOGNITION_PAUSED = 2; + /** + * Contents of EXTRA_MESSAGE_TYPE extra for recognition resumed events. + * @hide + */ + public static final int FLAG_MESSAGE_TYPE_RECOGNITION_RESUMED = 3; + + /** + * Extra key in the intent for the type of the message. + * @hide + */ + public static final String EXTRA_MESSAGE_TYPE = "android.media.soundtrigger.MESSAGE_TYPE"; + /** + * Extra key in the intent that holds the RecognitionEvent parcelable. + * @hide + */ + public static final String EXTRA_RECOGNITION_EVENT = "android.media.soundtrigger.RECOGNITION_EVENT"; + /** + * Extra key in the intent that holds the status in an error message. + * @hide + */ + public static final String EXTRA_STATUS = "android.media.soundtrigger.STATUS"; + + /** + * Loads a given sound model into the sound trigger. Note the model will be unloaded if there is + * an error/the system service is restarted. + * @hide + */ + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + public int loadSoundModel(SoundModel soundModel) { + if (soundModel == null) { + return STATUS_ERROR; + } + + try { + switch (soundModel.type) { + case SoundModel.TYPE_GENERIC_SOUND: + return mSoundTriggerService.loadGenericSoundModel( + (GenericSoundModel) soundModel); + case SoundModel.TYPE_KEYPHRASE: + return mSoundTriggerService.loadKeyphraseSoundModel( + (KeyphraseSoundModel) soundModel); + default: + Slog.e(TAG, "Unkown model type"); + return STATUS_ERROR; + } + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Starts recognition on the given model id. All events from the model will be sent to the + * PendingIntent. + * @hide + */ + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + public int startRecognition(UUID soundModelId, PendingIntent callbackIntent, + RecognitionConfig config) { + if (soundModelId == null || callbackIntent == null || config == null) { + return STATUS_ERROR; + } + try { + return mSoundTriggerService.startRecognitionForIntent(new ParcelUuid(soundModelId), + callbackIntent, config); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Stops the given model's recognition. + * @hide + */ + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + public int stopRecognition(UUID soundModelId) { + if (soundModelId == null) { + return STATUS_ERROR; + } + try { + return mSoundTriggerService.stopRecognitionForIntent(new ParcelUuid(soundModelId)); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Removes the given model from memory. Will also stop any pending recognitions. + * @hide + */ + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + public int unloadSoundModel(UUID soundModelId) { + if (soundModelId == null) { + return STATUS_ERROR; + } + try { + return mSoundTriggerService.unloadSoundModel( + new ParcelUuid(soundModelId)); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Returns true if the given model has had detection started on it. + * @hide + */ + @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) + public boolean isRecognitionActive(UUID soundModelId) { + if (soundModelId == null) { + return false; + } + try { + return mSoundTriggerService.isRecognitionActive( + new ParcelUuid(soundModelId)); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java index 1d5fb55e254a..f53eb15d482b 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java @@ -558,6 +558,13 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { } } + boolean isRecognitionRequested(UUID modelId) { + synchronized (mLock) { + ModelData modelData = mModelDataMap.get(modelId); + return modelData != null && modelData.isRequested(); + } + } + //---- SoundTrigger.StatusListener methods @Override public void onRecognition(RecognitionEvent event) { diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java index 9bca0128cc43..51c805da2cac 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java @@ -16,18 +16,25 @@ package com.android.server.soundtrigger; import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR; +import static android.hardware.soundtrigger.SoundTrigger.STATUS_OK; +import android.app.PendingIntent; import android.content.Context; +import android.content.Intent; import android.content.pm.PackageManager; import android.Manifest; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; +import android.hardware.soundtrigger.SoundTrigger.SoundModel; import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; +import android.media.soundtrigger.SoundTriggerManager; +import android.os.Bundle; import android.os.Parcel; import android.os.ParcelUuid; +import android.os.PowerManager; import android.os.RemoteException; import android.util.Slog; @@ -36,6 +43,7 @@ import com.android.internal.app.ISoundTriggerService; import java.io.FileDescriptor; import java.io.PrintWriter; +import java.util.TreeMap; import java.util.UUID; /** @@ -52,16 +60,23 @@ public class SoundTriggerService extends SystemService { private static final boolean DEBUG = true; final Context mContext; + private Object mLock; private final SoundTriggerServiceStub mServiceStub; private final LocalSoundTriggerService mLocalSoundTriggerService; private SoundTriggerDbHelper mDbHelper; private SoundTriggerHelper mSoundTriggerHelper; + private final TreeMap mLoadedModels; + private final TreeMap mIntentCallbacks; + private PowerManager.WakeLock mWakelock; public SoundTriggerService(Context context) { super(context); mContext = context; mServiceStub = new SoundTriggerServiceStub(); mLocalSoundTriggerService = new LocalSoundTriggerService(context); + mLoadedModels = new TreeMap(); + mIntentCallbacks = new TreeMap(); + mLock = new Object(); } @Override @@ -177,8 +192,357 @@ public class SoundTriggerService extends SystemService { mSoundTriggerHelper.unloadGenericSoundModel(soundModelId.getUuid()); mDbHelper.deleteGenericSoundModel(soundModelId.getUuid()); } + + @Override + public int loadGenericSoundModel(GenericSoundModel soundModel) { + enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); + if (!isInitialized()) return STATUS_ERROR; + if (soundModel == null || soundModel.uuid == null) { + Slog.e(TAG, "Invalid sound model"); + return STATUS_ERROR; + } + if (DEBUG) { + Slog.i(TAG, "loadGenericSoundModel(): id = " + soundModel.uuid); + } + synchronized (mLock) { + SoundModel oldModel = mLoadedModels.get(soundModel.uuid); + // If the model we're loading is actually different than what we had loaded, we + // should unload that other model now. We don't care about return codes since we + // don't know if the other model is loaded. + if (oldModel != null && !oldModel.equals(soundModel)) { + mSoundTriggerHelper.unloadGenericSoundModel(soundModel.uuid); + mIntentCallbacks.remove(soundModel.uuid); + } + mLoadedModels.put(soundModel.uuid, soundModel); + } + return STATUS_OK; + } + + @Override + public int loadKeyphraseSoundModel(KeyphraseSoundModel soundModel) { + enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); + if (!isInitialized()) return STATUS_ERROR; + if (soundModel == null || soundModel.uuid == null) { + Slog.e(TAG, "Invalid sound model"); + return STATUS_ERROR; + } + if (soundModel.keyphrases == null || soundModel.keyphrases.length != 1) { + Slog.e(TAG, "Only one keyphrase per model is currently supported."); + return STATUS_ERROR; + } + if (DEBUG) { + Slog.i(TAG, "loadKeyphraseSoundModel(): id = " + soundModel.uuid); + } + synchronized (mLock) { + SoundModel oldModel = mLoadedModels.get(soundModel.uuid); + // If the model we're loading is actually different than what we had loaded, we + // should unload that other model now. We don't care about return codes since we + // don't know if the other model is loaded. + if (oldModel != null && !oldModel.equals(soundModel)) { + mSoundTriggerHelper.unloadKeyphraseSoundModel(soundModel.keyphrases[0].id); + mIntentCallbacks.remove(soundModel.uuid); + } + mLoadedModels.put(soundModel.uuid, soundModel); + } + return STATUS_OK; + } + + @Override + public int startRecognitionForIntent(ParcelUuid soundModelId, PendingIntent callbackIntent, + SoundTrigger.RecognitionConfig config) { + enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); + if (!isInitialized()) return STATUS_ERROR; + if (DEBUG) { + Slog.i(TAG, "startRecognition(): id = " + soundModelId); + } + + synchronized (mLock) { + SoundModel soundModel = mLoadedModels.get(soundModelId.getUuid()); + if (soundModel == null) { + Slog.e(TAG, soundModelId + " is not loaded"); + return STATUS_ERROR; + } + LocalSoundTriggerRecognitionStatusCallback callback = mIntentCallbacks.get( + soundModelId.getUuid()); + if (callback != null) { + Slog.e(TAG, soundModelId + " is already running"); + return STATUS_ERROR; + } + callback = new LocalSoundTriggerRecognitionStatusCallback(soundModelId.getUuid(), + callbackIntent, config); + int ret; + switch (soundModel.type) { + case SoundModel.TYPE_KEYPHRASE: { + KeyphraseSoundModel keyphraseSoundModel = (KeyphraseSoundModel) soundModel; + ret = mSoundTriggerHelper.startKeyphraseRecognition( + keyphraseSoundModel.keyphrases[0].id, keyphraseSoundModel, callback, + config); + } break; + case SoundModel.TYPE_GENERIC_SOUND: + ret = mSoundTriggerHelper.startGenericRecognition(soundModel.uuid, + (GenericSoundModel) soundModel, callback, config); + break; + default: + Slog.e(TAG, "Unknown model type"); + return STATUS_ERROR; + } + + if (ret != STATUS_OK) { + Slog.e(TAG, "Failed to start model: " + ret); + return ret; + } + mIntentCallbacks.put(soundModelId.getUuid(), callback); + } + return STATUS_OK; + } + + @Override + public int stopRecognitionForIntent(ParcelUuid soundModelId) { + enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); + if (!isInitialized()) return STATUS_ERROR; + if (DEBUG) { + Slog.i(TAG, "stopRecognition(): id = " + soundModelId); + } + + synchronized (mLock) { + SoundModel soundModel = mLoadedModels.get(soundModelId.getUuid()); + if (soundModel == null) { + Slog.e(TAG, soundModelId + " is not loaded"); + return STATUS_ERROR; + } + LocalSoundTriggerRecognitionStatusCallback callback = mIntentCallbacks.get( + soundModelId.getUuid()); + if (callback == null) { + Slog.e(TAG, soundModelId + " is not running"); + return STATUS_ERROR; + } + int ret; + switch (soundModel.type) { + case SoundModel.TYPE_KEYPHRASE: + ret = mSoundTriggerHelper.stopKeyphraseRecognition( + ((KeyphraseSoundModel)soundModel).keyphrases[0].id, callback); + break; + case SoundModel.TYPE_GENERIC_SOUND: + ret = mSoundTriggerHelper.stopGenericRecognition(soundModel.uuid, callback); + break; + default: + Slog.e(TAG, "Unknown model type"); + return STATUS_ERROR; + } + + if (ret != STATUS_OK) { + Slog.e(TAG, "Failed to stop model: " + ret); + return ret; + } + mIntentCallbacks.remove(soundModelId.getUuid()); + } + return STATUS_OK; + } + + @Override + public int unloadSoundModel(ParcelUuid soundModelId) { + enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); + if (!isInitialized()) return STATUS_ERROR; + if (DEBUG) { + Slog.i(TAG, "unloadSoundModel(): id = " + soundModelId); + } + + synchronized (mLock) { + SoundModel soundModel = mLoadedModels.get(soundModelId.getUuid()); + if (soundModel == null) { + Slog.e(TAG, soundModelId + " is not loaded"); + return STATUS_ERROR; + } + int ret; + switch (soundModel.type) { + case SoundModel.TYPE_KEYPHRASE: + ret = mSoundTriggerHelper.unloadKeyphraseSoundModel( + ((KeyphraseSoundModel)soundModel).keyphrases[0].id); + break; + case SoundModel.TYPE_GENERIC_SOUND: + ret = mSoundTriggerHelper.unloadGenericSoundModel(soundModel.uuid); + break; + default: + Slog.e(TAG, "Unknown model type"); + return STATUS_ERROR; + } + if (ret != STATUS_OK) { + Slog.e(TAG, "Failed to unload model"); + return ret; + } + mLoadedModels.remove(soundModelId.getUuid()); + return STATUS_OK; + } + } + + @Override + public boolean isRecognitionActive(ParcelUuid parcelUuid) { + enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); + if (!isInitialized()) return false; + synchronized (mLock) { + LocalSoundTriggerRecognitionStatusCallback callback = + mIntentCallbacks.get(parcelUuid.getUuid()); + if (callback == null) { + return false; + } + return mSoundTriggerHelper.isRecognitionRequested(parcelUuid.getUuid()); + } + } } + private final class LocalSoundTriggerRecognitionStatusCallback + extends IRecognitionStatusCallback.Stub { + private UUID mUuid; + private PendingIntent mCallbackIntent; + private RecognitionConfig mRecognitionConfig; + + public LocalSoundTriggerRecognitionStatusCallback(UUID modelUuid, + PendingIntent callbackIntent, + RecognitionConfig config) { + mUuid = modelUuid; + mCallbackIntent = callbackIntent; + mRecognitionConfig = config; + } + + @Override + public boolean pingBinder() { + return mCallbackIntent != null; + } + + @Override + public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent event) { + if (mCallbackIntent == null) { + return; + } + grabWakeLock(); + + Slog.w(TAG, "Keyphrase sound trigger event: " + event); + Intent extras = new Intent(); + extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, + SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_EVENT); + extras.putExtra(SoundTriggerManager.EXTRA_RECOGNITION_EVENT, event); + try { + mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); + if (!mRecognitionConfig.allowMultipleTriggers) { + removeCallback(/*releaseWakeLock=*/false); + } + } catch (PendingIntent.CanceledException e) { + removeCallback(/*releaseWakeLock=*/true); + } + } + + @Override + public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { + if (mCallbackIntent == null) { + return; + } + grabWakeLock(); + + Slog.w(TAG, "Generic sound trigger event: " + event); + Intent extras = new Intent(); + extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, + SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_EVENT); + extras.putExtra(SoundTriggerManager.EXTRA_RECOGNITION_EVENT, event); + try { + mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); + if (!mRecognitionConfig.allowMultipleTriggers) { + removeCallback(/*releaseWakeLock=*/false); + } + } catch (PendingIntent.CanceledException e) { + removeCallback(/*releaseWakeLock=*/true); + } + } + + @Override + public void onError(int status) { + if (mCallbackIntent == null) { + return; + } + grabWakeLock(); + + Slog.i(TAG, "onError: " + status); + Intent extras = new Intent(); + extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, + SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_ERROR); + extras.putExtra(SoundTriggerManager.EXTRA_STATUS, status); + try { + mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); + // Remove the callback, but wait for the intent to finish before we let go of the + // wake lock + removeCallback(/*releaseWakeLock=*/false); + } catch (PendingIntent.CanceledException e) { + removeCallback(/*releaseWakeLock=*/true); + } + } + + @Override + public void onRecognitionPaused() { + if (mCallbackIntent == null) { + return; + } + grabWakeLock(); + + Slog.i(TAG, "onRecognitionPaused"); + Intent extras = new Intent(); + extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, + SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_PAUSED); + try { + mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); + } catch (PendingIntent.CanceledException e) { + removeCallback(/*releaseWakeLock=*/true); + } + } + + @Override + public void onRecognitionResumed() { + if (mCallbackIntent == null) { + return; + } + grabWakeLock(); + + Slog.i(TAG, "onRecognitionResumed"); + Intent extras = new Intent(); + extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, + SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_RESUMED); + try { + mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); + } catch (PendingIntent.CanceledException e) { + removeCallback(/*releaseWakeLock=*/true); + } + } + + private void removeCallback(boolean releaseWakeLock) { + mCallbackIntent = null; + synchronized (mLock) { + mIntentCallbacks.remove(mUuid); + if (releaseWakeLock) { + mWakelock.release(); + } + } + } + } + + private void grabWakeLock() { + synchronized (mLock) { + if (mWakelock == null) { + PowerManager pm = ((PowerManager) mContext.getSystemService(Context.POWER_SERVICE)); + mWakelock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, TAG); + } + mWakelock.acquire(); + } + } + + private PendingIntent.OnFinished mCallbackCompletedHandler = new PendingIntent.OnFinished() { + @Override + public void onSendFinished(PendingIntent pendingIntent, Intent intent, int resultCode, + String resultData, Bundle resultExtras) { + // We're only ever invoked when the callback is done, so release the lock. + synchronized (mLock) { + mWakelock.release(); + } + } + }; + public final class LocalSoundTriggerService extends SoundTriggerInternal { private final Context mContext; private SoundTriggerHelper mSoundTriggerHelper; -- 2.11.0