import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
+import android.os.AsyncTask;
import android.os.Handler;
import android.os.Message;
import android.os.RemoteException;
* always-on keyphrase detection APIs.
*/
public class AlwaysOnHotwordDetector {
- //---- States of Keyphrase availability. Return codes for getAvailability() ----//
+ //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----//
/**
* Indicates that this hotword detector is no longer valid for any recognition
* and should not be used anymore.
*/
public static final int STATE_KEYPHRASE_ENROLLED = 2;
+ /**
+ * Indicates that the detector isn't ready currently.
+ */
+ private static final int STATE_NOT_READY = 0;
+
// Keyphrase management actions. Used in getManageIntent() ----//
/** Indicates that we need to enroll. */
public static final int MANAGE_ACTION_ENROLL = 0;
= SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION;
static final String TAG = "AlwaysOnHotwordDetector";
+ // TODO: Set to false.
+ static final boolean DBG = true;
- private static final int MSG_HOTWORD_DETECTED = 1;
- private static final int MSG_DETECTION_STOPPED = 2;
+ private static final int MSG_STATE_CHANGED = 1;
+ private static final int MSG_HOTWORD_DETECTED = 2;
+ private static final int MSG_DETECTION_STOPPED = 3;
private final String mText;
private final String mLocale;
private final IVoiceInteractionManagerService mModelManagementService;
private final SoundTriggerListener mInternalCallback;
private final Callback mExternalCallback;
- private final boolean mDisabled;
private final Object mLock = new Object();
+ private final Handler mHandler;
/**
* The sound model for the keyphrase, derived from the model management service
* (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.
*/
private KeyphraseSoundModel mEnrolledSoundModel;
- private boolean mInvalidated;
+ private int mAvailability = STATE_NOT_READY;
/**
* Callbacks for always-on hotword detection.
*/
public interface Callback {
/**
+ * Called when the hotword availability changes.
+ * This indicates a change in the availability of recognition for the given keyphrase.
+ * It's called at least once with the initial availability.<p/>
+ *
+ * Availability implies whether the hardware on this system is capable of listening for
+ * the given keyphrase or not. <p/>
+ * If the return code is one of {@link #STATE_HARDWARE_UNAVAILABLE} or
+ * {@link #STATE_KEYPHRASE_UNSUPPORTED},
+ * detection is not possible and no further interaction should be
+ * performed with this detector. <br/>
+ * If it is {@link #STATE_KEYPHRASE_UNENROLLED} the caller may choose to begin
+ * an enrollment flow for the keyphrase. <br/>
+ * and for {@link #STATE_KEYPHRASE_ENROLLED} a recognition can be started as desired. <p/>
+ *
+ * If the return code is {@link #STATE_INVALID}, this detector is stale.
+ * A new detector should be obtained for use in the future.
+ */
+ void onAvailabilityChanged(int status);
+ /**
* Called when the keyphrase is spoken.
*
* @param data Optional trigger audio data, if it was requested during
KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
IVoiceInteractionService voiceInteractionService,
IVoiceInteractionManagerService modelManagementService) {
- mInvalidated = false;
mText = text;
mLocale = locale;
mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
mExternalCallback = callback;
- mInternalCallback = new SoundTriggerListener(new MyHandler());
+ mHandler = new MyHandler();
+ mInternalCallback = new SoundTriggerListener(mHandler);
mVoiceInteractionService = voiceInteractionService;
mModelManagementService = modelManagementService;
- if (mKeyphraseMetadata != null) {
- mEnrolledSoundModel = internalGetKeyphraseSoundModelLocked(mKeyphraseMetadata.id);
- }
- int initialAvailability = internalGetAvailabilityLocked();
- mDisabled = (initialAvailability == STATE_HARDWARE_UNAVAILABLE)
- || (initialAvailability == STATE_KEYPHRASE_UNSUPPORTED);
- }
-
- /**
- * Gets the state of always-on hotword detection for the given keyphrase and locale
- * on this system.
- * Availability implies that the hardware on this system is capable of listening for
- * the given keyphrase or not. <p/>
- * If the return code is one of {@link #STATE_HARDWARE_UNAVAILABLE} or
- * {@link #STATE_KEYPHRASE_UNSUPPORTED}, no further interaction should be performed with this
- * detector. <br/>
- * If the state is {@link #STATE_KEYPHRASE_UNENROLLED} the caller may choose to begin
- * an enrollment flow for the keyphrase. <br/>
- * For {@value #STATE_KEYPHRASE_ENROLLED} a recognition can be started as desired. <br/>
- * If the return code is {@link #STATE_INVALID}, this detector is stale and must not be used.
- * A new detector should be obtained and used.
- *
- * @return Indicates if always-on hotword detection is available for the given keyphrase.
- * The return code is one of {@link #STATE_HARDWARE_UNAVAILABLE},
- * {@link #STATE_KEYPHRASE_UNSUPPORTED}, {@link #STATE_KEYPHRASE_UNENROLLED},
- * {@link #STATE_KEYPHRASE_ENROLLED}, or {@link #STATE_INVALID}.
- */
- public int getAvailability() {
- synchronized (mLock) {
- return internalGetAvailabilityLocked();
- }
+ new RefreshAvailabiltyTask().execute();
}
/**
* Gets the recognition modes supported by the associated keyphrase.
*
* @throws UnsupportedOperationException if the keyphrase itself isn't supported.
- * Callers should check the availability by calling {@link #getAvailability()}
- * before calling this method to avoid this exception.
+ * Callers should only call this method after a supported state callback on
+ * {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
*/
public int getSupportedRecognitionModes() {
synchronized (mLock) {
}
private int getSupportedRecognitionModesLocked() {
- if (mDisabled) {
+ // This method only makes sense if we can actually support a recognition.
+ if (mAvailability != STATE_KEYPHRASE_ENROLLED
+ && mAvailability != STATE_KEYPHRASE_UNENROLLED) {
throw new UnsupportedOperationException(
"Getting supported recognition modes for the keyphrase is not supported");
}
* {@link #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO}.
* @return {@link #STATUS_OK} if the call succeeds, an error code otherwise.
* @throws UnsupportedOperationException if the recognition isn't supported.
- * Callers should check the availability by calling {@link #getAvailability()}
- * before calling this method to avoid this exception.
+ * Callers should only call this method after a supported state callback on
+ * {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
*/
public int startRecognition(int recognitionFlags) {
synchronized (mLock) {
}
private int startRecognitionLocked(int recognitionFlags) {
- if (internalGetAvailabilityLocked() != STATE_KEYPHRASE_ENROLLED) {
+ // This method only makes sense if we can start a recognition.
+ if (mAvailability != STATE_KEYPHRASE_ENROLLED) {
throw new UnsupportedOperationException(
"Recognition for the given keyphrase is not supported");
}
*
* @return {@link #STATUS_OK} if the call succeeds, an error code otherwise.
* @throws UnsupportedOperationException if the recognition isn't supported.
- * Callers should check the availability by calling {@link #getAvailability()}
- * before calling this method to avoid this exception.
+ * Callers should only call this method after a supported state callback on
+ * {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
*/
public int stopRecognition() {
synchronized (mLock) {
}
private int stopRecognitionLocked() {
- if (internalGetAvailabilityLocked() != STATE_KEYPHRASE_ENROLLED) {
+ // This method only makes sense if we can start a recognition.
+ if (mAvailability != STATE_KEYPHRASE_ENROLLED) {
throw new UnsupportedOperationException(
"Recognition for the given keyphrase is not supported");
}
* {@link #MANAGE_ACTION_UN_ENROLL}.
* @return An {@link Intent} to manage the given keyphrase.
* @throws UnsupportedOperationException if managing they keyphrase isn't supported.
- * Callers should check the availability by calling {@link #getAvailability()}
- * before calling this method to avoid this exception.
+ * Callers should only call this method after a supported state callback on
+ * {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
*/
public Intent getManageIntent(int action) {
- if (mDisabled) {
+ // This method only makes sense if we can actually support a recognition.
+ if (mAvailability != STATE_KEYPHRASE_ENROLLED
+ && mAvailability != STATE_KEYPHRASE_UNENROLLED) {
throw new UnsupportedOperationException(
"Managing the given keyphrase is not supported");
}
return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale);
}
- private int internalGetAvailabilityLocked() {
- if (mInvalidated) {
- return STATE_INVALID;
- }
-
- ModuleProperties dspModuleProperties = null;
- try {
- dspModuleProperties =
- mModelManagementService.getDspModuleProperties(mVoiceInteractionService);
- } catch (RemoteException e) {
- Slog.w(TAG, "RemoteException in getDspProperties!");
- }
- // No DSP available
- if (dspModuleProperties == null) {
- return STATE_HARDWARE_UNAVAILABLE;
- }
- // No enrollment application supports this keyphrase/locale
- if (mKeyphraseMetadata == null) {
- return STATE_KEYPHRASE_UNSUPPORTED;
- }
-
- // This keyphrase hasn't been enrolled.
- if (mEnrolledSoundModel == null) {
- return STATE_KEYPHRASE_UNENROLLED;
- }
- return STATE_KEYPHRASE_ENROLLED;
- }
-
/**
* Invalidates this hotword detector so that any future calls to this result
* in an IllegalStateException.
*/
void invalidate() {
synchronized (mLock) {
- mInvalidated = true;
+ mAvailability = STATE_INVALID;
+ notifyStateChangedLocked();
}
}
synchronized (mLock) {
// TODO: This should stop the recognition if it was using an enrolled sound model
// that's no longer available.
- if (mKeyphraseMetadata != null) {
- mEnrolledSoundModel = internalGetKeyphraseSoundModelLocked(mKeyphraseMetadata.id);
+ if (mAvailability == STATE_INVALID
+ || mAvailability == STATE_HARDWARE_UNAVAILABLE
+ || mAvailability == STATE_KEYPHRASE_UNSUPPORTED) {
+ Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config");
+ return;
}
+
+ // Execute a refresh availability task - which should then notify of a change.
+ new RefreshAvailabiltyTask().execute();
}
}
- /**
- * @return The corresponding {@link KeyphraseSoundModel} or null if none is found.
- */
- private KeyphraseSoundModel internalGetKeyphraseSoundModelLocked(int keyphraseId) {
- List<KeyphraseSoundModel> soundModels;
- try {
- soundModels = mModelManagementService
- .listRegisteredKeyphraseSoundModels(mVoiceInteractionService);
- if (soundModels == null || soundModels.isEmpty()) {
- Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId);
- return null;
- }
- for (KeyphraseSoundModel soundModel : soundModels) {
- if (soundModel.keyphrases == null) {
- continue;
- }
- for (Keyphrase keyphrase : soundModel.keyphrases) {
- if (keyphrase.id == keyphraseId) {
- return soundModel;
- }
- }
- }
- } catch (RemoteException e) {
- Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!");
- }
- return null;
+ private void notifyStateChangedLocked() {
+ Message message = Message.obtain(mHandler, MSG_STATE_CHANGED);
+ message.arg1 = mAvailability;
+ message.sendToTarget();
}
/** @hide */
@Override
public void handleMessage(Message msg) {
switch (msg.what) {
+ case MSG_STATE_CHANGED:
+ mExternalCallback.onAvailabilityChanged(msg.arg1);
+ break;
case MSG_HOTWORD_DETECTED:
mExternalCallback.onDetected((byte[]) msg.obj);
break;
}
}
}
+
+ class RefreshAvailabiltyTask extends AsyncTask<Void, Void, Void> {
+
+ @Override
+ public Void doInBackground(Void... params) {
+ int availability = internalGetInitialAvailability();
+ KeyphraseSoundModel soundModel = null;
+ // Fetch the sound model if the availability is one of the supported ones.
+ if (availability == STATE_NOT_READY
+ || availability == STATE_KEYPHRASE_UNENROLLED
+ || availability == STATE_KEYPHRASE_ENROLLED) {
+ soundModel =
+ internalGetKeyphraseSoundModel(mKeyphraseMetadata.id);
+ if (soundModel == null) {
+ availability = STATE_KEYPHRASE_UNENROLLED;
+ } else {
+ availability = STATE_KEYPHRASE_ENROLLED;
+ }
+ }
+
+ synchronized (mLock) {
+ if (DBG) {
+ Slog.d(TAG, "Hotword availability changed from " + mAvailability
+ + " -> " + availability);
+ }
+ mAvailability = availability;
+ mEnrolledSoundModel = soundModel;
+ notifyStateChangedLocked();
+ }
+ return null;
+ }
+
+ /**
+ * @return The initial availability without checking the enrollment status.
+ */
+ private int internalGetInitialAvailability() {
+ synchronized (mLock) {
+ // This detector has already been invalidated.
+ if (mAvailability == STATE_INVALID) {
+ return STATE_INVALID;
+ }
+ }
+
+ ModuleProperties dspModuleProperties = null;
+ try {
+ dspModuleProperties =
+ mModelManagementService.getDspModuleProperties(mVoiceInteractionService);
+ } catch (RemoteException e) {
+ Slog.w(TAG, "RemoteException in getDspProperties!");
+ }
+ // No DSP available
+ if (dspModuleProperties == null) {
+ return STATE_HARDWARE_UNAVAILABLE;
+ }
+ // No enrollment application supports this keyphrase/locale
+ if (mKeyphraseMetadata == null) {
+ return STATE_KEYPHRASE_UNSUPPORTED;
+ }
+ return STATE_NOT_READY;
+ }
+
+ /**
+ * @return The corresponding {@link KeyphraseSoundModel} or null if none is found.
+ */
+ private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) {
+ List<KeyphraseSoundModel> soundModels;
+ try {
+ soundModels = mModelManagementService
+ .listRegisteredKeyphraseSoundModels(mVoiceInteractionService);
+ if (soundModels == null || soundModels.isEmpty()) {
+ Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId);
+ return null;
+ }
+ for (int i = 0; i < soundModels.size(); i++) {
+ KeyphraseSoundModel soundModel = soundModels.get(i);
+ if (soundModel.keyphrases == null || soundModel.keyphrases.length == 0) {
+ continue;
+ }
+ for (int j = 0; i < soundModel.keyphrases.length; j++) {
+ Keyphrase keyphrase = soundModel.keyphrases[j];
+ if (keyphrase.id == keyphraseId) {
+ return soundModel;
+ }
+ }
+ }
+ } catch (RemoteException e) {
+ Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!");
+ }
+ return null;
+ }
+ }
}
}
public boolean addOrUpdateKeyphraseSoundModel(KeyphraseSoundModel soundModel) {
- SQLiteDatabase db = getWritableDatabase();
- ContentValues values = new ContentValues();
- // Generate a random ID for the model.
- values.put(SoundModelContract.KEY_ID, soundModel.uuid.toString());
- values.put(SoundModelContract.KEY_DATA, soundModel.data);
- values.put(SoundModelContract.KEY_TYPE, SoundTrigger.SoundModel.TYPE_KEYPHRASE);
-
- boolean status = true;
- if (db.insertWithOnConflict(
- SoundModelContract.TABLE, null, values, SQLiteDatabase.CONFLICT_REPLACE) != -1) {
- for (Keyphrase keyphrase : soundModel.keyphrases) {
- status &= addOrUpdateKeyphrase(db, soundModel.uuid, keyphrase);
+ synchronized(this) {
+ SQLiteDatabase db = getWritableDatabase();
+ ContentValues values = new ContentValues();
+ // Generate a random ID for the model.
+ values.put(SoundModelContract.KEY_ID, soundModel.uuid.toString());
+ values.put(SoundModelContract.KEY_DATA, soundModel.data);
+ values.put(SoundModelContract.KEY_TYPE, SoundTrigger.SoundModel.TYPE_KEYPHRASE);
+
+ boolean status = true;
+ if (db.insertWithOnConflict(SoundModelContract.TABLE, null, values,
+ SQLiteDatabase.CONFLICT_REPLACE) != -1) {
+ for (Keyphrase keyphrase : soundModel.keyphrases) {
+ status &= addOrUpdateKeyphraseLocked(db, soundModel.uuid, keyphrase);
+ }
+ db.close();
+ return status;
+ } else {
+ Slog.w(TAG, "Failed to persist sound model to database");
+ db.close();
+ return false;
}
- db.close();
- return status;
- } else {
- Slog.w(TAG, "Failed to persist sound model to database");
- db.close();
- return false;
}
}
- private boolean addOrUpdateKeyphrase(SQLiteDatabase db, UUID modelId, Keyphrase keyphrase) {
+ private boolean addOrUpdateKeyphraseLocked(
+ SQLiteDatabase db, UUID modelId, Keyphrase keyphrase) {
ContentValues values = new ContentValues();
values.put(KeyphraseContract.KEY_ID, keyphrase.id);
values.put(KeyphraseContract.KEY_RECOGNITION_MODES, keyphrase.recognitionModes);
* Deletes the sound model and associated keyphrases.
*/
public boolean deleteKeyphraseSoundModel(UUID uuid) {
- SQLiteDatabase db = getWritableDatabase();
- String modelId = uuid.toString();
- String soundModelClause = SoundModelContract.KEY_ID + "=" + modelId;
- boolean status = true;
- if (db.delete(SoundModelContract.TABLE, soundModelClause, null) == 0) {
- Slog.w(TAG, "No sound models deleted from the database");
- status = false;
- }
- String keyphraseClause = KeyphraseContract.KEY_SOUND_MODEL_ID + "=" + modelId;
- if (db.delete(KeyphraseContract.TABLE, keyphraseClause, null) == 0) {
- Slog.w(TAG, "No keyphrases deleted from the database");
- status = false;
+ synchronized(this) {
+ SQLiteDatabase db = getWritableDatabase();
+ String modelId = uuid.toString();
+ String soundModelClause = SoundModelContract.KEY_ID + "=" + modelId;
+ boolean status = true;
+ if (db.delete(SoundModelContract.TABLE, soundModelClause, null) == 0) {
+ Slog.w(TAG, "No sound models deleted from the database");
+ status = false;
+ }
+ String keyphraseClause = KeyphraseContract.KEY_SOUND_MODEL_ID + "=" + modelId;
+ if (db.delete(KeyphraseContract.TABLE, keyphraseClause, null) == 0) {
+ Slog.w(TAG, "No keyphrases deleted from the database");
+ status = false;
+ }
+ db.close();
+ return status;
}
- db.close();
- return status;
}
/**
* Lists all the keyphrase sound models currently registered with the system.
*/
public List<KeyphraseSoundModel> getKephraseSoundModels() {
- List<KeyphraseSoundModel> models = new ArrayList<>();
- String selectQuery = "SELECT * FROM " + SoundModelContract.TABLE;
- SQLiteDatabase db = getReadableDatabase();
- Cursor c = db.rawQuery(selectQuery, null);
-
- // looping through all rows and adding to list
- if (c.moveToFirst()) {
- do {
- int type = c.getInt(c.getColumnIndex(SoundModelContract.KEY_TYPE));
- if (type != SoundTrigger.SoundModel.TYPE_KEYPHRASE) {
- // Ignore non-keyphrase sound models.
- continue;
- }
- String id = c.getString(c.getColumnIndex(SoundModelContract.KEY_ID));
- byte[] data = c.getBlob(c.getColumnIndex(SoundModelContract.KEY_DATA));
- // Get all the keyphrases for this this sound model.
- // Validate the sound model.
- if (id == null) {
- Slog.w(TAG, "Ignoring sound model since it doesn't specify an ID");
- continue;
- }
- KeyphraseSoundModel model = new KeyphraseSoundModel(
- UUID.fromString(id), data, getKeyphrasesForSoundModel(db, id));
- if (DBG) {
- Slog.d(TAG, "Adding model: " + model);
- }
- models.add(model);
- } while (c.moveToNext());
+ synchronized(this) {
+ List<KeyphraseSoundModel> models = new ArrayList<>();
+ String selectQuery = "SELECT * FROM " + SoundModelContract.TABLE;
+ SQLiteDatabase db = getReadableDatabase();
+ Cursor c = db.rawQuery(selectQuery, null);
+
+ // looping through all rows and adding to list
+ if (c.moveToFirst()) {
+ do {
+ int type = c.getInt(c.getColumnIndex(SoundModelContract.KEY_TYPE));
+ if (type != SoundTrigger.SoundModel.TYPE_KEYPHRASE) {
+ // Ignore non-keyphrase sound models.
+ continue;
+ }
+ String id = c.getString(c.getColumnIndex(SoundModelContract.KEY_ID));
+ byte[] data = c.getBlob(c.getColumnIndex(SoundModelContract.KEY_DATA));
+ // Get all the keyphrases for this this sound model.
+ // Validate the sound model.
+ if (id == null) {
+ Slog.w(TAG, "Ignoring sound model since it doesn't specify an ID");
+ continue;
+ }
+ KeyphraseSoundModel model = new KeyphraseSoundModel(
+ UUID.fromString(id), data, getKeyphrasesForSoundModelLocked(db, id));
+ if (DBG) {
+ Slog.d(TAG, "Adding model: " + model);
+ }
+ models.add(model);
+ } while (c.moveToNext());
+ }
+ c.close();
+ db.close();
+ return models;
}
- c.close();
- db.close();
- return models;
}
- private Keyphrase[] getKeyphrasesForSoundModel(SQLiteDatabase db, String modelId) {
+ private Keyphrase[] getKeyphrasesForSoundModelLocked(SQLiteDatabase db, String modelId) {
List<Keyphrase> keyphrases = new ArrayList<>();
String selectQuery = "SELECT * FROM " + KeyphraseContract.TABLE
+ " WHERE " + KeyphraseContract.KEY_SOUND_MODEL_ID + " = '" + modelId + "'";
}
- private String getCommaSeparatedString(int[] users) {
+ private static String getCommaSeparatedString(int[] users) {
if (users == null || users.length == 0) {
return "";
}
return csv.substring(0, csv.length() - 1);
}
- private int[] getArrayForCommaSeparatedString(String text) {
+ private static int[] getArrayForCommaSeparatedString(String text) {
if (TextUtils.isEmpty(text)) {
return null;
}