From 4a89bece0d7281bcca765127e8a283e711c06514 Mon Sep 17 00:00:00 2001 From: "Cel A. Skeggs" Date: Sat, 15 Feb 2025 12:00:26 -0800 Subject: [PATCH 1/2] Add skill to disable/enable listening for wake word --- .../stypox/dicio/di/SettingsAccessWrapper.kt | 45 +++++++++++++++++++ .../org/stypox/dicio/di/SkillContextImpl.kt | 8 +++- .../org/stypox/dicio/eval/SkillHandler.kt | 2 + .../dicio/skills/listening/ListeningInfo.kt | 32 +++++++++++++ .../dicio/skills/listening/ListeningOutput.kt | 25 +++++++++++ .../dicio/skills/listening/ListeningSkill.kt | 23 ++++++++++ app/src/main/res/values/strings.xml | 6 +++ app/src/main/sentences/en/listening.yml | 13 ++++++ app/src/main/sentences/skill_definitions.yml | 6 +++ .../dicio/skill/context/NoSettingsAccess.kt | 5 +++ .../org/dicio/skill/context/SettingsAccess.kt | 5 +++ .../org/dicio/skill/context/SkillContext.kt | 2 + 12 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt create mode 100644 app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningInfo.kt create mode 100644 app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningOutput.kt create mode 100644 app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt create mode 100644 app/src/main/sentences/en/listening.yml create mode 100644 skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt create mode 100644 skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt diff --git a/app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt b/app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt new file mode 100644 index 000000000..10e1ccf43 --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt @@ -0,0 +1,45 @@ +package org.stypox.dicio.di + +import androidx.datastore.core.DataStore +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.dicio.skill.context.SettingsAccess +import org.stypox.dicio.settings.datastore.UserSettings +import org.stypox.dicio.settings.datastore.WakeDevice +import org.stypox.dicio.settings.datastore.copy +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SettingsAccessWrapper @Inject constructor( + private val dataStore: DataStore, +) : SettingsAccess { + + private val scope = CoroutineScope(Dispatchers.Main) + private var lastObservedSettings: UserSettings = UserSettings.getDefaultInstance() + + init { + scope.launch { + dataStore.data.collect { + lastObservedSettings = it + } + } + } + + override var wakeDeviceEnabled: Boolean + get() = lastObservedSettings.wakeDevice != WakeDevice.WAKE_DEVICE_NOTHING + set(value) { + scope.launch { + dataStore.updateData { + it.copy { + wakeDevice = if (value) { + WakeDevice.WAKE_DEVICE_OWW + } else { + WakeDevice.WAKE_DEVICE_NOTHING + } + } + } + } + } +} diff --git a/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt b/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt index db63d512a..fe0c0c68f 100644 --- a/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt +++ b/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt @@ -2,8 +2,11 @@ package org.stypox.dicio.di import android.annotation.SuppressLint import android.content.Context +import android.provider.Settings import dagger.hilt.android.qualifiers.ApplicationContext import org.dicio.numbers.ParserFormatter +import org.dicio.skill.context.NoSettingsAccess +import org.dicio.skill.context.SettingsAccess import org.dicio.skill.context.SkillContext import org.dicio.skill.context.SpeechOutputDevice import org.stypox.dicio.io.speech.NothingSpeechDevice @@ -18,6 +21,7 @@ class SkillContextImpl private constructor( // this constructor can take any SpeechOutputDevice to allow newForPreviews to provide // NothingSpeechDevice override val speechOutputDevice: SpeechOutputDevice, + override val settingsAccess: SettingsAccess, ) : SkillContext { @Inject @@ -25,7 +29,8 @@ class SkillContextImpl private constructor( @ApplicationContext android: Context, localeManager: LocaleManager, speechOutputDevice: SpeechOutputDeviceWrapper, - ) : this(android, localeManager, speechOutputDevice as SpeechOutputDevice) + settingsAccess: SettingsAccessWrapper, + ) : this(android, localeManager, speechOutputDevice as SpeechOutputDevice, settingsAccess as SettingsAccess) override val locale: Locale @@ -58,6 +63,7 @@ class SkillContextImpl private constructor( context, localeManager, NothingSpeechDevice(), + NoSettingsAccess(), ) @SuppressLint("StateFlowValueCalledInComposition") res.lastParserFormatter = Pair( diff --git a/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt b/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt index 3de47520e..b09398a6d 100644 --- a/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt +++ b/app/src/main/kotlin/org/stypox/dicio/eval/SkillHandler.kt @@ -20,6 +20,7 @@ import org.stypox.dicio.settings.datastore.UserSettingsModule import org.stypox.dicio.skills.calculator.CalculatorInfo import org.stypox.dicio.skills.current_time.CurrentTimeInfo import org.stypox.dicio.skills.fallback.text.TextFallbackInfo +import org.stypox.dicio.skills.listening.ListeningInfo import org.stypox.dicio.skills.lyrics.LyricsInfo import org.stypox.dicio.skills.media.MediaInfo import org.stypox.dicio.skills.navigation.NavigationInfo @@ -49,6 +50,7 @@ class SkillHandler @Inject constructor( TimerInfo, CurrentTimeInfo, MediaInfo, + ListeningInfo, ) // TODO add more fallback skills (e.g. search) diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningInfo.kt b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningInfo.kt new file mode 100644 index 000000000..3d6b643e9 --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningInfo.kt @@ -0,0 +1,32 @@ +package org.stypox.dicio.skills.listening + +import android.content.Context +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Hearing +import androidx.compose.runtime.Composable +import androidx.compose.ui.graphics.vector.rememberVectorPainter +import org.dicio.skill.context.SkillContext +import org.dicio.skill.skill.Skill +import org.dicio.skill.skill.SkillInfo +import org.stypox.dicio.R +import org.stypox.dicio.sentences.Sentences + +object ListeningInfo : SkillInfo("listening") { + override fun name(context: Context) = + context.getString(R.string.skill_name_listening) + + override fun sentenceExample(context: Context) = + context.getString(R.string.skill_sentence_example_listening) + + @Composable + override fun icon() = + rememberVectorPainter(Icons.Default.Hearing) + + override fun isAvailable(ctx: SkillContext): Boolean { + return Sentences.Listening[ctx.sentencesLanguage] != null + } + + override fun build(ctx: SkillContext): Skill<*> { + return ListeningSkill(ListeningInfo, Sentences.Listening[ctx.sentencesLanguage]!!) + } +} diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningOutput.kt b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningOutput.kt new file mode 100644 index 000000000..6d1010464 --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningOutput.kt @@ -0,0 +1,25 @@ +package org.stypox.dicio.skills.listening + +import org.dicio.skill.context.SkillContext +import org.stypox.dicio.R +import org.stypox.dicio.io.graphical.HeadlineSpeechSkillOutput +import org.stypox.dicio.util.getString + +class ListeningOutput( + private val previousEnable: Boolean, + private val newEnable: Boolean +) : HeadlineSpeechSkillOutput { + override fun getSpeechOutput(ctx: SkillContext): String = if (previousEnable) { + if (newEnable) { + ctx.getString(R.string.skill_listening_already_listening) + } else { + ctx.getString(R.string.skill_listening_stop_listening) + } + } else { + if (newEnable) { + ctx.getString(R.string.skill_listening_started_listening) + } else { + ctx.getString(R.string.skill_listening_not_listening) + } + } +} diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt new file mode 100644 index 000000000..81141e27b --- /dev/null +++ b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt @@ -0,0 +1,23 @@ +package org.stypox.dicio.skills.listening + +import org.dicio.skill.context.SkillContext +import org.dicio.skill.skill.SkillInfo +import org.dicio.skill.skill.SkillOutput +import org.dicio.skill.standard.StandardRecognizerData +import org.dicio.skill.standard.StandardRecognizerSkill +import org.stypox.dicio.sentences.Sentences.Listening + +class ListeningSkill(correspondingSkillInfo: SkillInfo, data: StandardRecognizerData) : + StandardRecognizerSkill(correspondingSkillInfo, data) { + + override suspend fun generateOutput(ctx: SkillContext, inputData: Listening): SkillOutput { + val newEnable = when (inputData) { + is Listening.Start -> true + is Listening.Stop -> false + } + if (ctx.settingsAccess.wakeDeviceEnabled != newEnable) { + ctx.settingsAccess.wakeDeviceEnabled = newEnable + } + return ListeningOutput(ctx.settingsAccess.wakeDeviceEnabled, newEnable) + } +} diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 1821daa77..d9aeaa215 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -203,4 +203,10 @@ Retry Stop Grant permissions + I am still listening for the wake word + Okay, I\'ll listen for the wake word + I wasn\'t listening + Okay + Control listening + Stop listening diff --git a/app/src/main/sentences/en/listening.yml b/app/src/main/sentences/en/listening.yml new file mode 100644 index 000000000..ce34322f6 --- /dev/null +++ b/app/src/main/sentences/en/listening.yml @@ -0,0 +1,13 @@ +stop: + - stop listening + - mute + - mute microphone + - turn off microphone + +start: + - start listening + - unmute + - on mute + - unmute microphone + - on mute microphone + - turn on microphone diff --git a/app/src/main/sentences/skill_definitions.yml b/app/src/main/sentences/skill_definitions.yml index 4785b7d55..38d6e63f5 100644 --- a/app/src/main/sentences/skill_definitions.yml +++ b/app/src/main/sentences/skill_definitions.yml @@ -53,6 +53,12 @@ skills: - id: what type: string + - id: listening + specificity: high + sentences: + - id: stop + - id: start + - id: search specificity: low sentences: diff --git a/skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt b/skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt new file mode 100644 index 000000000..b37d4aaad --- /dev/null +++ b/skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt @@ -0,0 +1,5 @@ +package org.dicio.skill.context; + +class NoSettingsAccess : SettingsAccess { + override var wakeDeviceEnabled: Boolean = false +} diff --git a/skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt b/skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt new file mode 100644 index 000000000..d619521cd --- /dev/null +++ b/skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt @@ -0,0 +1,5 @@ +package org.dicio.skill.context + +interface SettingsAccess { + var wakeDeviceEnabled: Boolean +} diff --git a/skill/src/main/java/org/dicio/skill/context/SkillContext.kt b/skill/src/main/java/org/dicio/skill/context/SkillContext.kt index 5fb8608c3..0285635f7 100644 --- a/skill/src/main/java/org/dicio/skill/context/SkillContext.kt +++ b/skill/src/main/java/org/dicio/skill/context/SkillContext.kt @@ -40,4 +40,6 @@ interface SkillContext { * The [SpeechOutputDevice] that should be used for skill speech output. */ val speechOutputDevice: SpeechOutputDevice + + val settingsAccess: SettingsAccess } From 13ce8aec72aaa2bd4f801e8c54cf1c18a6835a21 Mon Sep 17 00:00:00 2001 From: "Cel A. Skeggs" Date: Fri, 4 Apr 2025 14:24:12 -0700 Subject: [PATCH 2/2] Address pull request comments --- .../stypox/dicio/di/SettingsAccessWrapper.kt | 45 ------------------- .../org/stypox/dicio/di/SkillContextImpl.kt | 8 +--- .../org/stypox/dicio/io/wake/WakeService.kt | 19 +++++++- .../dicio/skills/listening/ListeningSkill.kt | 12 +++-- app/src/main/res/values/strings.xml | 2 +- app/src/main/sentences/en/listening.yml | 8 +--- .../dicio/skill/context/NoSettingsAccess.kt | 5 --- .../org/dicio/skill/context/SettingsAccess.kt | 5 --- .../org/dicio/skill/context/SkillContext.kt | 2 - 9 files changed, 30 insertions(+), 76 deletions(-) delete mode 100644 app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt delete mode 100644 skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt delete mode 100644 skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt diff --git a/app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt b/app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt deleted file mode 100644 index 10e1ccf43..000000000 --- a/app/src/main/kotlin/org/stypox/dicio/di/SettingsAccessWrapper.kt +++ /dev/null @@ -1,45 +0,0 @@ -package org.stypox.dicio.di - -import androidx.datastore.core.DataStore -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.dicio.skill.context.SettingsAccess -import org.stypox.dicio.settings.datastore.UserSettings -import org.stypox.dicio.settings.datastore.WakeDevice -import org.stypox.dicio.settings.datastore.copy -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SettingsAccessWrapper @Inject constructor( - private val dataStore: DataStore, -) : SettingsAccess { - - private val scope = CoroutineScope(Dispatchers.Main) - private var lastObservedSettings: UserSettings = UserSettings.getDefaultInstance() - - init { - scope.launch { - dataStore.data.collect { - lastObservedSettings = it - } - } - } - - override var wakeDeviceEnabled: Boolean - get() = lastObservedSettings.wakeDevice != WakeDevice.WAKE_DEVICE_NOTHING - set(value) { - scope.launch { - dataStore.updateData { - it.copy { - wakeDevice = if (value) { - WakeDevice.WAKE_DEVICE_OWW - } else { - WakeDevice.WAKE_DEVICE_NOTHING - } - } - } - } - } -} diff --git a/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt b/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt index fe0c0c68f..db63d512a 100644 --- a/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt +++ b/app/src/main/kotlin/org/stypox/dicio/di/SkillContextImpl.kt @@ -2,11 +2,8 @@ package org.stypox.dicio.di import android.annotation.SuppressLint import android.content.Context -import android.provider.Settings import dagger.hilt.android.qualifiers.ApplicationContext import org.dicio.numbers.ParserFormatter -import org.dicio.skill.context.NoSettingsAccess -import org.dicio.skill.context.SettingsAccess import org.dicio.skill.context.SkillContext import org.dicio.skill.context.SpeechOutputDevice import org.stypox.dicio.io.speech.NothingSpeechDevice @@ -21,7 +18,6 @@ class SkillContextImpl private constructor( // this constructor can take any SpeechOutputDevice to allow newForPreviews to provide // NothingSpeechDevice override val speechOutputDevice: SpeechOutputDevice, - override val settingsAccess: SettingsAccess, ) : SkillContext { @Inject @@ -29,8 +25,7 @@ class SkillContextImpl private constructor( @ApplicationContext android: Context, localeManager: LocaleManager, speechOutputDevice: SpeechOutputDeviceWrapper, - settingsAccess: SettingsAccessWrapper, - ) : this(android, localeManager, speechOutputDevice as SpeechOutputDevice, settingsAccess as SettingsAccess) + ) : this(android, localeManager, speechOutputDevice as SpeechOutputDevice) override val locale: Locale @@ -63,7 +58,6 @@ class SkillContextImpl private constructor( context, localeManager, NothingSpeechDevice(), - NoSettingsAccess(), ) @SuppressLint("StateFlowValueCalledInComposition") res.lastParserFormatter = Pair( diff --git a/app/src/main/kotlin/org/stypox/dicio/io/wake/WakeService.kt b/app/src/main/kotlin/org/stypox/dicio/io/wake/WakeService.kt index 167741940..479b7aee0 100644 --- a/app/src/main/kotlin/org/stypox/dicio/io/wake/WakeService.kt +++ b/app/src/main/kotlin/org/stypox/dicio/io/wake/WakeService.kt @@ -36,6 +36,7 @@ import org.stypox.dicio.di.WakeDeviceWrapper import org.stypox.dicio.eval.SkillEvaluator import java.time.Instant import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference import javax.inject.Inject @AndroidEntryPoint @@ -44,7 +45,7 @@ class WakeService : Service() { private val job = SupervisorJob() private val scope = CoroutineScope(Dispatchers.Default + job) - private var listening = AtomicBoolean(false) + private val listening = AtomicBoolean(false) @Inject lateinit var skillEvaluator: SkillEvaluator @@ -196,6 +197,8 @@ class WakeService : Service() { nextWakeWordAllowed = Instant.now().plusMillis(WAKE_WORD_BACKOFF_MILLIS) onWakeWordDetected() } + + lastHeard.set(Instant.now()) } } finally { ar.stop() @@ -309,6 +312,18 @@ class WakeService : Service() { ContextCompat.startForegroundService(context, intent) } + fun stop(context: Context) { + try { + context.startService(Intent(context, WakeService::class.java) + .apply { action = ACTION_STOP_WAKE_SERVICE }) + } catch (e: IllegalStateException) { + // Must not have been running. No problem with that. + } + } + + // Consider the service running if it processed any audio data within the past half second. + fun isRunning(): Boolean = lastHeard.get().isAfter(Instant.now().minusMillis(500)) + /** * On Android 10+ cancels any notification telling the user that the Dicio wake word was * triggered, which is not needed anymore after the main activity starts. @@ -320,6 +335,8 @@ class WakeService : Service() { } } + private val lastHeard = AtomicReference() + private val TAG = WakeService::class.simpleName private const val FOREGROUND_NOTIFICATION_CHANNEL_ID = "org.stypox.dicio.io.wake.WakeService.FOREGROUND" diff --git a/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt index 81141e27b..49b180e2d 100644 --- a/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt +++ b/app/src/main/kotlin/org/stypox/dicio/skills/listening/ListeningSkill.kt @@ -5,19 +5,23 @@ import org.dicio.skill.skill.SkillInfo import org.dicio.skill.skill.SkillOutput import org.dicio.skill.standard.StandardRecognizerData import org.dicio.skill.standard.StandardRecognizerSkill +import org.stypox.dicio.io.wake.WakeService import org.stypox.dicio.sentences.Sentences.Listening class ListeningSkill(correspondingSkillInfo: SkillInfo, data: StandardRecognizerData) : StandardRecognizerSkill(correspondingSkillInfo, data) { override suspend fun generateOutput(ctx: SkillContext, inputData: Listening): SkillOutput { - val newEnable = when (inputData) { + val wasRunning = WakeService.isRunning() + val shouldBeRunning = when (inputData) { is Listening.Start -> true is Listening.Stop -> false } - if (ctx.settingsAccess.wakeDeviceEnabled != newEnable) { - ctx.settingsAccess.wakeDeviceEnabled = newEnable + if (shouldBeRunning) { + WakeService.start(ctx.android) + } else { + WakeService.stop(ctx.android) } - return ListeningOutput(ctx.settingsAccess.wakeDeviceEnabled, newEnable) + return ListeningOutput(wasRunning, shouldBeRunning) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index d9aeaa215..688f367cb 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -207,6 +207,6 @@ Okay, I\'ll listen for the wake word I wasn\'t listening Okay - Control listening + Control wake word listening Stop listening diff --git a/app/src/main/sentences/en/listening.yml b/app/src/main/sentences/en/listening.yml index ce34322f6..85dde3ed9 100644 --- a/app/src/main/sentences/en/listening.yml +++ b/app/src/main/sentences/en/listening.yml @@ -1,13 +1,9 @@ stop: - stop listening - - mute - - mute microphone + - mute microphone? - turn off microphone start: - start listening - - unmute - - on mute - - unmute microphone - - on mute microphone + - unmute|(on mute) microphone? - turn on microphone diff --git a/skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt b/skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt deleted file mode 100644 index b37d4aaad..000000000 --- a/skill/src/main/java/org/dicio/skill/context/NoSettingsAccess.kt +++ /dev/null @@ -1,5 +0,0 @@ -package org.dicio.skill.context; - -class NoSettingsAccess : SettingsAccess { - override var wakeDeviceEnabled: Boolean = false -} diff --git a/skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt b/skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt deleted file mode 100644 index d619521cd..000000000 --- a/skill/src/main/java/org/dicio/skill/context/SettingsAccess.kt +++ /dev/null @@ -1,5 +0,0 @@ -package org.dicio.skill.context - -interface SettingsAccess { - var wakeDeviceEnabled: Boolean -} diff --git a/skill/src/main/java/org/dicio/skill/context/SkillContext.kt b/skill/src/main/java/org/dicio/skill/context/SkillContext.kt index 0285635f7..5fb8608c3 100644 --- a/skill/src/main/java/org/dicio/skill/context/SkillContext.kt +++ b/skill/src/main/java/org/dicio/skill/context/SkillContext.kt @@ -40,6 +40,4 @@ interface SkillContext { * The [SpeechOutputDevice] that should be used for skill speech output. */ val speechOutputDevice: SpeechOutputDevice - - val settingsAccess: SettingsAccess }