From 6f63a6d710f27bb8d81c17259504cc65cee21a2d Mon Sep 17 00:00:00 2001 From: Kelrap <99418823+Kelrap@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:15:17 -0500 Subject: [PATCH] Highlight audio text (#1333) * Highlight text as TTS plays - attempt 1 * Make highlighting actually work * Fix to minor version of punctuation issue * Highlights all applicable text * fix: filter out punctuation tokens in the client side when highlighing audio tokens * Highlight selection separate from normal selection * cleanup: further decouple tts highlighting and token selection, renamed temporarySelection => _highlightedTokens * fix: don't show token highlights for non-overlay messages --------- Co-authored-by: ggurdin <46800240+ggurdin@users.noreply.github.com> Co-authored-by: ggurdin --- lib/pages/chat/events/audio_player.dart | 12 +++++++ lib/pages/chat/events/message_content.dart | 7 +++- .../repo/lemma_activity_generator.dart | 2 +- .../toolbar/widgets/message_audio_card.dart | 1 + .../widgets/message_selection_overlay.dart | 36 +++++++++++++++++-- 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/lib/pages/chat/events/audio_player.dart b/lib/pages/chat/events/audio_player.dart index 42002dd5f..536e5f61b 100644 --- a/lib/pages/chat/events/audio_player.dart +++ b/lib/pages/chat/events/audio_player.dart @@ -13,6 +13,7 @@ import 'package:path_provider/path_provider.dart'; import 'package:fluffychat/config/app_config.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; +import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; import 'package:fluffychat/utils/error_reporter.dart'; import 'package:fluffychat/utils/localized_exception_extension.dart'; import 'package:fluffychat/utils/url_launcher.dart'; @@ -28,6 +29,7 @@ class AudioPlayerWidget extends StatefulWidget { final bool autoplay; final Function(bool)? setIsPlayingAudio; final double padding; + final MessageOverlayController? overlayController; // Pangea# static String? currentId; @@ -50,6 +52,7 @@ class AudioPlayerWidget extends StatefulWidget { this.sectionEndMS, this.setIsPlayingAudio, this.padding = 12.0, + this.overlayController, // Pangea# super.key, }); @@ -187,6 +190,15 @@ class AudioPlayerState extends State { audioPlayer.stop(); audioPlayer.seek(null); } + // #Pangea + // Pass current timestamp to overlay, so it can highlight as necessary + if (widget.matrixFile != null) { + widget.overlayController?.highlightCurrentText( + state.inMilliseconds, + widget.matrixFile!.tokens, + ); + } + // Pangea# }); onDurationChanged ??= audioPlayer.durationStream.listen((max) { if (max == null || max == Duration.zero) return; diff --git a/lib/pages/chat/events/message_content.dart b/lib/pages/chat/events/message_content.dart index 91fd9ea1f..215d57f70 100644 --- a/lib/pages/chat/events/message_content.dart +++ b/lib/pages/chat/events/message_content.dart @@ -337,7 +337,12 @@ class MessageContent extends StatelessWidget { selectedToken: token, ); }, - isSelected: overlayController?.isTokenSelected, + isSelected: overlayController != null + ? (token) { + return overlayController!.isTokenSelected(token) || + overlayController!.isTokenHighlighted(token); + } + : null, ); } diff --git a/lib/pangea/toolbar/repo/lemma_activity_generator.dart b/lib/pangea/toolbar/repo/lemma_activity_generator.dart index 1d13713f8..8c3adf8df 100644 --- a/lib/pangea/toolbar/repo/lemma_activity_generator.dart +++ b/lib/pangea/toolbar/repo/lemma_activity_generator.dart @@ -64,7 +64,7 @@ class LemmaActivityGenerator { final choices = sortedLemmas.take(4).toList(); if (!choices.contains(token.lemma.text)) { final random = Random(); - choices[random.nextInt(4)] = token.lemma.text; + choices[random.nextInt(choices.length - 1)] = token.lemma.text; } return choices; } diff --git a/lib/pangea/toolbar/widgets/message_audio_card.dart b/lib/pangea/toolbar/widgets/message_audio_card.dart index 0a4ea7ecd..c18e498e3 100644 --- a/lib/pangea/toolbar/widgets/message_audio_card.dart +++ b/lib/pangea/toolbar/widgets/message_audio_card.dart @@ -201,6 +201,7 @@ class MessageAudioCardState extends State { fontSize: AppConfig.messageFontSize * AppConfig.fontSizeFactor, padding: 0, + overlayController: widget.overlayController, ) : const CardErrorWidget( error: "Null audio file in message_audio_card", diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 985b7088e..5f907c03c 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -17,6 +17,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_text_model.dart'; +import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.dart'; import 'package:fluffychat/pangea/toolbar/enums/activity_type_enum.dart'; import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_toolbar.dart'; @@ -61,6 +62,7 @@ class MessageOverlayController extends State MessageMode toolbarMode = MessageMode.noneSelected; PangeaTokenText? _selectedSpan; + List? _highlightedTokens; List? tokens; bool initialized = false; @@ -129,6 +131,26 @@ class MessageOverlayController extends State setState(() {}); } + /// If sentence TTS is playing a word, highlight that word in message overlay + void highlightCurrentText(int currentPosition, List ttsTokens) { + final List textToSelect = []; + // Check if current time is between start and end times of tokens + for (final TTSToken token in ttsTokens) { + if (token.endMS > currentPosition) { + if (token.startMS < currentPosition) { + textToSelect.add(token); + } else { + break; + } + } + } + + if (const ListEquality().equals(textToSelect, _highlightedTokens)) return; + _highlightedTokens = + textToSelect.isEmpty ? null : textToSelect.map((t) => t.text).toList(); + setState(() {}); + } + void _setupSubscriptions() { _animationController = AnimationController( vsync: this, @@ -278,6 +300,9 @@ class MessageOverlayController extends State debugPrint("updateToolbarMode: $mode - clearing selectedSpan"); _selectedSpan = null; } + if (mode != MessageMode.textToSpeech) { + _highlightedTokens = null; + } toolbarMode = mode; }); } @@ -327,17 +352,24 @@ class MessageOverlayController extends State setState(() {}); } - /// Whether the given token is currently selected + /// Whether the given token is currently selected or highlighted bool isTokenSelected(PangeaToken token) { final isSelected = _selectedSpan?.offset == token.text.offset && _selectedSpan?.length == token.text.length; return isSelected; } + bool isTokenHighlighted(PangeaToken token) { + if (_highlightedTokens == null) return false; + return _highlightedTokens!.any( + (t) => t.offset == token.text.offset && t.length == token.text.length, + ); + } + PangeaToken? get selectedToken => tokens?.firstWhereOrNull(isTokenSelected); /// Whether the overlay is currently displaying a selection - bool get isSelection => _selectedSpan != null; + bool get isSelection => _selectedSpan != null || _highlightedTokens != null; PangeaTokenText? get selectedSpan => _selectedSpan;