From 4a7e9dade9390004ce77389f6443378158fae87e Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 10 Jun 2025 12:52:38 -0400 Subject: [PATCH 01/12] feat: add toolbar buttons for audio messages --- .../widgets/message_selection_overlay.dart | 16 ++ .../widgets/message_selection_positioner.dart | 20 ++- .../widgets/message_speech_to_text_card.dart | 119 ------------- .../toolbar/widgets/overlay_message.dart | 50 ++++-- .../toolbar/widgets/select_mode_buttons.dart | 168 +++++++++++++----- 5 files changed, 189 insertions(+), 184 deletions(-) delete mode 100644 lib/pangea/toolbar/widgets/message_speech_to_text_card.dart diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 844eafc26..4087457f6 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -91,9 +91,13 @@ class MessageOverlayController extends State final GlobalKey wordZoomKey = GlobalKey(); ReadingAssistanceMode? readingAssistanceMode; // default mode + bool showTranslation = false; String? translationText; + bool showTranscription = false; + String? transcriptText; + double maxWidth = AppConfig.toolbarMinWidth; ///////////////////////////////////// @@ -589,6 +593,18 @@ class MessageOverlayController extends State } } + void setShowTranscription(bool show, String? transcription) { + if (showTranscription == show) return; + if (show && transcription == null) return; + + if (mounted) { + setState(() { + showTranscription = show; + transcriptText = show ? transcription : null; + }); + } + } + ///////////////////////////////////// /// Build ///////////////////////////////////// diff --git a/lib/pangea/toolbar/widgets/message_selection_positioner.dart b/lib/pangea/toolbar/widgets/message_selection_positioner.dart index 53a8436ac..555154ad8 100644 --- a/lib/pangea/toolbar/widgets/message_selection_positioner.dart +++ b/lib/pangea/toolbar/widgets/message_selection_positioner.dart @@ -490,10 +490,22 @@ class MessageSelectionPositionerState extends State // measurement for items in the toolbar - bool get _showButtons => - (widget.pangeaMessageEvent?.shouldShowToolbar ?? false) && - widget.pangeaMessageEvent?.event.messageType == MessageTypes.Text && - (widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false); + bool get _showButtons { + if (!(widget.pangeaMessageEvent?.shouldShowToolbar ?? false)) { + return false; + } + + final type = widget.pangeaMessageEvent?.event.messageType; + if (![MessageTypes.Text, MessageTypes.Audio].contains(type)) { + return false; + } + + if (type == MessageTypes.Text) { + return widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false; + } + + return true; + } bool get showPracticeButtons => _showButtons && diff --git a/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart b/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart deleted file mode 100644 index 39f06683d..000000000 --- a/lib/pangea/toolbar/widgets/message_speech_to_text_card.dart +++ /dev/null @@ -1,119 +0,0 @@ -import 'dart:developer'; - -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; - -import 'package:fluffychat/config/app_config.dart'; -import 'package:fluffychat/l10n/l10n.dart'; -import 'package:fluffychat/pangea/common/utils/error_handler.dart'; -import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; -import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; -import 'package:fluffychat/widgets/matrix.dart'; - -class MessageSpeechToTextCard extends StatefulWidget { - final PangeaMessageEvent messageEvent; - final Color textColor; - - const MessageSpeechToTextCard({ - super.key, - required this.messageEvent, - required this.textColor, - }); - - @override - MessageSpeechToTextCardState createState() => MessageSpeechToTextCardState(); -} - -class MessageSpeechToTextCardState extends State { - SpeechToTextModel? _speechToTextResponse; - - bool _fetchingTranscription = true; - Object? error; - - String? get l1Code => - MatrixState.pangeaController.languageController.activeL1Code(); - String? get l2Code => - MatrixState.pangeaController.languageController.activeL2Code(); - - @override - void initState() { - super.initState(); - _fetchTranscription(); - } - - // look for transcription in message event - // if not found, call API to transcribe audio - Future _fetchTranscription() async { - try { - if (l1Code == null || l2Code == null) { - throw Exception('Language selection not found'); - } - - _speechToTextResponse ??= await widget.messageEvent.getSpeechToText( - l1Code!, - l2Code!, - ); - } catch (e, s) { - debugger(when: kDebugMode); - error = e; - ErrorHandler.logError( - e: e, - s: s, - data: widget.messageEvent.event.content, - ); - } finally { - if (mounted) { - setState(() => _fetchingTranscription = false); - } - } - } - - @override - Widget build(BuildContext context) { - if (_fetchingTranscription) { - return const LinearProgressIndicator(); - } - - // // done fetching but not results means some kind of error - if (_speechToTextResponse == null || error != null) { - return Row( - spacing: 8.0, - children: [ - Flexible( - child: RichText( - text: TextSpan( - style: AppConfig.messageTextStyle( - widget.messageEvent.event, - widget.textColor, - ), - children: [ - WidgetSpan( - alignment: PlaceholderAlignment.middle, - child: Icon( - Icons.error, - color: Theme.of(context).colorScheme.error, - ), - ), - const TextSpan(text: " "), - TextSpan( - text: L10n.of(context).oopsSomethingWentWrong, - ), - ], - ), - ), - ), - ], - ); - } - - return Text( - "${_speechToTextResponse?.transcript.text}", - style: AppConfig.messageTextStyle( - widget.messageEvent.event, - widget.textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ); - } -} diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index b5da23fc4..8b4ccb611 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -11,7 +11,6 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dar import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; -import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart'; import 'package:fluffychat/utils/date_time_extension.dart'; import 'package:fluffychat/utils/file_description.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -137,7 +136,8 @@ class OverlayMessage extends StatelessWidget { final showTranslation = overlayController.showTranslation && overlayController.translationText != null; - final showTranscription = pangeaMessageEvent?.isAudioMessage == true; + final showTranscription = overlayController.showTranscription && + overlayController.transcriptText != null; final content = Container( decoration: BoxDecoration( @@ -270,6 +270,27 @@ class OverlayMessage extends StatelessWidget { child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ + if (showTranscription) + Container( + width: messageWidth, + constraints: const BoxConstraints( + maxHeight: AppConfig.audioTranscriptionMaxHeight, + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: SingleChildScrollView( + child: Text( + overlayController.transcriptText!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), + ), + ), + ), sizeAnimation != null ? AnimatedBuilder( animation: sizeAnimation!, @@ -282,7 +303,7 @@ class OverlayMessage extends StatelessWidget { }, ) : content, - if (showTranscription || showTranslation) + if (showTranslation) Container( width: messageWidth, constraints: const BoxConstraints( @@ -296,20 +317,15 @@ class OverlayMessage extends StatelessWidget { 12.0, ), child: SingleChildScrollView( - child: showTranscription - ? MessageSpeechToTextCard( - messageEvent: pangeaMessageEvent!, - textColor: textColor, - ) - : Text( - overlayController.translationText!, - style: AppConfig.messageTextStyle( - event, - textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ), + child: Text( + overlayController.translationText!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), ), ), ), diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 9a5a4b700..ea93660c7 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -18,6 +18,7 @@ import 'package:fluffychat/pangea/common/widgets/pressable_button.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -25,7 +26,8 @@ import 'package:fluffychat/widgets/matrix.dart'; enum SelectMode { audio(Icons.volume_up), translate(Icons.translate), - practice(Symbols.fitness_center); + practice(Symbols.fitness_center), + transcription(Icons.translate); final IconData icon; const SelectMode(this.icon); @@ -39,6 +41,8 @@ enum SelectMode { return l10n.translationTooltip; case SelectMode.practice: return l10n.practice; + case SelectMode.transcription: + return l10n.speechToTextTooltip; } } } @@ -61,6 +65,17 @@ class SelectModeButtonsState extends State { static const double iconWidth = 36.0; static const double buttonSize = 40.0; + static List get textModes => [ + SelectMode.audio, + SelectMode.translate, + SelectMode.practice, + ]; + + static List get audioModes => [ + SelectMode.transcription, + SelectMode.practice, + ]; + SelectMode? _selectedMode; final AudioPlayer _audioPlayer = AudioPlayer(); @@ -74,6 +89,11 @@ class SelectModeButtonsState extends State { bool _isLoadingTranslation = false; PangeaRepresentation? _repEvent; + String? _translationError; + + bool _isLoadingTranscription = false; + SpeechToTextModel? _speechToTextResponse; + String? _transcriptionError; @override void initState() { @@ -112,12 +132,21 @@ class SelectModeButtonsState extends State { MatrixState.pangeaController.languageController.activeL2Code(); void _clear() { - setState(() => _audioError = null); + setState(() { + _audioError = null; + _translationError = null; + _transcriptionError = null; + }); + widget.overlayController.updateSelectedSpan(null); if (_selectedMode == SelectMode.translate) { widget.overlayController.setShowTranslation(false, null); } + + if (_selectedMode == SelectMode.transcription) { + widget.overlayController.setShowTranscription(false, null); + } } Future _updateMode(SelectMode? mode) async { @@ -158,6 +187,15 @@ class SelectModeButtonsState extends State { _repEvent!.text, ); } + + if (_selectedMode == SelectMode.transcription) { + await _loadTranscription(); + if (_speechToTextResponse == null) return; + widget.overlayController.setShowTranscription( + true, + _speechToTextResponse!.transcript.text, + ); + } } Future _fetchAudio() async { @@ -257,6 +295,17 @@ class SelectModeButtonsState extends State { } } + Future _fetchTranscription() async { + if (l1Code == null || messageEvent == null || _repEvent != null) { + return; + } + + _speechToTextResponse ??= await messageEvent!.getSpeechToText( + l1Code!, + l2Code!, + ); + } + Future _loadTranslation() async { if (!mounted) return; setState(() => _isLoadingTranslation = true); @@ -264,6 +313,7 @@ class SelectModeButtonsState extends State { try { await _fetchRepresentation(); } catch (err) { + _translationError = err.toString(); ErrorHandler.logError( e: err, data: {}, @@ -275,50 +325,78 @@ class SelectModeButtonsState extends State { } } + Future _loadTranscription() async { + if (!mounted) return; + setState(() => _isLoadingTranscription = true); + + try { + await _fetchTranscription(); + } catch (err) { + _transcriptionError = err.toString(); + ErrorHandler.logError( + e: err, + data: {}, + ); + } + + if (mounted) { + setState(() => _isLoadingTranscription = false); + } + } + + bool get _isError { + switch (_selectedMode) { + case SelectMode.audio: + return _audioError != null; + case SelectMode.translate: + return _translationError != null; + case SelectMode.transcription: + return _transcriptionError != null; + default: + return false; + } + } + + bool get _isLoading { + switch (_selectedMode) { + case SelectMode.audio: + return _isLoadingAudio; + case SelectMode.translate: + return _isLoadingTranslation; + case SelectMode.transcription: + return _isLoadingTranscription; + default: + return false; + } + } + Widget icon(SelectMode mode) { - if (mode == SelectMode.audio) { - if (_audioError != null) { - return Icon( - Icons.error_outline, - size: 20, - color: Theme.of(context).colorScheme.error, - ); - } - if (_isLoadingAudio) { - return const Center( - child: SizedBox( - height: 20.0, - width: 20.0, - child: CircularProgressIndicator.adaptive(), - ), - ); - } else { - return Icon( - _audioPlayer.playerState.playing == true - ? Icons.pause_outlined - : Icons.volume_up, - size: 20, - color: mode == _selectedMode ? Colors.white : null, - ); - } + if (_isError && mode == _selectedMode) { + return Icon( + Icons.error_outline, + size: 20, + color: Theme.of(context).colorScheme.error, + ); } - if (mode == SelectMode.translate) { - if (_isLoadingTranslation) { - return const Center( - child: SizedBox( - height: 20.0, - width: 20.0, - child: CircularProgressIndicator.adaptive(), - ), - ); - } else if (_repEvent != null) { - return Icon( - mode.icon, - size: 20, - color: mode == _selectedMode ? Colors.white : null, - ); - } + if (_isLoading && mode == _selectedMode) { + return const Center( + child: SizedBox( + height: 20.0, + width: 20.0, + child: CircularProgressIndicator.adaptive(), + ), + ); + } + + if (mode == SelectMode.audio) { + return Icon( + _audioPlayer.playerState.playing == true + ? Icons.pause_outlined + : Icons.volume_up, + size: 20, + color: mode == _selectedMode ? Colors.white : null, + ); } return Icon( @@ -330,6 +408,8 @@ class SelectModeButtonsState extends State { @override Widget build(BuildContext context) { + final modes = messageEvent?.isAudioMessage == true ? audioModes : textModes; + return Container( height: AppConfig.toolbarButtonsHeight, alignment: Alignment.bottomCenter, @@ -338,7 +418,7 @@ class SelectModeButtonsState extends State { mainAxisSize: MainAxisSize.min, spacing: 4.0, children: [ - for (final mode in SelectMode.values) + for (final mode in modes) Tooltip( message: mode.tooltip(context), child: PressableButton( From f7d8a098453a8a3d5edcec9165a9dff063915851 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 10 Jun 2025 14:37:59 -0400 Subject: [PATCH 02/12] show overlay and play maudio with one click --- lib/pages/chat/events/audio_player.dart | 31 ++++++++++--------- lib/pages/chat/events/message_content.dart | 9 ++++++ .../practice_match_card.dart | 1 - .../toolbar/widgets/message_audio_card.dart | 3 -- .../widgets/message_selection_overlay.dart | 7 ----- .../widgets/message_selection_positioner.dart | 3 ++ .../multiple_choice_activity.dart | 4 +-- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/pages/chat/events/audio_player.dart b/lib/pages/chat/events/audio_player.dart index 9dae24462..9a82dce86 100644 --- a/lib/pages/chat/events/audio_player.dart +++ b/lib/pages/chat/events/audio_player.dart @@ -22,7 +22,7 @@ import 'package:fluffychat/utils/file_description.dart'; import 'package:fluffychat/utils/localized_exception_extension.dart'; import 'package:fluffychat/utils/matrix_sdk_extensions/event_extension.dart'; import 'package:fluffychat/utils/url_launcher.dart'; -import '../../../widgets/fluffy_chat_app.dart'; +import 'package:fluffychat/widgets/fluffy_chat_app.dart'; import '../../../widgets/matrix.dart'; class AudioPlayerWidget extends StatefulWidget { @@ -36,9 +36,10 @@ class AudioPlayerWidget extends StatefulWidget { final String roomId; final String senderId; final PangeaAudioFile? matrixFile; - final Function(bool)? setIsPlayingAudio; final ChatController chatController; final MessageOverlayController? overlayController; + final VoidCallback? onPlay; + final bool autoplay; // Pangea# static const int wavesCount = 40; @@ -53,9 +54,10 @@ class AudioPlayerWidget extends StatefulWidget { required this.roomId, required this.senderId, this.matrixFile, - this.setIsPlayingAudio, required this.chatController, this.overlayController, + this.onPlay, + this.autoplay = false, // Pangea# super.key, }); @@ -76,9 +78,7 @@ class AudioPlayerState extends State { String? _durationString; // #Pangea - StreamSubscription? _onShowToolbar; StreamSubscription? _onAudioPositionChanged; - StreamSubscription? _onPlayerStateChanged; // Pangea# @override @@ -163,9 +163,7 @@ class AudioPlayerState extends State { audioPlayer.dispose(); matrix.voiceMessageEventId.value = matrix.audioPlayer = null; // #Pangea - _onShowToolbar?.cancel(); _onAudioPositionChanged?.cancel(); - _onPlayerStateChanged?.cancel(); // Pangea# } } @@ -253,6 +251,8 @@ class AudioPlayerState extends State { // #Pangea // if (matrix.voiceMessageEventId.value != widget.event.eventId) return; if (matrix.voiceMessageEventId.value != widget.eventId) return; + + matrix.audioPlayer?.dispose(); // Pangea# final audioPlayer = matrix.audioPlayer = AudioPlayer(); @@ -269,13 +269,6 @@ class AudioPlayerState extends State { ); } }); - - _onPlayerStateChanged?.cancel(); - _onPlayerStateChanged = audioPlayer.playingStream.listen( - (isPlaying) => setState(() { - widget.setIsPlayingAudio?.call(isPlaying); - }), - ); // Pangea# // #Pangea @@ -394,6 +387,10 @@ class AudioPlayerState extends State { final duration = Duration(milliseconds: durationInt); _durationString = duration.minuteSecondString; } + + // #Pangea + if (widget.autoplay) _onButtonTap(); + // Pangea# } @override @@ -465,7 +462,11 @@ class AudioPlayerState extends State { onLongPress: () => widget.event?.saveFile(context), // Pangea# - onTap: _onButtonTap, + onTap: () { + widget.onPlay != null + ? widget.onPlay!.call() + : _onButtonTap(); + }, child: Material( color: widget.color.withAlpha(64), borderRadius: BorderRadius.circular(64), diff --git a/lib/pages/chat/events/message_content.dart b/lib/pages/chat/events/message_content.dart index 05c87bef8..a4098996c 100644 --- a/lib/pages/chat/events/message_content.dart +++ b/lib/pages/chat/events/message_content.dart @@ -225,6 +225,15 @@ class MessageContent extends StatelessWidget { eventId: event.eventId, roomId: event.room.id, senderId: event.senderId, + onPlay: overlayController == null + ? () { + controller.showToolbar( + pangeaMessageEvent!.event, + pangeaMessageEvent: pangeaMessageEvent, + ); + } + : null, + autoplay: overlayController != null, // Pangea# ); } diff --git a/lib/pangea/toolbar/reading_assistance_input_row/practice_match_card.dart b/lib/pangea/toolbar/reading_assistance_input_row/practice_match_card.dart index eb1421b15..4fbbc7384 100644 --- a/lib/pangea/toolbar/reading_assistance_input_row/practice_match_card.dart +++ b/lib/pangea/toolbar/reading_assistance_input_row/practice_match_card.dart @@ -74,7 +74,6 @@ class MatchActivityCard extends StatelessWidget { MessageAudioCard( messageEvent: overlayController.pangeaMessageEvent!, overlayController: overlayController, - setIsPlayingAudio: overlayController.setIsPlayingAudio, ), Wrap( alignment: WrapAlignment.center, diff --git a/lib/pangea/toolbar/widgets/message_audio_card.dart b/lib/pangea/toolbar/widgets/message_audio_card.dart index 0aea47efc..e50a6458c 100644 --- a/lib/pangea/toolbar/widgets/message_audio_card.dart +++ b/lib/pangea/toolbar/widgets/message_audio_card.dart @@ -18,14 +18,12 @@ import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart class MessageAudioCard extends StatefulWidget { final PangeaMessageEvent messageEvent; final MessageOverlayController overlayController; - final Function(bool) setIsPlayingAudio; final VoidCallback? onError; const MessageAudioCard({ super.key, required this.messageEvent, required this.overlayController, - required this.setIsPlayingAudio, this.onError, }); @@ -91,7 +89,6 @@ class MessageAudioCardState extends State { senderId: widget.messageEvent.senderId, matrixFile: audioFile, color: Theme.of(context).colorScheme.onPrimaryContainer, - setIsPlayingAudio: widget.setIsPlayingAudio, fontSize: AppConfig.messageFontSize * AppConfig.fontSizeFactor, chatController: widget.overlayController.widget.chatController, overlayController: widget.overlayController, diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 4087457f6..0eaa7f65a 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -86,7 +86,6 @@ class MessageOverlayController extends State List? _highlightedTokens; bool initialized = false; - bool isPlayingAudio = false; final GlobalKey wordZoomKey = GlobalKey(); @@ -575,12 +574,6 @@ class MessageOverlayController extends State ); } - void setIsPlayingAudio(bool isPlaying) { - if (mounted) { - setState(() => isPlayingAudio = isPlaying); - } - } - void setShowTranslation(bool show, String? translation) { if (showTranslation == show) return; if (show && translation == null) return; diff --git a/lib/pangea/toolbar/widgets/message_selection_positioner.dart b/lib/pangea/toolbar/widgets/message_selection_positioner.dart index 555154ad8..8cca472de 100644 --- a/lib/pangea/toolbar/widgets/message_selection_positioner.dart +++ b/lib/pangea/toolbar/widgets/message_selection_positioner.dart @@ -138,6 +138,9 @@ class MessageSelectionPositionerState extends State void dispose() { _animationController.dispose(); _reactionSubscription?.cancel(); + MatrixState.pangeaController.matrixState.audioPlayer + ?..stop() + ..dispose(); super.dispose(); } diff --git a/lib/pangea/toolbar/widgets/practice_activity/multiple_choice_activity.dart b/lib/pangea/toolbar/widgets/practice_activity/multiple_choice_activity.dart index 3bf05cebf..b9c46062c 100644 --- a/lib/pangea/toolbar/widgets/practice_activity/multiple_choice_activity.dart +++ b/lib/pangea/toolbar/widgets/practice_activity/multiple_choice_activity.dart @@ -236,7 +236,6 @@ class MultipleChoiceActivityState extends State { messageEvent: widget.practiceCardController.widget.pangeaMessageEvent, overlayController: widget.overlayController, - setIsPlayingAudio: widget.overlayController.setIsPlayingAudio, onError: widget.onError, ), ChoicesArray( @@ -247,8 +246,7 @@ class MultipleChoiceActivityState extends State { choices: choices(context), isActive: true, id: currentRecordModel?.hashCode.toString(), - enableAudio: !widget.overlayController.isPlayingAudio && - practiceActivity.activityType.includeTTSOnClick, + enableAudio: practiceActivity.activityType.includeTTSOnClick, langCode: MatrixState.pangeaController.languageController.activeL2Code(), getDisplayCopy: _getDisplayCopy, From 99fd9f9cb077a4a8e9e012e5d29b6b79187d1df4 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 10 Jun 2025 15:01:32 -0400 Subject: [PATCH 03/12] always show transcription --- .../widgets/message_selection_overlay.dart | 20 ++++--- .../toolbar/widgets/overlay_message.dart | 46 ++++++++++----- .../toolbar/widgets/select_mode_buttons.dart | 57 +++++++------------ 3 files changed, 67 insertions(+), 56 deletions(-) diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 0eaa7f65a..73451081f 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -94,8 +94,8 @@ class MessageOverlayController extends State bool showTranslation = false; String? translationText; - bool showTranscription = false; - String? transcriptText; + String? transcriptionText; + String? transcriptionError; double maxWidth = AppConfig.toolbarMinWidth; @@ -586,14 +586,20 @@ class MessageOverlayController extends State } } - void setShowTranscription(bool show, String? transcription) { - if (showTranscription == show) return; - if (show && transcription == null) return; + void setTranscriptionText(String transcription) { + if (mounted) { + setState(() { + transcriptionError = null; + transcriptionText = transcription; + }); + } + } + void setTranscriptionError(String error) { if (mounted) { setState(() { - showTranscription = show; - transcriptText = show ? transcription : null; + transcriptionText = null; + transcriptionError = error; }); } } diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index 8b4ccb611..c4e848851 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -4,6 +4,7 @@ import 'package:flutter/material.dart'; import 'package:matrix/matrix.dart'; import 'package:fluffychat/config/app_config.dart'; +import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pages/chat/chat.dart'; import 'package:fluffychat/pages/chat/events/message_content.dart'; import 'package:fluffychat/pages/chat/events/reply_content.dart'; @@ -136,8 +137,7 @@ class OverlayMessage extends StatelessWidget { final showTranslation = overlayController.showTranslation && overlayController.translationText != null; - final showTranscription = overlayController.showTranscription && - overlayController.transcriptText != null; + final showTranscription = pangeaMessageEvent?.isAudioMessage == true; final content = Container( decoration: BoxDecoration( @@ -278,17 +278,37 @@ class OverlayMessage extends StatelessWidget { ), child: Padding( padding: const EdgeInsets.all(12.0), - child: SingleChildScrollView( - child: Text( - overlayController.transcriptText!, - style: AppConfig.messageTextStyle( - event, - textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ), - ), + child: overlayController.transcriptionError != null + ? Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.error_outline, + color: Theme.of(context).colorScheme.error, + ), + const SizedBox(width: 8), + Text( + L10n.of(context).oopsSomethingWentWrong, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith(fontStyle: FontStyle.italic), + ), + ], + ) + : overlayController.transcriptionText != null + ? SingleChildScrollView( + child: Text( + overlayController.transcriptionText!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), + ) + : const LinearProgressIndicator(), ), ), sizeAnimation != null diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index ea93660c7..2b434a835 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -27,7 +27,7 @@ enum SelectMode { audio(Icons.volume_up), translate(Icons.translate), practice(Symbols.fitness_center), - transcription(Icons.translate); + speechTranslation(Icons.translate); final IconData icon; const SelectMode(this.icon); @@ -41,7 +41,7 @@ enum SelectMode { return l10n.translationTooltip; case SelectMode.practice: return l10n.practice; - case SelectMode.transcription: + case SelectMode.speechTranslation: return l10n.speechToTextTooltip; } } @@ -72,7 +72,7 @@ class SelectModeButtonsState extends State { ]; static List get audioModes => [ - SelectMode.transcription, + SelectMode.speechTranslation, SelectMode.practice, ]; @@ -91,9 +91,7 @@ class SelectModeButtonsState extends State { PangeaRepresentation? _repEvent; String? _translationError; - bool _isLoadingTranscription = false; SpeechToTextModel? _speechToTextResponse; - String? _transcriptionError; @override void initState() { @@ -113,6 +111,10 @@ class SelectModeButtonsState extends State { ); } }); + + if (messageEvent?.isAudioMessage == true) { + _loadTranscription(); + } } @override @@ -135,7 +137,6 @@ class SelectModeButtonsState extends State { setState(() { _audioError = null; _translationError = null; - _transcriptionError = null; }); widget.overlayController.updateSelectedSpan(null); @@ -143,10 +144,6 @@ class SelectModeButtonsState extends State { if (_selectedMode == SelectMode.translate) { widget.overlayController.setShowTranslation(false, null); } - - if (_selectedMode == SelectMode.transcription) { - widget.overlayController.setShowTranscription(false, null); - } } Future _updateMode(SelectMode? mode) async { @@ -181,20 +178,6 @@ class SelectModeButtonsState extends State { if (_selectedMode == SelectMode.translate) { await _loadTranslation(); - if (_repEvent == null) return; - widget.overlayController.setShowTranslation( - true, - _repEvent!.text, - ); - } - - if (_selectedMode == SelectMode.transcription) { - await _loadTranscription(); - if (_speechToTextResponse == null) return; - widget.overlayController.setShowTranscription( - true, - _speechToTextResponse!.transcript.text, - ); } } @@ -312,6 +295,14 @@ class SelectModeButtonsState extends State { try { await _fetchRepresentation(); + if (_repEvent == null) { + throw "No representation found for the selected language."; + } + + widget.overlayController.setShowTranslation( + true, + _repEvent!.text, + ); } catch (err) { _translationError = err.toString(); ErrorHandler.logError( @@ -326,22 +317,20 @@ class SelectModeButtonsState extends State { } Future _loadTranscription() async { - if (!mounted) return; - setState(() => _isLoadingTranscription = true); - try { await _fetchTranscription(); + widget.overlayController.setTranscriptionText( + _speechToTextResponse!.transcript.text, + ); } catch (err) { - _transcriptionError = err.toString(); + widget.overlayController.setTranscriptionError( + err.toString(), + ); ErrorHandler.logError( e: err, data: {}, ); } - - if (mounted) { - setState(() => _isLoadingTranscription = false); - } } bool get _isError { @@ -350,8 +339,6 @@ class SelectModeButtonsState extends State { return _audioError != null; case SelectMode.translate: return _translationError != null; - case SelectMode.transcription: - return _transcriptionError != null; default: return false; } @@ -363,8 +350,6 @@ class SelectModeButtonsState extends State { return _isLoadingAudio; case SelectMode.translate: return _isLoadingTranslation; - case SelectMode.transcription: - return _isLoadingTranscription; default: return false; } From 296ddef06d60d43d29b4cc184a0cb0d9053c163d Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 12:12:26 -0400 Subject: [PATCH 04/12] chore: add SST translation event --- .../events/constants/pangea_event_types.dart | 1 + .../controllers/message_data_controller.dart | 52 +++++++ .../event_wrappers/pangea_message_event.dart | 47 +++--- .../pangea_representation_event.dart | 67 ++++++++ .../event_wrappers/pangea_tokens_event.dart | 42 ----- .../events/models/stt_translation_model.dart | 23 +++ .../widgets/message_selection_overlay.dart | 53 +++++-- .../toolbar/widgets/overlay_message.dart | 16 +- .../toolbar/widgets/select_mode_buttons.dart | 146 ++++++++++++------ 9 files changed, 312 insertions(+), 135 deletions(-) delete mode 100644 lib/pangea/events/event_wrappers/pangea_tokens_event.dart create mode 100644 lib/pangea/events/models/stt_translation_model.dart diff --git a/lib/pangea/events/constants/pangea_event_types.dart b/lib/pangea/events/constants/pangea_event_types.dart index ba7097f57..fe2a47e18 100644 --- a/lib/pangea/events/constants/pangea_event_types.dart +++ b/lib/pangea/events/constants/pangea_event_types.dart @@ -16,6 +16,7 @@ class PangeaEventTypes { static const tokens = "pangea.tokens"; static const choreoRecord = "pangea.record"; static const representation = "pangea.representation"; + static const sttTranslation = "pangea.stt_translation"; // static const vocab = "p.vocab"; static const roomInfo = "pangea.roomtopic"; diff --git a/lib/pangea/events/controllers/message_data_controller.dart b/lib/pangea/events/controllers/message_data_controller.dart index bae9f2512..6bbcc5af2 100644 --- a/lib/pangea/events/controllers/message_data_controller.dart +++ b/lib/pangea/events/controllers/message_data_controller.dart @@ -13,6 +13,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/token_api_models.dart'; import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart'; @@ -24,6 +25,7 @@ class MessageDataController extends BaseController { final Map> _tokensCache = {}; final Map> _representationCache = {}; + final Map> _sttTranslationCache = {}; late Timer _cacheTimer; MessageDataController(PangeaController pangeaController) { @@ -42,6 +44,7 @@ class MessageDataController extends BaseController { void _clearCache() { _tokensCache.clear(); _representationCache.clear(); + _sttTranslationCache.clear(); debugPrint("message data cache cleared."); } @@ -219,4 +222,53 @@ class MessageDataController extends BaseController { ); } } + + Future getSttTranslation({ + required String? repEventId, + required FullTextTranslationRequestModel req, + required Room? room, + }) => + _sttTranslationCache[req.hashCode] ??= _getSttTranslation( + repEventId: repEventId, + req: req, + room: room, + ).catchError((e, s) { + _sttTranslationCache.remove(req.hashCode); + return Future.error(e, s); + }); + + Future _getSttTranslation({ + required String? repEventId, + required FullTextTranslationRequestModel req, + required Room? room, + }) async { + final res = await FullTextTranslationRepo.translate( + accessToken: _pangeaController.userController.accessToken, + request: req, + ); + + final translation = SttTranslationModel( + translation: res.bestTranslation, + langCode: req.tgtLang, + ); + + if (repEventId != null && room != null) { + room + .sendPangeaEvent( + content: translation.toJson(), + parentEventId: repEventId, + type: PangeaEventTypes.sttTranslation, + ) + .catchError( + (e) => ErrorHandler.logError( + m: "error in _getSttTranslation.sendPangeaEvent", + e: e, + s: StackTrace.current, + data: req.toJson(), + ), + ); + } + + return translation; + } } diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index 1cf52e70f..d76519f38 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -16,6 +16,7 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_ev import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/morphs/morph_features_enum.dart'; import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart'; @@ -281,32 +282,11 @@ class PangeaMessageEvent { ?.content .speechToText; - if (speechToTextLocal != null) return speechToTextLocal; + if (speechToTextLocal != null) { + return speechToTextLocal; + } final matrixFile = await _event.downloadAndDecryptAttachment(); - // Pangea# - // File? file; - - // TODO: Test on mobile and see if we need this case, doeesn't seem so - // if (!kIsWeb) { - // final tempDir = await getTemporaryDirectory(); - // final fileName = Uri.encodeComponent( - // // #Pangea - // // widget.event.attachmentOrThumbnailMxcUrl()!.pathSegments.last, - // widget.messageEvent.event - // .attachmentOrThumbnailMxcUrl()! - // .pathSegments - // .last, - // // Pangea# - // ); - // file = File('${tempDir.path}/${fileName}_${matrixFile.name}'); - // await file.writeAsBytes(matrixFile.bytes); - // } - - // audioFile = file; - - debugPrint("mimeType ${matrixFile.mimeType}"); - debugPrint("encoding ${mimeTypeToAudioEncoding(matrixFile.mimeType)}"); final SpeechToTextModel response = await MatrixState.pangeaController.speechToText.get( @@ -341,6 +321,25 @@ class PangeaMessageEvent { return response; } + Future sttTranslationByLanguageGlobal({ + required String langCode, + required String l1Code, + required String l2Code, + }) async { + if (!representations.any( + (element) => element.content.speechToText != null, + )) { + await getSpeechToText(l1Code, l2Code); + } + + final rep = representations.firstWhereOrNull( + (element) => element.content.speechToText != null, + ); + + if (rep == null) return null; + return rep.getSttTranslation(userL1: l1Code, userL2: l2Code); + } + PangeaMessageTokens? _tokensSafe(Map? content) { try { if (content == null) return null; diff --git a/lib/pangea/events/event_wrappers/pangea_representation_event.dart b/lib/pangea/events/event_wrappers/pangea_representation_event.dart index b7607e36e..12b41b83d 100644 --- a/lib/pangea/events/event_wrappers/pangea_representation_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_representation_event.dart @@ -12,11 +12,13 @@ import 'package:sentry_flutter/sentry_flutter.dart'; import 'package:fluffychat/pangea/choreographer/event_wrappers/pangea_choreo_event.dart'; import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart'; import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart'; +import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_repo.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; +import 'package:fluffychat/pangea/events/models/stt_translation_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/events/repo/token_api_models.dart'; import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; @@ -210,6 +212,71 @@ class RepresentationEvent { ); } + List get sttTranslations { + if (content.speechToText == null) return []; + if (_event == null) { + Sentry.addBreadcrumb( + Breadcrumb( + message: "_event and _sttTranslations both null", + ), + ); + return []; + } + + final Set sttEvents = _event!.aggregatedEvents( + timeline, + PangeaEventTypes.sttTranslation, + ); + + if (sttEvents.isEmpty) return []; + final List sttTranslations = []; + for (final event in sttEvents) { + try { + sttTranslations.add( + SttTranslationModel.fromJson(event.content), + ); + } catch (e) { + Sentry.addBreadcrumb( + Breadcrumb( + message: "Failed to parse STT translation", + data: { + "eventID": event.eventId, + "content": event.content, + "error": e.toString(), + }, + ), + ); + } + } + + return sttTranslations; + } + + Future getSttTranslation({ + required String userL1, + required String userL2, + }) async { + if (content.speechToText == null) { + throw Exception( + "RepresentationEvent.getSttTranslation called on a representation without speechToText", + ); + } + + final local = sttTranslations.firstWhereOrNull((t) => t.langCode == userL1); + if (local != null) return local; + + return MatrixState.pangeaController.messageData.getSttTranslation( + repEventId: _event?.eventId, + room: _event?.room, + req: FullTextTranslationRequestModel( + text: content.speechToText!.transcript.text, + tgtLang: userL1, + userL2: userL2, + userL1: userL1, + ), + ); + } + ChoreoRecord? get choreo { if (_choreo != null) return _choreo; diff --git a/lib/pangea/events/event_wrappers/pangea_tokens_event.dart b/lib/pangea/events/event_wrappers/pangea_tokens_event.dart deleted file mode 100644 index 126d9cff8..000000000 --- a/lib/pangea/events/event_wrappers/pangea_tokens_event.dart +++ /dev/null @@ -1,42 +0,0 @@ -import 'dart:developer'; - -import 'package:flutter/foundation.dart'; - -import 'package:matrix/matrix.dart'; - -import 'package:fluffychat/pangea/common/utils/error_handler.dart'; -import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; -import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; -import '../constants/pangea_event_types.dart'; - -class TokensEvent { - Event event; - PangeaMessageTokens? _content; - - TokensEvent({required this.event}) { - if (event.type != PangeaEventTypes.tokens) { - throw Exception( - "${event.type} should not be used to make a TokensEvent", - ); - } - } - - PangeaMessageTokens? get _pangeaMessageTokens { - try { - _content ??= event.getPangeaContent(); - return _content!; - } catch (err, s) { - debugger(when: kDebugMode); - ErrorHandler.logError( - e: err, - s: s, - data: { - "event": event.toJson(), - }, - ); - return null; - } - } - - PangeaMessageTokens? get tokens => _pangeaMessageTokens; -} diff --git a/lib/pangea/events/models/stt_translation_model.dart b/lib/pangea/events/models/stt_translation_model.dart new file mode 100644 index 000000000..47f31be4f --- /dev/null +++ b/lib/pangea/events/models/stt_translation_model.dart @@ -0,0 +1,23 @@ +class SttTranslationModel { + final String translation; + final String langCode; + + SttTranslationModel({ + required this.translation, + required this.langCode, + }); + + factory SttTranslationModel.fromJson(Map json) { + return SttTranslationModel( + translation: json['translation'] as String, + langCode: json['lang_code'] as String, + ); + } + + Map toJson() { + return { + 'translation': translation, + 'lang_code': langCode, + }; + } +} diff --git a/lib/pangea/toolbar/widgets/message_selection_overlay.dart b/lib/pangea/toolbar/widgets/message_selection_overlay.dart index 73451081f..f2492126d 100644 --- a/lib/pangea/toolbar/widgets/message_selection_overlay.dart +++ b/lib/pangea/toolbar/widgets/message_selection_overlay.dart @@ -30,6 +30,7 @@ import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller. import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart'; +import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; import 'package:fluffychat/pangea/toolbar/reading_assistance_input_row/morph_selection.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_positioner.dart'; import 'package:fluffychat/pangea/toolbar/widgets/reading_assistance_content.dart'; @@ -91,11 +92,14 @@ class MessageOverlayController extends State ReadingAssistanceMode? readingAssistanceMode; // default mode + SpeechToTextModel? transcription; + String? transcriptionError; + bool showTranslation = false; - String? translationText; + String? translation; - String? transcriptionText; - String? transcriptionError; + bool showSpeechTranslation = false; + String? speechTranslation; double maxWidth = AppConfig.toolbarMinWidth; @@ -574,33 +578,50 @@ class MessageOverlayController extends State ); } - void setShowTranslation(bool show, String? translation) { + void setTranslation(String value) { + if (mounted) { + setState(() => translation = value); + } + } + + void setShowTranslation(bool show) { + if (!mounted) return; + if (translation == null) { + setState(() => showTranslation = false); + } + if (showTranslation == show) return; - if (show && translation == null) return; + setState(() => showTranslation = show); + } + void setSpeechTranslation(String value) { if (mounted) { - setState(() { - showTranslation = show; - translationText = show ? translation : null; - }); + setState(() => speechTranslation = value); + } + } + + void setShowSpeechTranslation(bool show) { + if (!mounted) return; + if (speechTranslation == null) { + setState(() => showSpeechTranslation = false); } + + if (showSpeechTranslation == show) return; + setState(() => showSpeechTranslation = show); } - void setTranscriptionText(String transcription) { + void setTranscription(SpeechToTextModel value) { if (mounted) { setState(() { transcriptionError = null; - transcriptionText = transcription; + transcription = value; }); } } - void setTranscriptionError(String error) { + void setTranscriptionError(String value) { if (mounted) { - setState(() { - transcriptionText = null; - transcriptionError = error; - }); + setState(() => transcriptionError = value); } } diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index c4e848851..91a528738 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -135,10 +135,13 @@ class OverlayMessage extends StatelessWidget { event.numberEmotes <= 3); final showTranslation = overlayController.showTranslation && - overlayController.translationText != null; + overlayController.translation != null; final showTranscription = pangeaMessageEvent?.isAudioMessage == true; + final showSpeechTranslation = overlayController.showSpeechTranslation && + overlayController.speechTranslation != null; + final content = Container( decoration: BoxDecoration( borderRadius: BorderRadius.circular( @@ -296,10 +299,11 @@ class OverlayMessage extends StatelessWidget { ), ], ) - : overlayController.transcriptionText != null + : overlayController.transcription != null ? SingleChildScrollView( child: Text( - overlayController.transcriptionText!, + overlayController + .transcription!.transcript.text, style: AppConfig.messageTextStyle( event, textColor, @@ -323,7 +327,7 @@ class OverlayMessage extends StatelessWidget { }, ) : content, - if (showTranslation) + if (showTranslation || showSpeechTranslation) Container( width: messageWidth, constraints: const BoxConstraints( @@ -338,7 +342,9 @@ class OverlayMessage extends StatelessWidget { ), child: SingleChildScrollView( child: Text( - overlayController.translationText!, + showTranslation + ? overlayController.translation! + : overlayController.speechTranslation!, style: AppConfig.messageTextStyle( event, textColor, diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 2b434a835..76cca02f3 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -18,7 +18,6 @@ import 'package:fluffychat/pangea/common/widgets/pressable_button.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; -import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_audio_card.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; import 'package:fluffychat/widgets/matrix.dart'; @@ -88,10 +87,12 @@ class SelectModeButtonsState extends State { StreamSubscription? _onAudioPositionChanged; bool _isLoadingTranslation = false; - PangeaRepresentation? _repEvent; String? _translationError; - SpeechToTextModel? _speechToTextResponse; + bool _isLoadingSpeechTranslation = false; + String? _speechTranslationError; + + Completer? _transcriptionCompleter; @override void initState() { @@ -113,7 +114,7 @@ class SelectModeButtonsState extends State { }); if (messageEvent?.isAudioMessage == true) { - _loadTranscription(); + _fetchTranscription(); } } @@ -129,9 +130,9 @@ class SelectModeButtonsState extends State { widget.overlayController.pangeaMessageEvent; String? get l1Code => - MatrixState.pangeaController.languageController.activeL1Code(); + MatrixState.pangeaController.languageController.userL1?.langCodeShort; String? get l2Code => - MatrixState.pangeaController.languageController.activeL2Code(); + MatrixState.pangeaController.languageController.userL2?.langCodeShort; void _clear() { setState(() { @@ -140,10 +141,8 @@ class SelectModeButtonsState extends State { }); widget.overlayController.updateSelectedSpan(null); - - if (_selectedMode == SelectMode.translate) { - widget.overlayController.setShowTranslation(false, null); - } + widget.overlayController.setShowTranslation(false); + widget.overlayController.setShowSpeechTranslation(false); } Future _updateMode(SelectMode? mode) async { @@ -177,7 +176,13 @@ class SelectModeButtonsState extends State { } if (_selectedMode == SelectMode.translate) { - await _loadTranslation(); + await _fetchTranslation(); + widget.overlayController.setShowTranslation(true); + } + + if (_selectedMode == SelectMode.speechTranslation) { + await _fetchSpeechTranslation(); + widget.overlayController.setShowSpeechTranslation(true); } } @@ -265,71 +270,112 @@ class SelectModeButtonsState extends State { } } - Future _fetchRepresentation() async { - if (l1Code == null || messageEvent == null || _repEvent != null) { + Future _fetchTranslation() async { + if (l1Code == null || + messageEvent == null || + widget.overlayController.translation != null) { return; } - _repEvent = messageEvent!.representationByLanguage(l1Code!)?.content; - if (_repEvent == null && mounted) { - _repEvent = await messageEvent?.representationByLanguageGlobal( + try { + if (mounted) setState(() => _isLoadingTranslation = true); + + PangeaRepresentation? rep = + messageEvent!.representationByLanguage(l1Code!)?.content; + + rep ??= await messageEvent?.representationByLanguageGlobal( langCode: l1Code!, ); - } - } - Future _fetchTranscription() async { - if (l1Code == null || messageEvent == null || _repEvent != null) { - return; + widget.overlayController.setTranslation(rep!.text); + } catch (e, s) { + _translationError = e.toString(); + ErrorHandler.logError( + e: e, + s: s, + m: 'Error fetching translation', + data: { + 'l1Code': l1Code, + 'messageEvent': messageEvent?.event.toJson(), + }, + ); + } finally { + if (mounted) setState(() => _isLoadingTranslation = false); } - - _speechToTextResponse ??= await messageEvent!.getSpeechToText( - l1Code!, - l2Code!, - ); } - Future _loadTranslation() async { - if (!mounted) return; - setState(() => _isLoadingTranslation = true); - + Future _fetchTranscription() async { try { - await _fetchRepresentation(); - if (_repEvent == null) { - throw "No representation found for the selected language."; + if (_transcriptionCompleter != null) { + // If a transcription is already in progress, wait for it to complete + await _transcriptionCompleter!.future; + return; + } + + _transcriptionCompleter = Completer(); + if (l1Code == null || messageEvent == null) { + _transcriptionCompleter?.completeError( + 'Language code or message event is null', + ); + return; } - widget.overlayController.setShowTranslation( - true, - _repEvent!.text, + final resp = await messageEvent!.getSpeechToText( + l1Code!, + l2Code!, ); + + widget.overlayController.setTranscription(resp!); + _transcriptionCompleter?.complete(resp.transcript.text); } catch (err) { - _translationError = err.toString(); + widget.overlayController.setTranscriptionError( + err.toString(), + ); + _transcriptionCompleter?.completeError(err); ErrorHandler.logError( e: err, data: {}, ); } + } - if (mounted) { - setState(() => _isLoadingTranslation = false); + Future _fetchSpeechTranslation() async { + if (messageEvent == null || + l1Code == null || + l2Code == null || + widget.overlayController.speechTranslation != null) { + return; } - } - Future _loadTranscription() async { try { - await _fetchTranscription(); - widget.overlayController.setTranscriptionText( - _speechToTextResponse!.transcript.text, - ); - } catch (err) { - widget.overlayController.setTranscriptionError( - err.toString(), + setState(() => _isLoadingSpeechTranslation = true); + + if (widget.overlayController.transcription == null) { + await _fetchTranscription(); + if (widget.overlayController.transcription == null) { + throw Exception('Transcription is null'); + } + } + + final translation = await messageEvent!.sttTranslationByLanguageGlobal( + langCode: l1Code!, + l1Code: l1Code!, + l2Code: l2Code!, ); + if (translation == null) { + throw Exception('Translation is null'); + } + + widget.overlayController.setSpeechTranslation(translation.translation); + } catch (err, s) { + debugPrint("Error fetching speech translation: $err, $s"); + _speechTranslationError = err.toString(); ErrorHandler.logError( e: err, data: {}, ); + } finally { + if (mounted) setState(() => _isLoadingSpeechTranslation = false); } } @@ -339,6 +385,8 @@ class SelectModeButtonsState extends State { return _audioError != null; case SelectMode.translate: return _translationError != null; + case SelectMode.speechTranslation: + return _speechTranslationError != null; default: return false; } @@ -350,6 +398,8 @@ class SelectModeButtonsState extends State { return _isLoadingAudio; case SelectMode.translate: return _isLoadingTranslation; + case SelectMode.speechTranslation: + return _isLoadingSpeechTranslation; default: return false; } From 1f5c722bcd92afe9c511dfed85ac2a5450cfd36f Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 13:09:04 -0400 Subject: [PATCH 05/12] add audio analytics --- lib/pages/chat/chat.dart | 207 ++++++++++++------ .../toolbar/models/speech_to_text_models.dart | 25 +++ .../toolbar/widgets/select_mode_buttons.dart | 1 + 3 files changed, 169 insertions(+), 64 deletions(-) diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 19a6181ab..bcf34389f 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -51,6 +51,7 @@ import 'package:fluffychat/pangea/events/models/representation_content_model.dar import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart'; import 'package:fluffychat/pangea/instructions/instructions_enum.dart'; +import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; import 'package:fluffychat/pangea/learning_settings/widgets/p_language_dialog.dart'; import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart'; import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart'; @@ -924,31 +925,7 @@ class ChatController extends State ), ]; - final newGrammarConstructs = - pangeaController.getAnalytics.newConstructCount( - constructs, - ConstructTypeEnum.morph, - ); - - final newVocabConstructs = - pangeaController.getAnalytics.newConstructCount( - constructs, - ConstructTypeEnum.vocab, - ); - - OverlayUtil.showOverlay( - overlayKey: "msg_analytics_feedback_$msgEventId", - followerAnchor: Alignment.bottomRight, - targetAnchor: Alignment.topRight, - context: context, - child: MessageAnalyticsFeedback( - overlayId: "msg_analytics_feedback_$msgEventId", - newGrammarConstructs: newGrammarConstructs, - newVocabConstructs: newVocabConstructs, - ), - transformTargetId: msgEventId, - ignorePointer: true, - ); + _showAnalyticsFeedback(constructs, msgEventId); pangeaController.putAnalytics.setState( AnalyticsStream( @@ -1130,44 +1107,54 @@ class ChatController extends State name: result.fileName ?? audioFile.path, ); - await room.sendFileEvent( - file, - inReplyTo: replyEvent, - extraContent: { - 'info': { - ...file.info, - 'duration': result.duration, - }, - 'org.matrix.msc3245.voice': {}, - 'org.matrix.msc1767.audio': { - 'duration': result.duration, - 'waveform': result.waveform, - }, - }, - // #Pangea - // ).catchError((e) { - ).catchError((e, s) { - ErrorHandler.logError( - e: e, - s: s, - data: { - 'roomId': roomId, - 'file': file.name, - 'duration': result.duration, - 'waveform': result.waveform, - }, - ); - // Pangea# - scaffoldMessenger.showSnackBar( - SnackBar( - content: Text( - (e as Object).toLocalizedString(context), - ), - ), - ); - return null; - }); - // #Pangea + await room + .sendFileEvent( + file, + inReplyTo: replyEvent, + extraContent: { + 'info': { + ...file.info, + 'duration': result.duration, + }, + 'org.matrix.msc3245.voice': {}, + 'org.matrix.msc1767.audio': { + 'duration': result.duration, + 'waveform': result.waveform, + }, + }, + // #Pangea + ) + .then(_sendVoiceMessageAnalytics) + .catchError((e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: { + 'roomId': roomId, + 'file': file.name, + 'duration': result.duration, + 'waveform': result.waveform, + }, + ); + scaffoldMessenger.showSnackBar( + SnackBar( + content: Text( + (e as Object).toLocalizedString(context), + ), + ), + ); + return null; + }); + // ).catchError((e) { + // scaffoldMessenger.showSnackBar( + // SnackBar( + // content: Text( + // (e as Object).toLocalizedString(context), + // ), + // ), + // ); + // return null; + // }); // setState(() { // replyEvent = null; // }); @@ -1618,7 +1605,8 @@ class ChatController extends State if (timeline == null || events.any( (e) => e.aggregatedEvents(timeline!, RelationshipTypes.reaction).any( - (re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji), + (re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji, + ), )) { return; } @@ -2058,6 +2046,97 @@ class ChatController extends State return false; } } + + Future _sendVoiceMessageAnalytics(String? eventId) async { + if (eventId == null) { + ErrorHandler.logError( + e: Exception('eventID null in voiceMessageAction'), + s: StackTrace.current, + data: { + 'roomId': roomId, + }, + ); + return; + } + + final event = await room.getEventById(eventId); + if (event == null) { + ErrorHandler.logError( + e: Exception('Event not found after sending voice message'), + s: StackTrace.current, + data: { + 'roomId': roomId, + }, + ); + return; + } + + try { + final messageEvent = PangeaMessageEvent( + event: event, + timeline: timeline!, + ownMessage: true, + ); + + final stt = await messageEvent.getSpeechToText( + choreographer.l1Lang?.langCodeShort ?? LanguageKeys.unknownLanguage, + choreographer.l2Lang?.langCodeShort ?? LanguageKeys.unknownLanguage, + ); + if (stt == null || stt.transcript.sttTokens.isEmpty) return; + final constructs = stt.constructs(roomId, eventId); + if (constructs.isEmpty) return; + + _showAnalyticsFeedback(constructs, eventId); + MatrixState.pangeaController.putAnalytics.setState( + AnalyticsStream( + eventId: eventId, + targetID: eventId, + roomId: room.id, + constructs: constructs, + ), + ); + } catch (e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: { + 'roomId': roomId, + 'eventId': eventId, + }, + ); + } + } + + void _showAnalyticsFeedback( + List constructs, + String eventId, + ) { + final newGrammarConstructs = + pangeaController.getAnalytics.newConstructCount( + constructs, + ConstructTypeEnum.morph, + ); + + final newVocabConstructs = pangeaController.getAnalytics.newConstructCount( + constructs, + ConstructTypeEnum.vocab, + ); + + OverlayUtil.showOverlay( + overlayKey: "msg_analytics_feedback_$eventId", + followerAnchor: Alignment.bottomRight, + targetAnchor: Alignment.topRight, + context: context, + child: MessageAnalyticsFeedback( + overlayId: "msg_analytics_feedback_$eventId", + newGrammarConstructs: newGrammarConstructs, + newVocabConstructs: newVocabConstructs, + ), + transformTargetId: eventId, + ignorePointer: true, + closePrevOverlay: false, + ); + } // Pangea# late final ValueNotifier _displayChatDetailsColumn; diff --git a/lib/pangea/toolbar/models/speech_to_text_models.dart b/lib/pangea/toolbar/models/speech_to_text_models.dart index d43f45643..1b7b676af 100644 --- a/lib/pangea/toolbar/models/speech_to_text_models.dart +++ b/lib/pangea/toolbar/models/speech_to_text_models.dart @@ -6,6 +6,8 @@ import 'package:flutter/material.dart'; import 'package:matrix/matrix.dart'; import 'package:fluffychat/config/app_config.dart'; +import 'package:fluffychat/pangea/analytics_misc/construct_use_type_enum.dart'; +import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/toolbar/enums/audio_encoding_enum.dart'; @@ -230,4 +232,27 @@ class SpeechToTextModel { Map toJson() => { "results": results.map((e) => e.toJson()).toList(), }; + + List constructs( + String roomId, + String eventId, + ) { + final List constructs = []; + final metadata = ConstructUseMetaData( + roomId: roomId, + eventId: eventId, + timeStamp: DateTime.now(), + ); + for (final sstToken in transcript.sttTokens) { + final token = sstToken.token; + constructs.addAll( + token.allUses( + ConstructUseTypeEnum.pvm, + metadata, + ConstructUseTypeEnum.pvm.pointValue, + ), + ); + } + return constructs; + } } diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 76cca02f3..2903eaa2e 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -138,6 +138,7 @@ class SelectModeButtonsState extends State { setState(() { _audioError = null; _translationError = null; + _speechTranslationError = null; }); widget.overlayController.updateSelectedSpan(null); From 1a69972b4cc11f057a8d1531e5c808a42bba6b1d Mon Sep 17 00:00:00 2001 From: Kelrap Date: Wed, 11 Jun 2025 13:11:12 -0400 Subject: [PATCH 06/12] Hide reply action in chats where user can't send messages --- lib/pages/chat/chat.dart | 9 ++++++++- lib/pangea/toolbar/widgets/overlay_header.dart | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 19a6181ab..87ad65665 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -1390,6 +1390,12 @@ class ChatController extends State return true; } +// #Pangea + bool get canReplySelectedEvents { + return room.canSendDefaultMessages; + } + // Pangea# + bool get canPinSelectedEvents { if (isArchived || !room.canChangeStateEvent(EventTypes.RoomPinnedEvents) || @@ -1618,7 +1624,8 @@ class ChatController extends State if (timeline == null || events.any( (e) => e.aggregatedEvents(timeline!, RelationshipTypes.reaction).any( - (re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji), + (re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji, + ), )) { return; } diff --git a/lib/pangea/toolbar/widgets/overlay_header.dart b/lib/pangea/toolbar/widgets/overlay_header.dart index faae598c8..009f78f73 100644 --- a/lib/pangea/toolbar/widgets/overlay_header.dart +++ b/lib/pangea/toolbar/widgets/overlay_header.dart @@ -63,7 +63,8 @@ class OverlayHeaderState extends State { padding: const EdgeInsets.symmetric(vertical: 8.0), child: Row( children: [ - if (controller.selectedEvents.length == 1) + if (controller.selectedEvents.length == 1 && + controller.canReplySelectedEvents) IconButton( icon: const Icon(Symbols.reply_all), tooltip: l10n.reply, From 77c6e24863a168d3d0eb18c36d1dc363a91ceee5 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 14:43:48 -0400 Subject: [PATCH 07/12] add voice uses to analytics popups --- lib/l10n/intl_en.arb | 3 +- .../analytics_misc/learning_skills_enum.dart | 4 +- .../toolbar/models/speech_to_text_models.dart | 1 + .../toolbar/widgets/overlay_message.dart | 158 ++++++++++-------- 4 files changed, 94 insertions(+), 72 deletions(-) diff --git a/lib/l10n/intl_en.arb b/lib/l10n/intl_en.arb index 8c5954014..661c781f4 100644 --- a/lib/l10n/intl_en.arb +++ b/lib/l10n/intl_en.arb @@ -5014,5 +5014,6 @@ "getStartedFriendsButton": "Invite a friend", "groupChat": "Group Chat", "directMessage": "Direct Message", - "newDirectMessage": "New direct message" + "newDirectMessage": "New direct message", + "speakingExercisesTooltip": "Speaking practice" } \ No newline at end of file diff --git a/lib/pangea/analytics_misc/learning_skills_enum.dart b/lib/pangea/analytics_misc/learning_skills_enum.dart index ca01e155a..60fad7847 100644 --- a/lib/pangea/analytics_misc/learning_skills_enum.dart +++ b/lib/pangea/analytics_misc/learning_skills_enum.dart @@ -7,7 +7,7 @@ import 'package:fluffychat/l10n/l10n.dart'; enum LearningSkillsEnum { writing(isVisible: true, icon: Symbols.edit_square), reading(isVisible: true, icon: Symbols.two_pager), - speaking(isVisible: false), + speaking(isVisible: true, icon: Icons.mic_outlined), hearing(isVisible: true, icon: Icons.volume_up), other(isVisible: false); @@ -27,6 +27,8 @@ enum LearningSkillsEnum { return L10n.of(context).readingExercisesTooltip; case LearningSkillsEnum.hearing: return L10n.of(context).listeningExercisesTooltip; + case LearningSkillsEnum.speaking: + return L10n.of(context).speakingExercisesTooltip; default: return ""; } diff --git a/lib/pangea/toolbar/models/speech_to_text_models.dart b/lib/pangea/toolbar/models/speech_to_text_models.dart index 1b7b676af..4b00929a0 100644 --- a/lib/pangea/toolbar/models/speech_to_text_models.dart +++ b/lib/pangea/toolbar/models/speech_to_text_models.dart @@ -245,6 +245,7 @@ class SpeechToTextModel { ); for (final sstToken in transcript.sttTokens) { final token = sstToken.token; + if (!token.lemma.saveVocab) continue; constructs.addAll( token.allUses( ConstructUseTypeEnum.pvm, diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index 91a528738..7717303ae 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -142,6 +142,86 @@ class OverlayMessage extends StatelessWidget { final showSpeechTranslation = overlayController.showSpeechTranslation && overlayController.speechTranslation != null; + final transcription = showTranscription + ? Container( + width: messageWidth, + constraints: const BoxConstraints( + maxHeight: AppConfig.audioTranscriptionMaxHeight, + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: overlayController.transcriptionError != null + ? Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.error_outline, + color: Theme.of(context).colorScheme.error, + ), + const SizedBox(width: 8), + Text( + L10n.of(context).oopsSomethingWentWrong, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith(fontStyle: FontStyle.italic), + ), + ], + ) + : overlayController.transcription != null + ? SingleChildScrollView( + child: Text( + overlayController.transcription!.transcript.text, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), + ) + : Row( + mainAxisSize: MainAxisSize.min, + children: [ + CircularProgressIndicator.adaptive( + backgroundColor: textColor, + ), + ], + ), + ), + ) + : const SizedBox(); + + final translation = showTranslation || showSpeechTranslation + ? Container( + width: messageWidth, + constraints: const BoxConstraints( + maxHeight: AppConfig.audioTranscriptionMaxHeight, + ), + child: Padding( + padding: const EdgeInsets.fromLTRB( + 12.0, + 20.0, + 12.0, + 12.0, + ), + child: SingleChildScrollView( + child: Text( + showTranslation + ? overlayController.translation! + : overlayController.speechTranslation!, + style: AppConfig.messageTextStyle( + event, + textColor, + ).copyWith( + fontStyle: FontStyle.italic, + ), + ), + ), + ), + ) + : const SizedBox(); + final content = Container( decoration: BoxDecoration( borderRadius: BorderRadius.circular( @@ -159,6 +239,8 @@ class OverlayMessage extends StatelessWidget { mainAxisSize: MainAxisSize.min, crossAxisAlignment: CrossAxisAlignment.start, children: [ + if (readingAssistanceMode == ReadingAssistanceMode.transitionMode) + transcription, if (event.relationshipType == RelationshipTypes.reply) FutureBuilder( future: event.getReplyEvent( @@ -257,6 +339,8 @@ class OverlayMessage extends StatelessWidget { ], ), ), + if (readingAssistanceMode == ReadingAssistanceMode.transitionMode) + translation, ], ), ), @@ -273,48 +357,8 @@ class OverlayMessage extends StatelessWidget { child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - if (showTranscription) - Container( - width: messageWidth, - constraints: const BoxConstraints( - maxHeight: AppConfig.audioTranscriptionMaxHeight, - ), - child: Padding( - padding: const EdgeInsets.all(12.0), - child: overlayController.transcriptionError != null - ? Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon( - Icons.error_outline, - color: Theme.of(context).colorScheme.error, - ), - const SizedBox(width: 8), - Text( - L10n.of(context).oopsSomethingWentWrong, - style: AppConfig.messageTextStyle( - event, - textColor, - ).copyWith(fontStyle: FontStyle.italic), - ), - ], - ) - : overlayController.transcription != null - ? SingleChildScrollView( - child: Text( - overlayController - .transcription!.transcript.text, - style: AppConfig.messageTextStyle( - event, - textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ), - ) - : const LinearProgressIndicator(), - ), - ), + if (readingAssistanceMode != ReadingAssistanceMode.transitionMode) + transcription, sizeAnimation != null ? AnimatedBuilder( animation: sizeAnimation!, @@ -327,34 +371,8 @@ class OverlayMessage extends StatelessWidget { }, ) : content, - if (showTranslation || showSpeechTranslation) - Container( - width: messageWidth, - constraints: const BoxConstraints( - maxHeight: AppConfig.audioTranscriptionMaxHeight, - ), - child: Padding( - padding: const EdgeInsets.fromLTRB( - 12.0, - 20.0, - 12.0, - 12.0, - ), - child: SingleChildScrollView( - child: Text( - showTranslation - ? overlayController.translation! - : overlayController.speechTranslation!, - style: AppConfig.messageTextStyle( - event, - textColor, - ).copyWith( - fontStyle: FontStyle.italic, - ), - ), - ), - ), - ), + if (readingAssistanceMode != ReadingAssistanceMode.transitionMode) + translation, ], ), ), From 2adc550b6f8f7e7cace052bbd3d6f809eceea8d5 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 14:53:46 -0400 Subject: [PATCH 08/12] remove practice mode button from audio messages --- lib/pangea/toolbar/widgets/select_mode_buttons.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 2903eaa2e..0c353c3f4 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -72,7 +72,7 @@ class SelectModeButtonsState extends State { static List get audioModes => [ SelectMode.speechTranslation, - SelectMode.practice, + // SelectMode.practice, ]; SelectMode? _selectedMode; From 3e21a5f3f749c3009353f7c219a8294ba1191790 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 15:03:52 -0400 Subject: [PATCH 09/12] fix: set user base language by user selection --- lib/pangea/login/pages/user_settings.dart | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pangea/login/pages/user_settings.dart b/lib/pangea/login/pages/user_settings.dart index d8aab1105..3e9faeb4f 100644 --- a/lib/pangea/login/pages/user_settings.dart +++ b/lib/pangea/login/pages/user_settings.dart @@ -210,9 +210,8 @@ class UserSettingsState extends State { _pangeaController.subscriptionController.reinitialize(), _pangeaController.userController.updateProfile( (profile) { - if (_systemLanguage != null) { - profile.userSettings.sourceLanguage = _systemLanguage!.langCode; - } + profile.userSettings.sourceLanguage = + selectedBaseLanguage?.langCode ?? _systemLanguage?.langCode; profile.userSettings.targetLanguage = selectedTargetLanguage!.langCode; profile.userSettings.cefrLevel = selectedCefrLevel; From cb1719cae331831be673dd06129e101bebee4d43 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 15:16:09 -0400 Subject: [PATCH 10/12] chore: maintain timemout message until activity search fully loads --- .../activity_suggestions/activity_suggestions_area.dart | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pangea/activity_suggestions/activity_suggestions_area.dart b/lib/pangea/activity_suggestions/activity_suggestions_area.dart index eef871623..b4ce264a0 100644 --- a/lib/pangea/activity_suggestions/activity_suggestions_area.dart +++ b/lib/pangea/activity_suggestions/activity_suggestions_area.dart @@ -97,7 +97,6 @@ class ActivitySuggestionsAreaState extends State { setState(() { _activityItems.clear(); _loading = true; - _timeout = false; }); final ActivityPlanRequest request = ActivityPlanRequest( @@ -124,10 +123,16 @@ class ActivitySuggestionsAreaState extends State { Future.delayed(const Duration(seconds: 5), () { if (mounted) _setActivityItems(retries: retries + 1); }); - return ActivityPlanResponse(activityPlans: []); + + return Future.error( + TimeoutException( + L10n.of(context).activitySuggestionTimeoutMessage, + ), + ); }, ); _activityItems.addAll(resp.activityPlans); + _timeout = false; } finally { if (mounted) setState(() => _loading = false); } From 7d0973b7aaef14a1609f81042ac0e8ef5afefa09 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 15:20:38 -0400 Subject: [PATCH 11/12] chore: set invite step to complete if user has DM with non-bot user --- lib/pangea/onboarding/onboarding.dart | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/pangea/onboarding/onboarding.dart b/lib/pangea/onboarding/onboarding.dart index 1ef67ed57..7393e88d9 100644 --- a/lib/pangea/onboarding/onboarding.dart +++ b/lib/pangea/onboarding/onboarding.dart @@ -40,13 +40,13 @@ class OnboardingController extends State { (r) => r.isSpace, ); case OnboardingStepsEnum.inviteFriends: - return hasInvitedFriends; + return MatrixState.pangeaController.matrixState.client.rooms.any( + (r) => + r.isDirectChat && r.directChatMatrixID != BotName.byEnvironment, + ); } } - static bool get hasInvitedFriends => - _onboardingStorage.read('invite_friends') ?? false; - static bool get hasBotDM => MatrixState.pangeaController.matrixState.client.rooms.any((room) { if (room.isDirectChat && @@ -66,8 +66,6 @@ class OnboardingController extends State { Future inviteFriends() async { FluffyShare.shareInviteLink(context); - await _onboardingStorage.write('invite_friends', true); - if (mounted) setState(() {}); } Future startChatWithBot() async { From a36ac5cc81283043c9c7c7305cf1ee8b6d90b552 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 11 Jun 2025 15:23:28 -0400 Subject: [PATCH 12/12] simplify --- lib/pages/chat/chat.dart | 6 ------ lib/pangea/toolbar/widgets/overlay_header.dart | 5 ++++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 87ad65665..4b577e5b2 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -1390,12 +1390,6 @@ class ChatController extends State return true; } -// #Pangea - bool get canReplySelectedEvents { - return room.canSendDefaultMessages; - } - // Pangea# - bool get canPinSelectedEvents { if (isArchived || !room.canChangeStateEvent(EventTypes.RoomPinnedEvents) || diff --git a/lib/pangea/toolbar/widgets/overlay_header.dart b/lib/pangea/toolbar/widgets/overlay_header.dart index 009f78f73..cb4db5d21 100644 --- a/lib/pangea/toolbar/widgets/overlay_header.dart +++ b/lib/pangea/toolbar/widgets/overlay_header.dart @@ -63,8 +63,11 @@ class OverlayHeaderState extends State { padding: const EdgeInsets.symmetric(vertical: 8.0), child: Row( children: [ + // #Pangea + // if (controller.selectedEvents.length == 1) if (controller.selectedEvents.length == 1 && - controller.canReplySelectedEvents) + controller.room.canSendDefaultMessages) + // Pangea# IconButton( icon: const Icon(Symbols.reply_all), tooltip: l10n.reply,