word focus turned off and tts on Choice click (#1118)

* word focus turned off and tts on Choice click

* play audio on word selection

lib/pages/chat/chat.dart

@@ -140,6 +140,7 @@ class ChatController extends State<ChatPageWithRoom>
   Timer? typingTimeout;
   bool currentlyTyping = false;
   // #Pangea
+
   // bool dragging = false;
 
   // void onDragEntered(_) => setState(() => dragging = true);

lib/pangea/choreographer/controllers/choreographer.dart

@@ -17,6 +17,7 @@ import 'package:fluffychat/pangea/models/tokens_event_content_model.dart';
 import 'package:fluffychat/pangea/utils/any_state_holder.dart';
 import 'package:fluffychat/pangea/utils/error_handler.dart';
 import 'package:fluffychat/pangea/utils/overlay.dart';
+import 'package:fluffychat/pangea/widgets/chat/tts_controller.dart';
 import 'package:fluffychat/pangea/widgets/igc/paywall_card.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
@@ -40,6 +41,7 @@ class Choreographer {
   late IgcController igc;
   late AlternativeTranslator altTranslator;
   late ErrorService errorService;
+  final tts = TtsController();
 
   bool isFetching = false;
   int _timesClicked = 0;
@@ -66,6 +68,8 @@ class Choreographer {
         .subscriptionController.trialActivationStream.stream
         .listen((_) => _onChangeListener);
 
+    tts.setupTTS();
+
     clear();
   }
 
@@ -454,6 +458,7 @@ class Choreographer {
     choreoRecord = ChoreoRecord.newRecord;
     itController.clear();
     igc.clear();
+    //@ggurdin - why is this commented out?
     // errorService.clear();
     _resetDebounceTimer();
   }
@@ -477,6 +482,7 @@ class Choreographer {
   dispose() {
     _textController.dispose();
     trialStream?.cancel();
+    tts.dispose();
   }
 
   LanguageModel? get l2Lang {

lib/pangea/choreographer/widgets/choice_array.dart

@@ -2,6 +2,7 @@ import 'dart:developer';
 import 'dart:math';
 
 import 'package:collection/collection.dart';
+import 'package:fluffychat/pangea/widgets/chat/tts_controller.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_gen/gen_l10n/l10n.dart';
@@ -20,6 +21,10 @@ class ChoicesArray extends StatefulWidget {
   final String originalSpan;
   final String Function(int) uniqueKeyForLayerLink;
 
+  /// If null then should not be used
+  /// We don't want tts in the case of L1 options
+  final TtsController? tts;
+
   /// Used to unqiuely identify the keys for choices, in cases where multiple
   /// choices could have identical text, like in back-to-back practice activities
   final String? id;
@@ -35,6 +40,7 @@ class ChoicesArray extends StatefulWidget {
     required this.originalSpan,
     required this.uniqueKeyForLayerLink,
     required this.selectedChoiceIndex,
+    required this.tts,
     this.isActive = true,
     this.onLongPress,
     this.id,
@@ -73,7 +79,11 @@ class ChoicesArrayState extends State<ChoicesArray> {
                     theme: theme,
                     onLongPress: widget.isActive ? widget.onLongPress : null,
                     onPressed: widget.isActive
-                        ? widget.onPressed
+                        ? (String value, int index) {
+                            widget.onPressed(value, index);
+                            // TODO - what to pass here as eventID?
+                            widget.tts?.tryToSpeak(value, context, null);
+                          }
                         : (String value, int index) {
                             debugger(when: kDebugMode);
                           },

lib/pangea/choreographer/widgets/it_bar.dart

@@ -418,6 +418,7 @@ class ITChoices extends StatelessWidget {
         onLongPress: (value, index) => showCard(context, index),
         uniqueKeyForLayerLink: (int index) => "itChoices$index",
         selectedChoiceIndex: null,
+        tts: controller.choreographer.tts,
       );
     } catch (e) {
       debugger(when: kDebugMode);

lib/pangea/choreographer/widgets/translation_finished_flow.dart

@@ -67,7 +67,7 @@ class AlternativeTranslations extends StatelessWidget {
   @override
   Widget build(BuildContext context) {
     return ChoicesArray(
-      originalSpan: controller.choreographer.itController.sourceText ?? "dummy",
+      originalSpan: controller.sourceText ?? "dummy",
       isLoading:
           controller.choreographer.altTranslator.loadingAlternativeTranslations,
       // choices: controller.choreographer.altTranslator.similarityResponse.scores
@@ -82,6 +82,7 @@ class AlternativeTranslations extends StatelessWidget {
       },
       uniqueKeyForLayerLink: (int index) => "altTranslation$index",
       selectedChoiceIndex: null,
+      tts: controller.choreographer.tts,
     );
   }
 }

lib/pangea/enum/activity_type_enum.dart

@@ -26,6 +26,16 @@ extension ActivityTypeExtension on ActivityTypeEnum {
     }
   }
 
+  bool get includeTTSOnClick {
+    switch (this) {
+      case ActivityTypeEnum.wordMeaning:
+        return false;
+      case ActivityTypeEnum.wordFocusListening:
+      case ActivityTypeEnum.hiddenWordListening:
+        return true;
+    }
+  }
+
   ActivityTypeEnum fromString(String value) {
     final split = value.split('.').last;
     switch (split) {

lib/pangea/models/pangea_token_model.dart

@@ -197,6 +197,7 @@ class PangeaToken {
       case ActivityTypeEnum.wordMeaning:
         return canBeDefined;
       case ActivityTypeEnum.wordFocusListening:
+        return false;
       case ActivityTypeEnum.hiddenWordListening:
         return canBeHeard;
     }

lib/pangea/models/practice_activities.dart/practice_activity_model.dart

@@ -4,6 +4,7 @@ import 'package:collection/collection.dart';
 import 'package:fluffychat/pangea/enum/activity_display_instructions_enum.dart';
 import 'package:fluffychat/pangea/enum/activity_type_enum.dart';
 import 'package:fluffychat/pangea/enum/construct_type_enum.dart';
+import 'package:fluffychat/pangea/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/models/practice_activities.dart/multiple_choice_activity_model.dart';
 import 'package:fluffychat/pangea/utils/error_handler.dart';
 import 'package:flutter/foundation.dart';
@@ -200,13 +201,20 @@ class PracticeActivityRequest {
 }
 
 class PracticeActivityModel {
+  // deprecated in favor of targetTokens
   final List<ConstructIdentifier> tgtConstructs;
+
+  // being added after creation from request info
+  // TODO - replace tgtConstructs with targetTokens in server return
+  List<PangeaToken>? targetTokens;
+
   final String langCode;
   final ActivityTypeEnum activityType;
   final ActivityContent content;
 
   PracticeActivityModel({
     required this.tgtConstructs,
+    required this.targetTokens,
     required this.langCode,
     required this.activityType,
     required this.content,
@@ -244,6 +252,11 @@ class PracticeActivityModel {
       activityType:
           ActivityTypeEnum.wordMeaning.fromString(json['activity_type']),
       content: ActivityContent.fromJson(contentMap),
+      targetTokens: json['target_tokens'] is List
+          ? (json['target_tokens'] as List)
+              .map((e) => PangeaToken.fromJson(e as Map<String, dynamic>))
+              .toList()
+          : null,
     );
   }
 
@@ -256,6 +269,7 @@ class PracticeActivityModel {
       'lang_code': langCode,
       'activity_type': activityType.string,
       'content': content.toJson(),
+      'target_tokens': targetTokens?.map((e) => e.toJson()).toList(),
     };
   }
 

lib/pangea/widgets/chat/message_audio_card.dart

@@ -59,11 +59,11 @@ class MessageAudioCardState extends State<MessageAudioCard> {
 
   @override
   void didUpdateWidget(covariant oldWidget) {
-    if (oldWidget.selection != widget.selection && widget.selection != null) {
-      debugPrint('selection changed');
-      setSectionStartAndEndFromSelection();
-      playSelectionAudio();
-    }
+    // if (oldWidget.selection != widget.selection && widget.selection != null) {
+    //   debugPrint('selection changed');
+    //   setSectionStartAndEndFromSelection();
+    //   playSelectionAudio();
+    // }
     super.didUpdateWidget(oldWidget);
   }
 

lib/pangea/widgets/chat/message_selection_overlay.dart

@@ -18,7 +18,6 @@ import 'package:fluffychat/pangea/widgets/chat/message_toolbar_buttons.dart';
 import 'package:fluffychat/pangea/widgets/chat/overlay_footer.dart';
 import 'package:fluffychat/pangea/widgets/chat/overlay_header.dart';
 import 'package:fluffychat/pangea/widgets/chat/overlay_message.dart';
-import 'package:fluffychat/pangea/widgets/chat/tts_controller.dart';
 import 'package:fluffychat/widgets/avatar.dart';
 import 'package:fluffychat/widgets/matrix.dart';
 import 'package:flutter/foundation.dart';
@@ -67,7 +66,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
 
   PangeaMessageEvent? get pangeaMessageEvent => widget._pangeaMessageEvent;
 
-  final TtsController tts = TtsController();
   bool _isPlayingAudio = false;
 
   bool get showToolbarButtons =>
@@ -139,8 +137,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
         );
       },
     ).listen((_) => setState(() {}));
-
-    tts.setupTTS();
   }
 
   MessageAnalyticsEntry? get messageAnalyticsEntry =>
@@ -297,6 +293,14 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
       }
     }
 
+    if (_selectedSpan != null) {
+      widget.chatController.choreographer.tts.tryToSpeak(
+        token.text.content,
+        context,
+        pangeaMessageEvent!.eventId,
+      );
+    }
+
     setState(() {});
   }
 
@@ -450,7 +454,7 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
   void dispose() {
     _animationController.dispose();
     _reactionSubscription?.cancel();
-    tts.dispose();
+
     super.dispose();
   }
 
@@ -562,7 +566,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
               MessageToolbar(
                 pangeaMessageEvent: pangeaMessageEvent!,
                 overLayController: this,
-                ttsController: tts,
               ),
             const SizedBox(height: 8),
             SizedBox(

lib/pangea/widgets/chat/message_toolbar.dart

@@ -27,15 +27,16 @@ const double minCardHeight = 70;
 class MessageToolbar extends StatelessWidget {
   final PangeaMessageEvent pangeaMessageEvent;
   final MessageOverlayController overLayController;
-  final TtsController ttsController;
 
   const MessageToolbar({
     super.key,
     required this.pangeaMessageEvent,
     required this.overLayController,
-    required this.ttsController,
   });
 
+  TtsController get ttsController =>
+      overLayController.widget.chatController.choreographer.tts;
+
   Widget toolbarContent(BuildContext context) {
     final bool subscribed =
         MatrixState.pangeaController.subscriptionController.isSubscribed;
@@ -135,7 +136,6 @@ class MessageToolbar extends StatelessWidget {
         return PracticeActivityCard(
           pangeaMessageEvent: pangeaMessageEvent,
           overlayController: overLayController,
-          ttsController: ttsController,
         );
       default:
         debugger(when: kDebugMode);

lib/pangea/widgets/chat/tts_controller.dart

@@ -145,7 +145,8 @@ class TtsController {
   Future<void> tryToSpeak(
     String text,
     BuildContext context,
-    String eventID,
+    // TODO - make non-nullable again
+    String? eventID,
   ) async {
     if (_isLanguageFullySupported) {
       await _speak(text);
@@ -157,7 +158,9 @@ class TtsController {
           'availableLangCodes': _availableLangCodes,
         },
       );
-      await _showMissingVoicePopup(context, eventID);
+      if (eventID != null) {
+        await _showMissingVoicePopup(context, eventID);
+      }
     }
   }
 

lib/pangea/widgets/igc/span_card.dart

@@ -280,6 +280,7 @@ class WordMatchContent extends StatelessWidget {
                           uniqueKeyForLayerLink: (int index) =>
                               "wordMatch$index",
                           selectedChoiceIndex: controller.selectedChoiceIndex,
+                          tts: controller.widget.scm.choreographer.tts,
                         ),
                       const SizedBox(height: 12),
                       PromptAndFeedback(controller: controller),

lib/pangea/widgets/practice_activity/multiple_choice_activity.dart

@@ -5,6 +5,7 @@ import 'package:fluffychat/config/app_config.dart';
 import 'package:fluffychat/pangea/choreographer/widgets/choice_array.dart';
 import 'package:fluffychat/pangea/controllers/put_analytics_controller.dart';
 import 'package:fluffychat/pangea/enum/activity_type_enum.dart';
+import 'package:fluffychat/pangea/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/models/practice_activities.dart/practice_activity_model.dart';
 import 'package:fluffychat/pangea/models/practice_activities.dart/practice_activity_record_model.dart';
 import 'package:fluffychat/pangea/utils/error_handler.dart';
@@ -21,7 +22,6 @@ import 'package:matrix/matrix.dart';
 class MultipleChoiceActivity extends StatefulWidget {
   final PracticeActivityCardState practiceCardController;
   final PracticeActivityModel currentActivity;
-  final TtsController tts;
   final Event event;
   final VoidCallback? onError;
 
@@ -29,7 +29,6 @@ class MultipleChoiceActivity extends StatefulWidget {
     super.key,
     required this.practiceCardController,
     required this.currentActivity,
-    required this.tts,
     required this.event,
     this.onError,
   });
@@ -46,6 +45,8 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
 
   @override
   void initState() {
+    speakTargetTokens();
+
     super.initState();
   }
 
@@ -55,18 +56,59 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
     if (widget.practiceCardController.currentCompletionRecord?.responses
             .isEmpty ??
         false) {
+      speakTargetTokens();
+
       setState(() => selectedChoiceIndex = null);
     }
   }
 
-  void updateChoice(String value, int index) {
-    if (currentRecordModel?.hasTextResponse(value) ?? false) {
-      return;
+  void speakTargetTokens() {
+    if (widget.practiceCardController.currentActivity?.targetTokens != null) {
+      widget.practiceCardController.tts.tryToSpeak(
+        PangeaToken.reconstructText(
+          widget.practiceCardController.currentActivity!.targetTokens!,
+        ),
+        context,
+        null,
+      );
     }
+  }
+
+  TtsController get tts => widget.practiceCardController.tts;
 
+  void updateChoice(String value, int index) {
     final bool isCorrect =
         widget.currentActivity.content.isCorrect(value, index);
 
+    // If the activity is not set to include TTS on click, and the choice is correct, speak the target tokens
+    // We have to check if tokens
+    if (!widget.currentActivity.activityType.includeTTSOnClick &&
+        isCorrect &&
+        mounted) {
+      // should be set by now but just in case we make a mistake
+      if (widget.practiceCardController.currentActivity?.targetTokens == null) {
+        debugger(when: kDebugMode);
+        ErrorHandler.logError(
+          e: "Missing target tokens in multiple choice activity",
+          data: {
+            "currentActivity": widget.practiceCardController.currentActivity,
+          },
+        );
+      } else {
+        tts.tryToSpeak(
+          PangeaToken.reconstructText(
+            widget.practiceCardController.currentActivity!.targetTokens!,
+          ),
+          context,
+          null,
+        );
+      }
+    }
+
+    if (currentRecordModel?.hasTextResponse(value) ?? false) {
+      return;
+    }
+
     currentRecordModel?.addResponse(
       text: value,
       score: isCorrect ? 1 : 0,
@@ -136,7 +178,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
             ActivityTypeEnum.wordFocusListening)
           WordAudioButton(
             text: practiceActivity.content.answer,
-            ttsController: widget.tts,
+            ttsController: tts,
             eventID: widget.event.eventId,
           ),
         if (practiceActivity.activityType ==
@@ -146,7 +188,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
                 widget.practiceCardController.widget.pangeaMessageEvent,
             overlayController:
                 widget.practiceCardController.widget.overlayController,
-            tts: widget.practiceCardController.widget.overlayController.tts,
+            tts: tts,
             setIsPlayingAudio: widget.practiceCardController.widget
                 .overlayController.setIsPlayingAudio,
             onError: widget.onError,
@@ -170,6 +212,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
               .toList(),
           isActive: true,
           id: currentRecordModel?.hashCode.toString(),
+          tts: practiceActivity.activityType.includeTTSOnClick ? tts : null,
         ),
       ],
     );

lib/pangea/widgets/practice_activity/practice_activity_card.dart

@@ -30,13 +30,11 @@ import 'package:flutter/material.dart';
 class PracticeActivityCard extends StatefulWidget {
   final PangeaMessageEvent pangeaMessageEvent;
   final MessageOverlayController overlayController;
-  final TtsController ttsController;
 
   const PracticeActivityCard({
     super.key,
     required this.pangeaMessageEvent,
     required this.overlayController,
-    required this.ttsController,
   });
 
   @override
@@ -59,6 +57,9 @@ class PracticeActivityCardState extends State<PracticeActivityCard> {
   Duration appropriateTimeForJoy = const Duration(milliseconds: 1500);
   bool savoringTheJoy = false;
 
+  TtsController get tts =>
+      widget.overlayController.widget.chatController.choreographer.tts;
+
   @override
   void initState() {
     super.initState();
@@ -100,76 +101,86 @@ class PracticeActivityCardState extends State<PracticeActivityCard> {
   Future<PracticeActivityModel?> _fetchActivity({
     ActivityQualityFeedback? activityFeedback,
   }) async {
-    // try {
-    debugPrint('Fetching activity');
-    _updateFetchingActivity(true);
-
-    // target tokens can be empty if activities have been completed for each
-    // it's set on initialization and then removed when each activity is completed
-    if (!mounted ||
-        !pangeaController.languageController.languagesSet ||
-        widget.overlayController.messageAnalyticsEntry == null) {
-      debugger(when: kDebugMode);
-      _updateFetchingActivity(false);
-      return null;
-    }
+    try {
+      debugPrint('Fetching activity');
+      _updateFetchingActivity(true);
+
+      // target tokens can be empty if activities have been completed for each
+      // it's set on initialization and then removed when each activity is completed
+      if (!mounted ||
+          !pangeaController.languageController.languagesSet ||
+          widget.overlayController.messageAnalyticsEntry == null) {
+        debugger(when: kDebugMode);
+        _updateFetchingActivity(false);
+        return null;
+      }
 
-    final nextActivitySpecs =
-        widget.overlayController.messageAnalyticsEntry?.nextActivity;
-    // the client is going to be choosing the next activity now
-    // if nothing is set then it must be done with practice
-    if (nextActivitySpecs == null) {
-      debugPrint("No next activity set, exiting practice flow");
-      _updateFetchingActivity(false);
-      return null;
-    }
+      final nextActivitySpecs =
+          widget.overlayController.messageAnalyticsEntry?.nextActivity;
+      // the client is going to be choosing the next activity now
+      // if nothing is set then it must be done with practice
+      if (nextActivitySpecs == null) {
+        debugPrint("No next activity set, exiting practice flow");
+        _updateFetchingActivity(false);
+        return null;
+      }
 
-    // check if we already have an activity matching the specs
-    final existingActivity = practiceActivities.firstWhereOrNull(
-      (activity) =>
-          nextActivitySpecs.matchesActivity(activity.practiceActivity),
-    );
-    if (existingActivity != null) {
-      debugPrint('found existing activity');
+      // check if we already have an activity matching the specs
+      final existingActivity = practiceActivities.firstWhereOrNull(
+        (activity) =>
+            nextActivitySpecs.matchesActivity(activity.practiceActivity),
+      );
+      if (existingActivity != null) {
+        debugPrint('found existing activity');
+        _updateFetchingActivity(false);
+        existingActivity.practiceActivity.targetTokens =
+            nextActivitySpecs.tokens;
+        return existingActivity.practiceActivity;
+      }
+
+      debugPrint(
+        "client requesting ${nextActivitySpecs.activityType.string} for: ${nextActivitySpecs.tokens.map((t) => "word: ${t.text.content} xp: ${t.xp}").join(' ')}",
+      );
+
+      final PracticeActivityModelResponse? activityResponse =
+          await pangeaController.practiceGenerationController
+              .getPracticeActivity(
+        MessageActivityRequest(
+          userL1: pangeaController.languageController.userL1!.langCode,
+          userL2: pangeaController.languageController.userL2!.langCode,
+          messageText: widget.pangeaMessageEvent.messageDisplayText,
+          messageTokens: widget.overlayController.tokens!,
+          activityQualityFeedback: activityFeedback,
+          targetTokens: nextActivitySpecs.tokens,
+          targetType: nextActivitySpecs.activityType,
+        ),
+        widget.pangeaMessageEvent,
+      );
+
+      currentActivityCompleter = activityResponse?.eventCompleter;
       _updateFetchingActivity(false);
-      return existingActivity.practiceActivity;
-    }
 
-    debugPrint(
-      "client requesting ${nextActivitySpecs.activityType.string} for: ${nextActivitySpecs.tokens.map((t) => "word: ${t.text.content} xp: ${t.xp}").join(' ')}",
-    );
+      if (activityResponse == null || activityResponse.activity == null) {
+        debugPrint('No activity found');
+        return null;
+      }
 
-    final PracticeActivityModelResponse? activityResponse =
-        await pangeaController.practiceGenerationController.getPracticeActivity(
-      MessageActivityRequest(
-        userL1: pangeaController.languageController.userL1!.langCode,
-        userL2: pangeaController.languageController.userL2!.langCode,
-        messageText: widget.pangeaMessageEvent.messageDisplayText,
-        messageTokens: widget.overlayController.tokens!,
-        activityQualityFeedback: activityFeedback,
-        targetTokens: nextActivitySpecs.tokens,
-        targetType: nextActivitySpecs.activityType,
-      ),
-      widget.pangeaMessageEvent,
-    );
+      activityResponse.activity!.targetTokens = nextActivitySpecs.tokens;
 
-    currentActivityCompleter = activityResponse?.eventCompleter;
-    _updateFetchingActivity(false);
-
-    return activityResponse?.activity;
-    // } catch (e, s) {
-    //   debugger(when: kDebugMode);
-    //   ErrorHandler.logError(
-    //     e: e,
-    //     s: s,
-    //     m: 'Failed to get new activity',
-    //     data: {
-    //       'activity': currentActivity,
-    //       'record': currentCompletionRecord,
-    //     },
-    //   );
-    //   return null;
-    // }
+      return activityResponse.activity;
+    } catch (e, s) {
+      debugger(when: kDebugMode);
+      ErrorHandler.logError(
+        e: e,
+        s: s,
+        m: 'Failed to get new activity',
+        data: {
+          'activity': currentActivity,
+          'record': currentCompletionRecord,
+        },
+      );
+      return null;
+    }
   }
 
   ConstructUseMetaData get metadata => ConstructUseMetaData(
@@ -313,7 +324,6 @@ class PracticeActivityCardState extends State<PracticeActivityCard> {
         return MultipleChoiceActivity(
           practiceCardController: this,
           currentActivity: currentActivity!,
-          tts: widget.ttsController,
           event: widget.pangeaMessageEvent.event,
           onError: _onError,
         );