Merge branch 'main' into public-space-refinement

pull/2245/head
ggurdin 5 months ago committed by GitHub
commit 8c433245a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -5014,5 +5014,6 @@
"getStartedFriendsButton": "Invite a friend",
"groupChat": "Group Chat",
"directMessage": "Direct Message",
"newDirectMessage": "New direct message"
"newDirectMessage": "New direct message",
"speakingExercisesTooltip": "Speaking practice"
}

@ -51,6 +51,7 @@ import 'package:fluffychat/pangea/events/models/representation_content_model.dar
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
import 'package:fluffychat/pangea/instructions/instructions_enum.dart';
import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
import 'package:fluffychat/pangea/learning_settings/widgets/p_language_dialog.dart';
import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
@ -924,31 +925,7 @@ class ChatController extends State<ChatPageWithRoom>
),
];
final newGrammarConstructs =
pangeaController.getAnalytics.newConstructCount(
constructs,
ConstructTypeEnum.morph,
);
final newVocabConstructs =
pangeaController.getAnalytics.newConstructCount(
constructs,
ConstructTypeEnum.vocab,
);
OverlayUtil.showOverlay(
overlayKey: "msg_analytics_feedback_$msgEventId",
followerAnchor: Alignment.bottomRight,
targetAnchor: Alignment.topRight,
context: context,
child: MessageAnalyticsFeedback(
overlayId: "msg_analytics_feedback_$msgEventId",
newGrammarConstructs: newGrammarConstructs,
newVocabConstructs: newVocabConstructs,
),
transformTargetId: msgEventId,
ignorePointer: true,
);
_showAnalyticsFeedback(constructs, msgEventId);
pangeaController.putAnalytics.setState(
AnalyticsStream(
@ -1130,44 +1107,54 @@ class ChatController extends State<ChatPageWithRoom>
name: result.fileName ?? audioFile.path,
);
await room.sendFileEvent(
file,
inReplyTo: replyEvent,
extraContent: {
'info': {
...file.info,
'duration': result.duration,
},
'org.matrix.msc3245.voice': {},
'org.matrix.msc1767.audio': {
'duration': result.duration,
'waveform': result.waveform,
},
},
// #Pangea
// ).catchError((e) {
).catchError((e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
'roomId': roomId,
'file': file.name,
'duration': result.duration,
'waveform': result.waveform,
},
);
// Pangea#
scaffoldMessenger.showSnackBar(
SnackBar(
content: Text(
(e as Object).toLocalizedString(context),
),
),
);
return null;
});
// #Pangea
await room
.sendFileEvent(
file,
inReplyTo: replyEvent,
extraContent: {
'info': {
...file.info,
'duration': result.duration,
},
'org.matrix.msc3245.voice': {},
'org.matrix.msc1767.audio': {
'duration': result.duration,
'waveform': result.waveform,
},
},
// #Pangea
)
.then(_sendVoiceMessageAnalytics)
.catchError((e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
'roomId': roomId,
'file': file.name,
'duration': result.duration,
'waveform': result.waveform,
},
);
scaffoldMessenger.showSnackBar(
SnackBar(
content: Text(
(e as Object).toLocalizedString(context),
),
),
);
return null;
});
// ).catchError((e) {
// scaffoldMessenger.showSnackBar(
// SnackBar(
// content: Text(
// (e as Object).toLocalizedString(context),
// ),
// ),
// );
// return null;
// });
// setState(() {
// replyEvent = null;
// });
@ -1618,7 +1605,8 @@ class ChatController extends State<ChatPageWithRoom>
if (timeline == null ||
events.any(
(e) => e.aggregatedEvents(timeline!, RelationshipTypes.reaction).any(
(re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji),
(re) => re.content.tryGetMap('m.relates_to')?['key'] == emoji,
),
)) {
return;
}
@ -2058,6 +2046,97 @@ class ChatController extends State<ChatPageWithRoom>
return false;
}
}
Future<void> _sendVoiceMessageAnalytics(String? eventId) async {
if (eventId == null) {
ErrorHandler.logError(
e: Exception('eventID null in voiceMessageAction'),
s: StackTrace.current,
data: {
'roomId': roomId,
},
);
return;
}
final event = await room.getEventById(eventId);
if (event == null) {
ErrorHandler.logError(
e: Exception('Event not found after sending voice message'),
s: StackTrace.current,
data: {
'roomId': roomId,
},
);
return;
}
try {
final messageEvent = PangeaMessageEvent(
event: event,
timeline: timeline!,
ownMessage: true,
);
final stt = await messageEvent.getSpeechToText(
choreographer.l1Lang?.langCodeShort ?? LanguageKeys.unknownLanguage,
choreographer.l2Lang?.langCodeShort ?? LanguageKeys.unknownLanguage,
);
if (stt == null || stt.transcript.sttTokens.isEmpty) return;
final constructs = stt.constructs(roomId, eventId);
if (constructs.isEmpty) return;
_showAnalyticsFeedback(constructs, eventId);
MatrixState.pangeaController.putAnalytics.setState(
AnalyticsStream(
eventId: eventId,
targetID: eventId,
roomId: room.id,
constructs: constructs,
),
);
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
'roomId': roomId,
'eventId': eventId,
},
);
}
}
void _showAnalyticsFeedback(
List<OneConstructUse> constructs,
String eventId,
) {
final newGrammarConstructs =
pangeaController.getAnalytics.newConstructCount(
constructs,
ConstructTypeEnum.morph,
);
final newVocabConstructs = pangeaController.getAnalytics.newConstructCount(
constructs,
ConstructTypeEnum.vocab,
);
OverlayUtil.showOverlay(
overlayKey: "msg_analytics_feedback_$eventId",
followerAnchor: Alignment.bottomRight,
targetAnchor: Alignment.topRight,
context: context,
child: MessageAnalyticsFeedback(
overlayId: "msg_analytics_feedback_$eventId",
newGrammarConstructs: newGrammarConstructs,
newVocabConstructs: newVocabConstructs,
),
transformTargetId: eventId,
ignorePointer: true,
closePrevOverlay: false,
);
}
// Pangea#
late final ValueNotifier<bool> _displayChatDetailsColumn;

@ -22,7 +22,7 @@ import 'package:fluffychat/utils/file_description.dart';
import 'package:fluffychat/utils/localized_exception_extension.dart';
import 'package:fluffychat/utils/matrix_sdk_extensions/event_extension.dart';
import 'package:fluffychat/utils/url_launcher.dart';
import '../../../widgets/fluffy_chat_app.dart';
import 'package:fluffychat/widgets/fluffy_chat_app.dart';
import '../../../widgets/matrix.dart';
class AudioPlayerWidget extends StatefulWidget {
@ -36,9 +36,10 @@ class AudioPlayerWidget extends StatefulWidget {
final String roomId;
final String senderId;
final PangeaAudioFile? matrixFile;
final Function(bool)? setIsPlayingAudio;
final ChatController chatController;
final MessageOverlayController? overlayController;
final VoidCallback? onPlay;
final bool autoplay;
// Pangea#
static const int wavesCount = 40;
@ -53,9 +54,10 @@ class AudioPlayerWidget extends StatefulWidget {
required this.roomId,
required this.senderId,
this.matrixFile,
this.setIsPlayingAudio,
required this.chatController,
this.overlayController,
this.onPlay,
this.autoplay = false,
// Pangea#
super.key,
});
@ -76,9 +78,7 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
String? _durationString;
// #Pangea
StreamSubscription? _onShowToolbar;
StreamSubscription? _onAudioPositionChanged;
StreamSubscription? _onPlayerStateChanged;
// Pangea#
@override
@ -163,9 +163,7 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
audioPlayer.dispose();
matrix.voiceMessageEventId.value = matrix.audioPlayer = null;
// #Pangea
_onShowToolbar?.cancel();
_onAudioPositionChanged?.cancel();
_onPlayerStateChanged?.cancel();
// Pangea#
}
}
@ -253,6 +251,8 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
// #Pangea
// if (matrix.voiceMessageEventId.value != widget.event.eventId) return;
if (matrix.voiceMessageEventId.value != widget.eventId) return;
matrix.audioPlayer?.dispose();
// Pangea#
final audioPlayer = matrix.audioPlayer = AudioPlayer();
@ -269,13 +269,6 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
);
}
});
_onPlayerStateChanged?.cancel();
_onPlayerStateChanged = audioPlayer.playingStream.listen(
(isPlaying) => setState(() {
widget.setIsPlayingAudio?.call(isPlaying);
}),
);
// Pangea#
// #Pangea
@ -394,6 +387,10 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
final duration = Duration(milliseconds: durationInt);
_durationString = duration.minuteSecondString;
}
// #Pangea
if (widget.autoplay) _onButtonTap();
// Pangea#
}
@override
@ -465,7 +462,11 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
onLongPress: () =>
widget.event?.saveFile(context),
// Pangea#
onTap: _onButtonTap,
onTap: () {
widget.onPlay != null
? widget.onPlay!.call()
: _onButtonTap();
},
child: Material(
color: widget.color.withAlpha(64),
borderRadius: BorderRadius.circular(64),

@ -225,6 +225,15 @@ class MessageContent extends StatelessWidget {
eventId: event.eventId,
roomId: event.room.id,
senderId: event.senderId,
onPlay: overlayController == null
? () {
controller.showToolbar(
pangeaMessageEvent!.event,
pangeaMessageEvent: pangeaMessageEvent,
);
}
: null,
autoplay: overlayController != null,
// Pangea#
);
}

@ -97,7 +97,6 @@ class ActivitySuggestionsAreaState extends State<ActivitySuggestionsArea> {
setState(() {
_activityItems.clear();
_loading = true;
_timeout = false;
});
final ActivityPlanRequest request = ActivityPlanRequest(
@ -124,10 +123,16 @@ class ActivitySuggestionsAreaState extends State<ActivitySuggestionsArea> {
Future.delayed(const Duration(seconds: 5), () {
if (mounted) _setActivityItems(retries: retries + 1);
});
return ActivityPlanResponse(activityPlans: []);
return Future<ActivityPlanResponse>.error(
TimeoutException(
L10n.of(context).activitySuggestionTimeoutMessage,
),
);
},
);
_activityItems.addAll(resp.activityPlans);
_timeout = false;
} finally {
if (mounted) setState(() => _loading = false);
}

@ -7,7 +7,7 @@ import 'package:fluffychat/l10n/l10n.dart';
enum LearningSkillsEnum {
writing(isVisible: true, icon: Symbols.edit_square),
reading(isVisible: true, icon: Symbols.two_pager),
speaking(isVisible: false),
speaking(isVisible: true, icon: Icons.mic_outlined),
hearing(isVisible: true, icon: Icons.volume_up),
other(isVisible: false);
@ -27,6 +27,8 @@ enum LearningSkillsEnum {
return L10n.of(context).readingExercisesTooltip;
case LearningSkillsEnum.hearing:
return L10n.of(context).listeningExercisesTooltip;
case LearningSkillsEnum.speaking:
return L10n.of(context).speakingExercisesTooltip;
default:
return "";
}

@ -16,6 +16,7 @@ class PangeaEventTypes {
static const tokens = "pangea.tokens";
static const choreoRecord = "pangea.record";
static const representation = "pangea.representation";
static const sttTranslation = "pangea.stt_translation";
// static const vocab = "p.vocab";
static const roomInfo = "pangea.roomtopic";

@ -13,6 +13,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
@ -24,6 +25,7 @@ class MessageDataController extends BaseController {
final Map<int, Future<TokensResponseModel>> _tokensCache = {};
final Map<int, Future<PangeaRepresentation>> _representationCache = {};
final Map<int, Future<SttTranslationModel>> _sttTranslationCache = {};
late Timer _cacheTimer;
MessageDataController(PangeaController pangeaController) {
@ -42,6 +44,7 @@ class MessageDataController extends BaseController {
void _clearCache() {
_tokensCache.clear();
_representationCache.clear();
_sttTranslationCache.clear();
debugPrint("message data cache cleared.");
}
@ -219,4 +222,53 @@ class MessageDataController extends BaseController {
);
}
}
Future<SttTranslationModel> getSttTranslation({
required String? repEventId,
required FullTextTranslationRequestModel req,
required Room? room,
}) =>
_sttTranslationCache[req.hashCode] ??= _getSttTranslation(
repEventId: repEventId,
req: req,
room: room,
).catchError((e, s) {
_sttTranslationCache.remove(req.hashCode);
return Future<SttTranslationModel>.error(e, s);
});
Future<SttTranslationModel> _getSttTranslation({
required String? repEventId,
required FullTextTranslationRequestModel req,
required Room? room,
}) async {
final res = await FullTextTranslationRepo.translate(
accessToken: _pangeaController.userController.accessToken,
request: req,
);
final translation = SttTranslationModel(
translation: res.bestTranslation,
langCode: req.tgtLang,
);
if (repEventId != null && room != null) {
room
.sendPangeaEvent(
content: translation.toJson(),
parentEventId: repEventId,
type: PangeaEventTypes.sttTranslation,
)
.catchError(
(e) => ErrorHandler.logError(
m: "error in _getSttTranslation.sendPangeaEvent",
e: e,
s: StackTrace.current,
data: req.toJson(),
),
);
}
return translation;
}
}

@ -16,6 +16,7 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_ev
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/morphs/morph_features_enum.dart';
import 'package:fluffychat/pangea/practice_activities/activity_type_enum.dart';
@ -281,32 +282,11 @@ class PangeaMessageEvent {
?.content
.speechToText;
if (speechToTextLocal != null) return speechToTextLocal;
if (speechToTextLocal != null) {
return speechToTextLocal;
}
final matrixFile = await _event.downloadAndDecryptAttachment();
// Pangea#
// File? file;
// TODO: Test on mobile and see if we need this case, doeesn't seem so
// if (!kIsWeb) {
// final tempDir = await getTemporaryDirectory();
// final fileName = Uri.encodeComponent(
// // #Pangea
// // widget.event.attachmentOrThumbnailMxcUrl()!.pathSegments.last,
// widget.messageEvent.event
// .attachmentOrThumbnailMxcUrl()!
// .pathSegments
// .last,
// // Pangea#
// );
// file = File('${tempDir.path}/${fileName}_${matrixFile.name}');
// await file.writeAsBytes(matrixFile.bytes);
// }
// audioFile = file;
debugPrint("mimeType ${matrixFile.mimeType}");
debugPrint("encoding ${mimeTypeToAudioEncoding(matrixFile.mimeType)}");
final SpeechToTextModel response =
await MatrixState.pangeaController.speechToText.get(
@ -341,6 +321,25 @@ class PangeaMessageEvent {
return response;
}
Future<SttTranslationModel?> sttTranslationByLanguageGlobal({
required String langCode,
required String l1Code,
required String l2Code,
}) async {
if (!representations.any(
(element) => element.content.speechToText != null,
)) {
await getSpeechToText(l1Code, l2Code);
}
final rep = representations.firstWhereOrNull(
(element) => element.content.speechToText != null,
);
if (rep == null) return null;
return rep.getSttTranslation(userL1: l1Code, userL2: l2Code);
}
PangeaMessageTokens? _tokensSafe(Map<String, dynamic>? content) {
try {
if (content == null) return null;

@ -12,11 +12,13 @@ import 'package:sentry_flutter/sentry_flutter.dart';
import 'package:fluffychat/pangea/choreographer/event_wrappers/pangea_choreo_event.dart';
import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart';
import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart';
import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_repo.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
import 'package:fluffychat/pangea/events/models/stt_translation_model.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
@ -210,6 +212,71 @@ class RepresentationEvent {
);
}
List<SttTranslationModel> get sttTranslations {
if (content.speechToText == null) return [];
if (_event == null) {
Sentry.addBreadcrumb(
Breadcrumb(
message: "_event and _sttTranslations both null",
),
);
return [];
}
final Set<Event> sttEvents = _event!.aggregatedEvents(
timeline,
PangeaEventTypes.sttTranslation,
);
if (sttEvents.isEmpty) return [];
final List<SttTranslationModel> sttTranslations = [];
for (final event in sttEvents) {
try {
sttTranslations.add(
SttTranslationModel.fromJson(event.content),
);
} catch (e) {
Sentry.addBreadcrumb(
Breadcrumb(
message: "Failed to parse STT translation",
data: {
"eventID": event.eventId,
"content": event.content,
"error": e.toString(),
},
),
);
}
}
return sttTranslations;
}
Future<SttTranslationModel> getSttTranslation({
required String userL1,
required String userL2,
}) async {
if (content.speechToText == null) {
throw Exception(
"RepresentationEvent.getSttTranslation called on a representation without speechToText",
);
}
final local = sttTranslations.firstWhereOrNull((t) => t.langCode == userL1);
if (local != null) return local;
return MatrixState.pangeaController.messageData.getSttTranslation(
repEventId: _event?.eventId,
room: _event?.room,
req: FullTextTranslationRequestModel(
text: content.speechToText!.transcript.text,
tgtLang: userL1,
userL2: userL2,
userL1: userL1,
),
);
}
ChoreoRecord? get choreo {
if (_choreo != null) return _choreo;

@ -1,42 +0,0 @@
import 'dart:developer';
import 'package:flutter/foundation.dart';
import 'package:matrix/matrix.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
import '../constants/pangea_event_types.dart';
class TokensEvent {
Event event;
PangeaMessageTokens? _content;
TokensEvent({required this.event}) {
if (event.type != PangeaEventTypes.tokens) {
throw Exception(
"${event.type} should not be used to make a TokensEvent",
);
}
}
PangeaMessageTokens? get _pangeaMessageTokens {
try {
_content ??= event.getPangeaContent<PangeaMessageTokens>();
return _content!;
} catch (err, s) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: err,
s: s,
data: {
"event": event.toJson(),
},
);
return null;
}
}
PangeaMessageTokens? get tokens => _pangeaMessageTokens;
}

@ -0,0 +1,23 @@
class SttTranslationModel {
final String translation;
final String langCode;
SttTranslationModel({
required this.translation,
required this.langCode,
});
factory SttTranslationModel.fromJson(Map<String, dynamic> json) {
return SttTranslationModel(
translation: json['translation'] as String,
langCode: json['lang_code'] as String,
);
}
Map<String, dynamic> toJson() {
return {
'translation': translation,
'lang_code': langCode,
};
}
}

@ -210,9 +210,8 @@ class UserSettingsState extends State<UserSettingsPage> {
_pangeaController.subscriptionController.reinitialize(),
_pangeaController.userController.updateProfile(
(profile) {
if (_systemLanguage != null) {
profile.userSettings.sourceLanguage = _systemLanguage!.langCode;
}
profile.userSettings.sourceLanguage =
selectedBaseLanguage?.langCode ?? _systemLanguage?.langCode;
profile.userSettings.targetLanguage =
selectedTargetLanguage!.langCode;
profile.userSettings.cefrLevel = selectedCefrLevel;

@ -40,13 +40,13 @@ class OnboardingController extends State<Onboarding> {
(r) => r.isSpace,
);
case OnboardingStepsEnum.inviteFriends:
return hasInvitedFriends;
return MatrixState.pangeaController.matrixState.client.rooms.any(
(r) =>
r.isDirectChat && r.directChatMatrixID != BotName.byEnvironment,
);
}
}
static bool get hasInvitedFriends =>
_onboardingStorage.read('invite_friends') ?? false;
static bool get hasBotDM =>
MatrixState.pangeaController.matrixState.client.rooms.any((room) {
if (room.isDirectChat &&
@ -66,8 +66,6 @@ class OnboardingController extends State<Onboarding> {
Future<void> inviteFriends() async {
FluffyShare.shareInviteLink(context);
await _onboardingStorage.write('invite_friends', true);
if (mounted) setState(() {});
}
Future<void> startChatWithBot() async {

@ -6,6 +6,8 @@ import 'package:flutter/material.dart';
import 'package:matrix/matrix.dart';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/pangea/analytics_misc/construct_use_type_enum.dart';
import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart';
import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/toolbar/enums/audio_encoding_enum.dart';
@ -230,4 +232,28 @@ class SpeechToTextModel {
Map<String, dynamic> toJson() => {
"results": results.map((e) => e.toJson()).toList(),
};
List<OneConstructUse> constructs(
String roomId,
String eventId,
) {
final List<OneConstructUse> constructs = [];
final metadata = ConstructUseMetaData(
roomId: roomId,
eventId: eventId,
timeStamp: DateTime.now(),
);
for (final sstToken in transcript.sttTokens) {
final token = sstToken.token;
if (!token.lemma.saveVocab) continue;
constructs.addAll(
token.allUses(
ConstructUseTypeEnum.pvm,
metadata,
ConstructUseTypeEnum.pvm.pointValue,
),
);
}
return constructs;
}
}

@ -74,7 +74,6 @@ class MatchActivityCard extends StatelessWidget {
MessageAudioCard(
messageEvent: overlayController.pangeaMessageEvent!,
overlayController: overlayController,
setIsPlayingAudio: overlayController.setIsPlayingAudio,
),
Wrap(
alignment: WrapAlignment.center,

@ -18,14 +18,12 @@ import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart
class MessageAudioCard extends StatefulWidget {
final PangeaMessageEvent messageEvent;
final MessageOverlayController overlayController;
final Function(bool) setIsPlayingAudio;
final VoidCallback? onError;
const MessageAudioCard({
super.key,
required this.messageEvent,
required this.overlayController,
required this.setIsPlayingAudio,
this.onError,
});
@ -91,7 +89,6 @@ class MessageAudioCardState extends State<MessageAudioCard> {
senderId: widget.messageEvent.senderId,
matrixFile: audioFile,
color: Theme.of(context).colorScheme.onPrimaryContainer,
setIsPlayingAudio: widget.setIsPlayingAudio,
fontSize: AppConfig.messageFontSize * AppConfig.fontSizeFactor,
chatController: widget.overlayController.widget.chatController,
overlayController: widget.overlayController,

@ -30,6 +30,7 @@ import 'package:fluffychat/pangea/toolbar/controllers/text_to_speech_controller.
import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart';
import 'package:fluffychat/pangea/toolbar/enums/message_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
import 'package:fluffychat/pangea/toolbar/reading_assistance_input_row/morph_selection.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_positioner.dart';
import 'package:fluffychat/pangea/toolbar/widgets/reading_assistance_content.dart';
@ -86,13 +87,19 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
List<PangeaTokenText>? _highlightedTokens;
bool initialized = false;
bool isPlayingAudio = false;
final GlobalKey<ReadingAssistanceContentState> wordZoomKey = GlobalKey();
ReadingAssistanceMode? readingAssistanceMode; // default mode
SpeechToTextModel? transcription;
String? transcriptionError;
bool showTranslation = false;
String? translationText;
String? translation;
bool showSpeechTranslation = false;
String? speechTranslation;
double maxWidth = AppConfig.toolbarMinWidth;
@ -571,24 +578,53 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
);
}
void setIsPlayingAudio(bool isPlaying) {
void setTranslation(String value) {
if (mounted) {
setState(() => isPlayingAudio = isPlaying);
setState(() => translation = value);
}
}
void setShowTranslation(bool show, String? translation) {
void setShowTranslation(bool show) {
if (!mounted) return;
if (translation == null) {
setState(() => showTranslation = false);
}
if (showTranslation == show) return;
if (show && translation == null) return;
setState(() => showTranslation = show);
}
void setSpeechTranslation(String value) {
if (mounted) {
setState(() => speechTranslation = value);
}
}
void setShowSpeechTranslation(bool show) {
if (!mounted) return;
if (speechTranslation == null) {
setState(() => showSpeechTranslation = false);
}
if (showSpeechTranslation == show) return;
setState(() => showSpeechTranslation = show);
}
void setTranscription(SpeechToTextModel value) {
if (mounted) {
setState(() {
showTranslation = show;
translationText = show ? translation : null;
transcriptionError = null;
transcription = value;
});
}
}
void setTranscriptionError(String value) {
if (mounted) {
setState(() => transcriptionError = value);
}
}
/////////////////////////////////////
/// Build
/////////////////////////////////////

@ -138,6 +138,9 @@ class MessageSelectionPositionerState extends State<MessageSelectionPositioner>
void dispose() {
_animationController.dispose();
_reactionSubscription?.cancel();
MatrixState.pangeaController.matrixState.audioPlayer
?..stop()
..dispose();
super.dispose();
}
@ -490,10 +493,22 @@ class MessageSelectionPositionerState extends State<MessageSelectionPositioner>
// measurement for items in the toolbar
bool get _showButtons =>
(widget.pangeaMessageEvent?.shouldShowToolbar ?? false) &&
widget.pangeaMessageEvent?.event.messageType == MessageTypes.Text &&
(widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false);
bool get _showButtons {
if (!(widget.pangeaMessageEvent?.shouldShowToolbar ?? false)) {
return false;
}
final type = widget.pangeaMessageEvent?.event.messageType;
if (![MessageTypes.Text, MessageTypes.Audio].contains(type)) {
return false;
}
if (type == MessageTypes.Text) {
return widget.pangeaMessageEvent?.messageDisplayLangIsL2 ?? false;
}
return true;
}
bool get showPracticeButtons =>
_showButtons &&

@ -1,119 +0,0 @@
import 'dart:developer';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/l10n/l10n.dart';
import 'package:fluffychat/pangea/common/utils/error_handler.dart';
import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
import 'package:fluffychat/pangea/toolbar/models/speech_to_text_models.dart';
import 'package:fluffychat/widgets/matrix.dart';
class MessageSpeechToTextCard extends StatefulWidget {
final PangeaMessageEvent messageEvent;
final Color textColor;
const MessageSpeechToTextCard({
super.key,
required this.messageEvent,
required this.textColor,
});
@override
MessageSpeechToTextCardState createState() => MessageSpeechToTextCardState();
}
class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
SpeechToTextModel? _speechToTextResponse;
bool _fetchingTranscription = true;
Object? error;
String? get l1Code =>
MatrixState.pangeaController.languageController.activeL1Code();
String? get l2Code =>
MatrixState.pangeaController.languageController.activeL2Code();
@override
void initState() {
super.initState();
_fetchTranscription();
}
// look for transcription in message event
// if not found, call API to transcribe audio
Future<void> _fetchTranscription() async {
try {
if (l1Code == null || l2Code == null) {
throw Exception('Language selection not found');
}
_speechToTextResponse ??= await widget.messageEvent.getSpeechToText(
l1Code!,
l2Code!,
);
} catch (e, s) {
debugger(when: kDebugMode);
error = e;
ErrorHandler.logError(
e: e,
s: s,
data: widget.messageEvent.event.content,
);
} finally {
if (mounted) {
setState(() => _fetchingTranscription = false);
}
}
}
@override
Widget build(BuildContext context) {
if (_fetchingTranscription) {
return const LinearProgressIndicator();
}
// // done fetching but not results means some kind of error
if (_speechToTextResponse == null || error != null) {
return Row(
spacing: 8.0,
children: [
Flexible(
child: RichText(
text: TextSpan(
style: AppConfig.messageTextStyle(
widget.messageEvent.event,
widget.textColor,
),
children: [
WidgetSpan(
alignment: PlaceholderAlignment.middle,
child: Icon(
Icons.error,
color: Theme.of(context).colorScheme.error,
),
),
const TextSpan(text: " "),
TextSpan(
text: L10n.of(context).oopsSomethingWentWrong,
),
],
),
),
),
],
);
}
return Text(
"${_speechToTextResponse?.transcript.text}",
style: AppConfig.messageTextStyle(
widget.messageEvent.event,
widget.textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
);
}
}

@ -63,7 +63,11 @@ class OverlayHeaderState extends State<OverlayHeader> {
padding: const EdgeInsets.symmetric(vertical: 8.0),
child: Row(
children: [
if (controller.selectedEvents.length == 1)
// #Pangea
// if (controller.selectedEvents.length == 1)
if (controller.selectedEvents.length == 1 &&
controller.room.canSendDefaultMessages)
// Pangea#
IconButton(
icon: const Icon(Symbols.reply_all),
tooltip: l10n.reply,

@ -4,6 +4,7 @@ import 'package:flutter/material.dart';
import 'package:matrix/matrix.dart';
import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/l10n/l10n.dart';
import 'package:fluffychat/pages/chat/chat.dart';
import 'package:fluffychat/pages/chat/events/message_content.dart';
import 'package:fluffychat/pages/chat/events/reply_content.dart';
@ -11,7 +12,6 @@ import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dar
import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
import 'package:fluffychat/pangea/toolbar/enums/reading_assistance_mode_enum.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_selection_overlay.dart';
import 'package:fluffychat/pangea/toolbar/widgets/message_speech_to_text_card.dart';
import 'package:fluffychat/utils/date_time_extension.dart';
import 'package:fluffychat/utils/file_description.dart';
import 'package:fluffychat/widgets/matrix.dart';
@ -135,10 +135,93 @@ class OverlayMessage extends StatelessWidget {
event.numberEmotes <= 3);
final showTranslation = overlayController.showTranslation &&
overlayController.translationText != null;
overlayController.translation != null;
final showTranscription = pangeaMessageEvent?.isAudioMessage == true;
final showSpeechTranslation = overlayController.showSpeechTranslation &&
overlayController.speechTranslation != null;
final transcription = showTranscription
? Container(
width: messageWidth,
constraints: const BoxConstraints(
maxHeight: AppConfig.audioTranscriptionMaxHeight,
),
child: Padding(
padding: const EdgeInsets.all(12.0),
child: overlayController.transcriptionError != null
? Row(
mainAxisSize: MainAxisSize.min,
children: [
Icon(
Icons.error_outline,
color: Theme.of(context).colorScheme.error,
),
const SizedBox(width: 8),
Text(
L10n.of(context).oopsSomethingWentWrong,
style: AppConfig.messageTextStyle(
event,
textColor,
).copyWith(fontStyle: FontStyle.italic),
),
],
)
: overlayController.transcription != null
? SingleChildScrollView(
child: Text(
overlayController.transcription!.transcript.text,
style: AppConfig.messageTextStyle(
event,
textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
),
)
: Row(
mainAxisSize: MainAxisSize.min,
children: [
CircularProgressIndicator.adaptive(
backgroundColor: textColor,
),
],
),
),
)
: const SizedBox();
final translation = showTranslation || showSpeechTranslation
? Container(
width: messageWidth,
constraints: const BoxConstraints(
maxHeight: AppConfig.audioTranscriptionMaxHeight,
),
child: Padding(
padding: const EdgeInsets.fromLTRB(
12.0,
20.0,
12.0,
12.0,
),
child: SingleChildScrollView(
child: Text(
showTranslation
? overlayController.translation!
: overlayController.speechTranslation!,
style: AppConfig.messageTextStyle(
event,
textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
),
),
),
)
: const SizedBox();
final content = Container(
decoration: BoxDecoration(
borderRadius: BorderRadius.circular(
@ -156,6 +239,8 @@ class OverlayMessage extends StatelessWidget {
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.start,
children: <Widget>[
if (readingAssistanceMode == ReadingAssistanceMode.transitionMode)
transcription,
if (event.relationshipType == RelationshipTypes.reply)
FutureBuilder<Event?>(
future: event.getReplyEvent(
@ -254,6 +339,8 @@ class OverlayMessage extends StatelessWidget {
],
),
),
if (readingAssistanceMode == ReadingAssistanceMode.transitionMode)
translation,
],
),
),
@ -270,6 +357,8 @@ class OverlayMessage extends StatelessWidget {
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
if (readingAssistanceMode != ReadingAssistanceMode.transitionMode)
transcription,
sizeAnimation != null
? AnimatedBuilder(
animation: sizeAnimation!,
@ -282,37 +371,8 @@ class OverlayMessage extends StatelessWidget {
},
)
: content,
if (showTranscription || showTranslation)
Container(
width: messageWidth,
constraints: const BoxConstraints(
maxHeight: AppConfig.audioTranscriptionMaxHeight,
),
child: Padding(
padding: const EdgeInsets.fromLTRB(
12.0,
20.0,
12.0,
12.0,
),
child: SingleChildScrollView(
child: showTranscription
? MessageSpeechToTextCard(
messageEvent: pangeaMessageEvent!,
textColor: textColor,
)
: Text(
overlayController.translationText!,
style: AppConfig.messageTextStyle(
event,
textColor,
).copyWith(
fontStyle: FontStyle.italic,
),
),
),
),
),
if (readingAssistanceMode != ReadingAssistanceMode.transitionMode)
translation,
],
),
),

@ -236,7 +236,6 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
messageEvent:
widget.practiceCardController.widget.pangeaMessageEvent,
overlayController: widget.overlayController,
setIsPlayingAudio: widget.overlayController.setIsPlayingAudio,
onError: widget.onError,
),
ChoicesArray(
@ -247,8 +246,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
choices: choices(context),
isActive: true,
id: currentRecordModel?.hashCode.toString(),
enableAudio: !widget.overlayController.isPlayingAudio &&
practiceActivity.activityType.includeTTSOnClick,
enableAudio: practiceActivity.activityType.includeTTSOnClick,
langCode:
MatrixState.pangeaController.languageController.activeL2Code(),
getDisplayCopy: _getDisplayCopy,

@ -25,7 +25,8 @@ import 'package:fluffychat/widgets/matrix.dart';
enum SelectMode {
audio(Icons.volume_up),
translate(Icons.translate),
practice(Symbols.fitness_center);
practice(Symbols.fitness_center),
speechTranslation(Icons.translate);
final IconData icon;
const SelectMode(this.icon);
@ -39,6 +40,8 @@ enum SelectMode {
return l10n.translationTooltip;
case SelectMode.practice:
return l10n.practice;
case SelectMode.speechTranslation:
return l10n.speechToTextTooltip;
}
}
}
@ -61,6 +64,17 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
static const double iconWidth = 36.0;
static const double buttonSize = 40.0;
static List<SelectMode> get textModes => [
SelectMode.audio,
SelectMode.translate,
SelectMode.practice,
];
static List<SelectMode> get audioModes => [
SelectMode.speechTranslation,
// SelectMode.practice,
];
SelectMode? _selectedMode;
final AudioPlayer _audioPlayer = AudioPlayer();
@ -73,7 +87,12 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
StreamSubscription? _onAudioPositionChanged;
bool _isLoadingTranslation = false;
PangeaRepresentation? _repEvent;
String? _translationError;
bool _isLoadingSpeechTranslation = false;
String? _speechTranslationError;
Completer<String>? _transcriptionCompleter;
@override
void initState() {
@ -93,6 +112,10 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
);
}
});
if (messageEvent?.isAudioMessage == true) {
_fetchTranscription();
}
}
@override
@ -107,17 +130,20 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
widget.overlayController.pangeaMessageEvent;
String? get l1Code =>
MatrixState.pangeaController.languageController.activeL1Code();
MatrixState.pangeaController.languageController.userL1?.langCodeShort;
String? get l2Code =>
MatrixState.pangeaController.languageController.activeL2Code();
MatrixState.pangeaController.languageController.userL2?.langCodeShort;
void _clear() {
setState(() => _audioError = null);
widget.overlayController.updateSelectedSpan(null);
setState(() {
_audioError = null;
_translationError = null;
_speechTranslationError = null;
});
if (_selectedMode == SelectMode.translate) {
widget.overlayController.setShowTranslation(false, null);
}
widget.overlayController.updateSelectedSpan(null);
widget.overlayController.setShowTranslation(false);
widget.overlayController.setShowSpeechTranslation(false);
}
Future<void> _updateMode(SelectMode? mode) async {
@ -151,12 +177,13 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
}
if (_selectedMode == SelectMode.translate) {
await _loadTranslation();
if (_repEvent == null) return;
widget.overlayController.setShowTranslation(
true,
_repEvent!.text,
);
await _fetchTranslation();
widget.overlayController.setShowTranslation(true);
}
if (_selectedMode == SelectMode.speechTranslation) {
await _fetchSpeechTranslation();
widget.overlayController.setShowSpeechTranslation(true);
}
}
@ -244,81 +271,168 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
}
}
Future<void> _fetchRepresentation() async {
if (l1Code == null || messageEvent == null || _repEvent != null) {
Future<void> _fetchTranslation() async {
if (l1Code == null ||
messageEvent == null ||
widget.overlayController.translation != null) {
return;
}
_repEvent = messageEvent!.representationByLanguage(l1Code!)?.content;
if (_repEvent == null && mounted) {
_repEvent = await messageEvent?.representationByLanguageGlobal(
try {
if (mounted) setState(() => _isLoadingTranslation = true);
PangeaRepresentation? rep =
messageEvent!.representationByLanguage(l1Code!)?.content;
rep ??= await messageEvent?.representationByLanguageGlobal(
langCode: l1Code!,
);
widget.overlayController.setTranslation(rep!.text);
} catch (e, s) {
_translationError = e.toString();
ErrorHandler.logError(
e: e,
s: s,
m: 'Error fetching translation',
data: {
'l1Code': l1Code,
'messageEvent': messageEvent?.event.toJson(),
},
);
} finally {
if (mounted) setState(() => _isLoadingTranslation = false);
}
}
Future<void> _loadTranslation() async {
if (!mounted) return;
setState(() => _isLoadingTranslation = true);
Future<void> _fetchTranscription() async {
try {
await _fetchRepresentation();
if (_transcriptionCompleter != null) {
// If a transcription is already in progress, wait for it to complete
await _transcriptionCompleter!.future;
return;
}
_transcriptionCompleter = Completer<String>();
if (l1Code == null || messageEvent == null) {
_transcriptionCompleter?.completeError(
'Language code or message event is null',
);
return;
}
final resp = await messageEvent!.getSpeechToText(
l1Code!,
l2Code!,
);
widget.overlayController.setTranscription(resp!);
_transcriptionCompleter?.complete(resp.transcript.text);
} catch (err) {
widget.overlayController.setTranscriptionError(
err.toString(),
);
_transcriptionCompleter?.completeError(err);
ErrorHandler.logError(
e: err,
data: {},
);
}
}
if (mounted) {
setState(() => _isLoadingTranslation = false);
Future<void> _fetchSpeechTranslation() async {
if (messageEvent == null ||
l1Code == null ||
l2Code == null ||
widget.overlayController.speechTranslation != null) {
return;
}
}
Widget icon(SelectMode mode) {
if (mode == SelectMode.audio) {
if (_audioError != null) {
return Icon(
Icons.error_outline,
size: 20,
color: Theme.of(context).colorScheme.error,
);
try {
setState(() => _isLoadingSpeechTranslation = true);
if (widget.overlayController.transcription == null) {
await _fetchTranscription();
if (widget.overlayController.transcription == null) {
throw Exception('Transcription is null');
}
}
if (_isLoadingAudio) {
return const Center(
child: SizedBox(
height: 20.0,
width: 20.0,
child: CircularProgressIndicator.adaptive(),
),
);
} else {
return Icon(
_audioPlayer.playerState.playing == true
? Icons.pause_outlined
: Icons.volume_up,
size: 20,
color: mode == _selectedMode ? Colors.white : null,
);
final translation = await messageEvent!.sttTranslationByLanguageGlobal(
langCode: l1Code!,
l1Code: l1Code!,
l2Code: l2Code!,
);
if (translation == null) {
throw Exception('Translation is null');
}
widget.overlayController.setSpeechTranslation(translation.translation);
} catch (err, s) {
debugPrint("Error fetching speech translation: $err, $s");
_speechTranslationError = err.toString();
ErrorHandler.logError(
e: err,
data: {},
);
} finally {
if (mounted) setState(() => _isLoadingSpeechTranslation = false);
}
}
if (mode == SelectMode.translate) {
if (_isLoadingTranslation) {
return const Center(
child: SizedBox(
height: 20.0,
width: 20.0,
child: CircularProgressIndicator.adaptive(),
),
);
} else if (_repEvent != null) {
return Icon(
mode.icon,
size: 20,
color: mode == _selectedMode ? Colors.white : null,
);
}
bool get _isError {
switch (_selectedMode) {
case SelectMode.audio:
return _audioError != null;
case SelectMode.translate:
return _translationError != null;
case SelectMode.speechTranslation:
return _speechTranslationError != null;
default:
return false;
}
}
bool get _isLoading {
switch (_selectedMode) {
case SelectMode.audio:
return _isLoadingAudio;
case SelectMode.translate:
return _isLoadingTranslation;
case SelectMode.speechTranslation:
return _isLoadingSpeechTranslation;
default:
return false;
}
}
Widget icon(SelectMode mode) {
if (_isError && mode == _selectedMode) {
return Icon(
Icons.error_outline,
size: 20,
color: Theme.of(context).colorScheme.error,
);
}
if (_isLoading && mode == _selectedMode) {
return const Center(
child: SizedBox(
height: 20.0,
width: 20.0,
child: CircularProgressIndicator.adaptive(),
),
);
}
if (mode == SelectMode.audio) {
return Icon(
_audioPlayer.playerState.playing == true
? Icons.pause_outlined
: Icons.volume_up,
size: 20,
color: mode == _selectedMode ? Colors.white : null,
);
}
return Icon(
@ -330,6 +444,8 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
@override
Widget build(BuildContext context) {
final modes = messageEvent?.isAudioMessage == true ? audioModes : textModes;
return Container(
height: AppConfig.toolbarButtonsHeight,
alignment: Alignment.bottomCenter,
@ -338,7 +454,7 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
mainAxisSize: MainAxisSize.min,
spacing: 4.0,
children: [
for (final mode in SelectMode.values)
for (final mode in modes)
Tooltip(
message: mode.tooltip(context),
child: PressableButton(

Loading…
Cancel
Save