From 291dcfc192397c394d4a29433bc1b543170e22ff Mon Sep 17 00:00:00 2001 From: Kelrap Date: Mon, 12 Aug 2024 14:58:11 -0400 Subject: [PATCH 1/9] Toolbar can show downward --- lib/pangea/widgets/chat/message_toolbar.dart | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/pangea/widgets/chat/message_toolbar.dart b/lib/pangea/widgets/chat/message_toolbar.dart index 3c63c81fc..7c5682933 100644 --- a/lib/pangea/widgets/chat/message_toolbar.dart +++ b/lib/pangea/widgets/chat/message_toolbar.dart @@ -59,8 +59,12 @@ class ToolbarDisplayController { } void showToolbar(BuildContext context, {MessageMode? mode}) { - // Close keyboard, if open - FocusManager.instance.primaryFocus?.unfocus(); + // Don't show toolbar if keyboard open + if (controller.inputFocus.hasFocus) { + FocusManager.instance.primaryFocus?.unfocus(); + return; + } + bool toolbarUp = true; if (highlighted) return; if (controller.selectMode) { @@ -86,13 +90,12 @@ class ToolbarDisplayController { if (targetOffset.dy < 320) { final spaceBeneath = MediaQuery.of(context).size.height - (targetOffset.dy + transformTargetSize.height); - // If toolbar is open, opening toolbar beneath without scrolling can cause issues - // if (spaceBeneath >= 320) { - // toolbarUp = false; - // } + if (spaceBeneath >= 320) { + toolbarUp = false; + } // See if it's possible to scroll up to make space - if (controller.scrollController.offset - targetOffset.dy + 320 >= + else if (controller.scrollController.offset - targetOffset.dy + 320 >= controller.scrollController.position.minScrollExtent && controller.scrollController.offset - targetOffset.dy + 320 <= controller.scrollController.position.maxScrollExtent) { From 20ffe7ba37cc9f8398a42044d20c3c72b94fb49b Mon Sep 17 00:00:00 2001 From: Kelrap Date: Mon, 29 Jul 2024 14:49:12 -0400 Subject: [PATCH 2/9] Pass previous X messages to IGC --- .../controllers/choreographer.dart | 33 +++++++++++++++++++ .../controllers/igc_controller.dart | 1 + .../pangea_message_event.dart | 15 +++++++++ lib/pangea/repo/igc_repo.dart | 16 +++++++++ 4 files changed, 65 insertions(+) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index 1a11de0e3..eab917f4d 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -9,16 +9,19 @@ import 'package:fluffychat/pangea/controllers/pangea_controller.dart'; import 'package:fluffychat/pangea/controllers/subscription_controller.dart'; import 'package:fluffychat/pangea/enum/assistance_state_enum.dart'; import 'package:fluffychat/pangea/enum/edit_type.dart'; +import 'package:fluffychat/pangea/matrix_event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/models/it_step.dart'; import 'package:fluffychat/pangea/models/representation_content_model.dart'; import 'package:fluffychat/pangea/models/space_model.dart'; import 'package:fluffychat/pangea/models/tokens_event_content_model.dart'; +import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:fluffychat/pangea/utils/any_state_holder.dart'; import 'package:fluffychat/pangea/utils/error_handler.dart'; import 'package:fluffychat/pangea/utils/overlay.dart'; import 'package:fluffychat/pangea/widgets/igc/paywall_card.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:matrix/matrix.dart'; import 'package:sentry_flutter/sentry_flutter.dart'; import '../../../widgets/matrix.dart'; @@ -66,6 +69,36 @@ class Choreographer { clear(); } + List prevMessages() { + const int howFarBack = 5; + final List events = chatController.timeline?.events ?? []; + final List messages = []; + for (final Event event in events) { + if (event.messageType == MessageTypes.Text || + event.messageType == MessageTypes.Audio) { + final Map? content = + (event.messageType == MessageTypes.Text) + ? event.content + : (event as PangeaMessageEvent) + .getSpeechToTextLocalOnly(l1LangCode, l2LangCode) + ?.toJson(); + if (content != null) { + messages.add( + PreviousMessage( + event.content, + event.senderId, + event.originServerTs, + ), + ); + if (messages.length >= howFarBack) { + return messages; + } + } + } + } + return messages; + } + void send(BuildContext context) { if (isFetching) return; diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index ab0e48669..89941ccec 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -43,6 +43,7 @@ class IgcController { userL2: choreographer.l2LangCode!, enableIGC: choreographer.igcEnabled && !onlyTokensAndLanguageDetection, enableIT: choreographer.itEnabled && !onlyTokensAndLanguageDetection, + prevMessages: choreographer.prevMessages(), ); final IGCTextData igcTextDataResponse = await IgcRepo.getIGC( diff --git a/lib/pangea/matrix_event_wrappers/pangea_message_event.dart b/lib/pangea/matrix_event_wrappers/pangea_message_event.dart index f1c9e5082..edc4abf0e 100644 --- a/lib/pangea/matrix_event_wrappers/pangea_message_event.dart +++ b/lib/pangea/matrix_event_wrappers/pangea_message_event.dart @@ -269,6 +269,21 @@ class PangeaMessageEvent { null; }).toSet(); + SpeechToTextModel? getSpeechToTextLocalOnly( + String? l1Code, + String? l2Code, + ) { + if (l1Code == null || l2Code == null) { + return null; + } + return representations + .firstWhereOrNull( + (element) => element.content.speechToText != null, + ) + ?.content + .speechToText; + } + Future getSpeechToText( String l1Code, String l2Code, diff --git a/lib/pangea/repo/igc_repo.dart b/lib/pangea/repo/igc_repo.dart index e32f6cab7..ce50367cd 100644 --- a/lib/pangea/repo/igc_repo.dart +++ b/lib/pangea/repo/igc_repo.dart @@ -94,12 +94,27 @@ class IgcRepo { } } +/// Previous text/audio messages sent in chat +/// Contain message content, sender, and timestamp +class PreviousMessage { + Map content; + String sender; + DateTime timestamp; + + PreviousMessage( + this.content, + this.sender, + this.timestamp, + ); +} + class IGCRequestBody { String fullText; String userL1; String userL2; bool enableIT; bool enableIGC; + List prevMessages; IGCRequestBody({ required this.fullText, @@ -107,6 +122,7 @@ class IGCRequestBody { required this.userL2, required this.enableIGC, required this.enableIT, + required this.prevMessages, }); Map toJson() => { From 61d20f8b3732227c92b4584b088499634befc98c Mon Sep 17 00:00:00 2001 From: Kelrap Date: Tue, 30 Jul 2024 11:03:21 -0400 Subject: [PATCH 3/9] Add prevMessages to more places --- .../choreographer/controllers/span_data_controller.dart | 2 ++ lib/pangea/constants/model_keys.dart | 1 + lib/pangea/models/igc_text_data_model.dart | 4 ++++ lib/pangea/repo/igc_repo.dart | 2 ++ lib/pangea/repo/span_data_repo.dart | 6 ++++++ 5 files changed, 15 insertions(+) diff --git a/lib/pangea/choreographer/controllers/span_data_controller.dart b/lib/pangea/choreographer/controllers/span_data_controller.dart index 24470454c..910246abf 100644 --- a/lib/pangea/choreographer/controllers/span_data_controller.dart +++ b/lib/pangea/choreographer/controllers/span_data_controller.dart @@ -54,6 +54,7 @@ class SpanDataController { userL2: choreographer.l2LangCode!, enableIGC: choreographer.igcEnabled, enableIT: choreographer.itEnabled, + prevMessages: choreographer.prevMessages(), span: span, ); final int cacheKey = req.hashCode; @@ -71,6 +72,7 @@ class SpanDataController { userL2: choreographer.l2LangCode!, enableIGC: choreographer.igcEnabled, enableIT: choreographer.itEnabled, + prevMessages: choreographer.prevMessages(), span: span, ), ); diff --git a/lib/pangea/constants/model_keys.dart b/lib/pangea/constants/model_keys.dart index b42061446..d252a3c7c 100644 --- a/lib/pangea/constants/model_keys.dart +++ b/lib/pangea/constants/model_keys.dart @@ -69,6 +69,7 @@ class ModelKey { static const String permissions = "permissions"; static const String enableIGC = "enable_igc"; static const String enableIT = "enable_it"; + static const String prevMessages = "prev_messages"; static const String originalSent = "original_sent"; static const String originalWritten = "original_written"; diff --git a/lib/pangea/models/igc_text_data_model.dart b/lib/pangea/models/igc_text_data_model.dart index 5f32f92d1..588318011 100644 --- a/lib/pangea/models/igc_text_data_model.dart +++ b/lib/pangea/models/igc_text_data_model.dart @@ -26,6 +26,7 @@ class IGCTextData { bool enableIT; bool enableIGC; bool loading = false; + Map prevMessages; IGCTextData({ required this.detections, @@ -37,6 +38,7 @@ class IGCTextData { required this.userL2, required this.enableIT, required this.enableIGC, + required this.prevMessages, }); factory IGCTextData.fromJson(Map json) { @@ -76,6 +78,7 @@ class IGCTextData { userL2: json[ModelKey.userL2], enableIT: json["enable_it"], enableIGC: json["enable_igc"], + prevMessages: json["prev_messages"], ); } @@ -93,6 +96,7 @@ class IGCTextData { ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, + "prev_messages": prevMessages, }; /// if we haven't run IGC or IT or there are no matches, we use the highest validated detection diff --git a/lib/pangea/repo/igc_repo.dart b/lib/pangea/repo/igc_repo.dart index ce50367cd..0cb3e2e46 100644 --- a/lib/pangea/repo/igc_repo.dart +++ b/lib/pangea/repo/igc_repo.dart @@ -88,6 +88,7 @@ class IgcRepo { userL2: "en", enableIT: true, enableIGC: true, + prevMessages: {}, ); return igcTextData; @@ -131,5 +132,6 @@ class IGCRequestBody { ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, + "prev_messages": prevMessages, }; } diff --git a/lib/pangea/repo/span_data_repo.dart b/lib/pangea/repo/span_data_repo.dart index 7073581a1..df6b26c76 100644 --- a/lib/pangea/repo/span_data_repo.dart +++ b/lib/pangea/repo/span_data_repo.dart @@ -5,6 +5,7 @@ import 'package:fluffychat/pangea/config/environment.dart'; import 'package:fluffychat/pangea/enum/span_choice_type.dart'; import 'package:fluffychat/pangea/enum/span_data_type.dart'; import 'package:fluffychat/pangea/models/span_data.dart'; +import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:http/http.dart'; import '../constants/model_keys.dart'; @@ -47,6 +48,7 @@ class SpanDetailsRepoReqAndRes { String userL2; bool enableIT; bool enableIGC; + List prevMessages; SpanData span; SpanDetailsRepoReqAndRes({ @@ -54,6 +56,7 @@ class SpanDetailsRepoReqAndRes { required this.userL2, required this.enableIGC, required this.enableIT, + required this.prevMessages, required this.span, }); @@ -62,6 +65,7 @@ class SpanDetailsRepoReqAndRes { ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, + "prev_messages": prevMessages, 'span': span.toJson(), }; @@ -71,6 +75,7 @@ class SpanDetailsRepoReqAndRes { userL2: json['user_l2'] as String, enableIT: json['enable_it'] as bool, enableIGC: json['enable_igc'] as bool, + prevMessages: json['prev_messages'], span: SpanData.fromJson(json['span']), ); @@ -132,6 +137,7 @@ final mockRequest = SpanDetailsRepoReqAndRes( userL2: "en", enableIGC: true, enableIT: true, + prevMessages: [], span: spanDataRepomockSpan, ); From 55d75efa061e9f33aa1f89418f8bd758077fe380 Mon Sep 17 00:00:00 2001 From: Kelrap Date: Tue, 30 Jul 2024 11:05:51 -0400 Subject: [PATCH 4/9] Forgot to save earlier --- lib/pangea/models/igc_text_data_model.dart | 3 ++- lib/pangea/repo/igc_repo.dart | 2 +- lib/pangea/repo/span_data_repo.dart | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pangea/models/igc_text_data_model.dart b/lib/pangea/models/igc_text_data_model.dart index 588318011..eba1c1476 100644 --- a/lib/pangea/models/igc_text_data_model.dart +++ b/lib/pangea/models/igc_text_data_model.dart @@ -5,6 +5,7 @@ import 'package:fluffychat/pangea/controllers/language_detection_controller.dart import 'package:fluffychat/pangea/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/models/span_card_model.dart'; +import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:fluffychat/pangea/utils/error_handler.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -26,7 +27,7 @@ class IGCTextData { bool enableIT; bool enableIGC; bool loading = false; - Map prevMessages; + List prevMessages; IGCTextData({ required this.detections, diff --git a/lib/pangea/repo/igc_repo.dart b/lib/pangea/repo/igc_repo.dart index 0cb3e2e46..552895068 100644 --- a/lib/pangea/repo/igc_repo.dart +++ b/lib/pangea/repo/igc_repo.dart @@ -88,7 +88,7 @@ class IgcRepo { userL2: "en", enableIT: true, enableIGC: true, - prevMessages: {}, + prevMessages: [], ); return igcTextData; diff --git a/lib/pangea/repo/span_data_repo.dart b/lib/pangea/repo/span_data_repo.dart index df6b26c76..360ddd421 100644 --- a/lib/pangea/repo/span_data_repo.dart +++ b/lib/pangea/repo/span_data_repo.dart @@ -75,7 +75,7 @@ class SpanDetailsRepoReqAndRes { userL2: json['user_l2'] as String, enableIT: json['enable_it'] as bool, enableIGC: json['enable_igc'] as bool, - prevMessages: json['prev_messages'], + prevMessages: json['prev_messages'] as List, span: SpanData.fromJson(json['span']), ); @@ -90,6 +90,7 @@ class SpanDetailsRepoReqAndRes { if (other.userL2 != userL2) return false; if (other.enableIT != enableIT) return false; if (other.enableIGC != enableIGC) return false; + if (!other.prevMessages.equals(prevMessages)) return false; if (const ListEquality().equals( other.span.choices?.sorted((a, b) => b.value.compareTo(a.value)), span.choices?.sorted((a, b) => b.value.compareTo(a.value)), @@ -109,6 +110,7 @@ class SpanDetailsRepoReqAndRes { userL2.hashCode, enableIT.hashCode, enableIGC.hashCode, + prevMessages.hashCode, if (span.choices != null) Object.hashAll( span.choices! From 2ca6d102aa96b8369454a62027f7d0eff4d1bcc4 Mon Sep 17 00:00:00 2001 From: Kelrap Date: Wed, 31 Jul 2024 11:28:26 -0400 Subject: [PATCH 5/9] Add json methods to PreviousMessage --- .../controllers/choreographer.dart | 48 +++++++++++-------- lib/pangea/constants/model_keys.dart | 3 ++ lib/pangea/repo/igc_repo.dart | 30 +++++++++--- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index eab917f4d..8768ed4ff 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -71,28 +71,36 @@ class Choreographer { List prevMessages() { const int howFarBack = 5; - final List events = chatController.timeline?.events ?? []; + final List events = chatController.timeline?.events + .where( + (e) => + e.messageType == MessageTypes.Text || + e.messageType == MessageTypes.Audio, + ) + .toList() ?? + []; + // Sort from most recent to least + events.sort( + (a, b) => b.originServerTs.compareTo(a.originServerTs), + ); final List messages = []; for (final Event event in events) { - if (event.messageType == MessageTypes.Text || - event.messageType == MessageTypes.Audio) { - final Map? content = - (event.messageType == MessageTypes.Text) - ? event.content - : (event as PangeaMessageEvent) - .getSpeechToTextLocalOnly(l1LangCode, l2LangCode) - ?.toJson(); - if (content != null) { - messages.add( - PreviousMessage( - event.content, - event.senderId, - event.originServerTs, - ), - ); - if (messages.length >= howFarBack) { - return messages; - } + final Map? content = + (event.messageType == MessageTypes.Text) + ? event.content + : (event as PangeaMessageEvent) + .getSpeechToTextLocalOnly(l1LangCode, l2LangCode) + ?.toJson(); + if (content != null) { + messages.add( + PreviousMessage( + event.content, + event.senderId, + event.originServerTs, + ), + ); + if (messages.length >= howFarBack) { + return messages; } } } diff --git a/lib/pangea/constants/model_keys.dart b/lib/pangea/constants/model_keys.dart index d252a3c7c..195e919b4 100644 --- a/lib/pangea/constants/model_keys.dart +++ b/lib/pangea/constants/model_keys.dart @@ -70,6 +70,9 @@ class ModelKey { static const String enableIGC = "enable_igc"; static const String enableIT = "enable_it"; static const String prevMessages = "prev_messages"; + static const String prevContent = "prev_content"; + static const String prevSender = "prev_sender"; + static const String prevTimestamp = "prev_timestamp"; static const String originalSent = "original_sent"; static const String originalWritten = "original_written"; diff --git a/lib/pangea/repo/igc_repo.dart b/lib/pangea/repo/igc_repo.dart index 552895068..50dba9c2a 100644 --- a/lib/pangea/repo/igc_repo.dart +++ b/lib/pangea/repo/igc_repo.dart @@ -95,18 +95,33 @@ class IgcRepo { } } -/// Previous text/audio messages sent in chat +/// Previous text/audio message sent in chat /// Contain message content, sender, and timestamp class PreviousMessage { Map content; String sender; DateTime timestamp; - PreviousMessage( - this.content, - this.sender, - this.timestamp, - ); + PreviousMessage({ + required this.content, + required this.sender, + required this.timestamp, + }); + + factory PreviousMessage.fromJson(Map json) => + PreviousMessage( + content: jsonDecode(json[ModelKey.prevContent]) ?? {}, + sender: json[ModelKey.prevSender] ?? "", + timestamp: json[ModelKey.prevTimestamp] == null + ? DateTime.now() + : DateTime.parse(json[ModelKey.prevTimestamp]), + ); + + Map toJson() => { + ModelKey.prevContent: jsonEncode(content), + ModelKey.prevSender: sender, + ModelKey.prevTimestamp: timestamp.toIso8601String(), + }; } class IGCRequestBody { @@ -132,6 +147,7 @@ class IGCRequestBody { ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, - "prev_messages": prevMessages, + ModelKey.prevMessages: + jsonEncode(prevMessages.map((x) => x.toJson()).toList()), }; } From 4f924656e0fb49b37ce6da5c7f91bba45009e793 Mon Sep 17 00:00:00 2001 From: Kelrap Date: Mon, 5 Aug 2024 11:37:24 -0400 Subject: [PATCH 6/9] Fix minor error --- lib/pangea/choreographer/controllers/choreographer.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index 8768ed4ff..c462ff3b4 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -94,9 +94,9 @@ class Choreographer { if (content != null) { messages.add( PreviousMessage( - event.content, - event.senderId, - event.originServerTs, + content: event.content, + sender: event.senderId, + timestamp: event.originServerTs, ), ); if (messages.length >= howFarBack) { From 34d02bb95c13f5bf8956e8f8e0c55d3f3d08642d Mon Sep 17 00:00:00 2001 From: Kelrap Date: Tue, 13 Aug 2024 09:04:44 -0400 Subject: [PATCH 7/9] Remove prev message from span --- .../choreographer/controllers/span_data_controller.dart | 2 -- lib/pangea/repo/span_data_repo.dart | 8 -------- 2 files changed, 10 deletions(-) diff --git a/lib/pangea/choreographer/controllers/span_data_controller.dart b/lib/pangea/choreographer/controllers/span_data_controller.dart index 910246abf..24470454c 100644 --- a/lib/pangea/choreographer/controllers/span_data_controller.dart +++ b/lib/pangea/choreographer/controllers/span_data_controller.dart @@ -54,7 +54,6 @@ class SpanDataController { userL2: choreographer.l2LangCode!, enableIGC: choreographer.igcEnabled, enableIT: choreographer.itEnabled, - prevMessages: choreographer.prevMessages(), span: span, ); final int cacheKey = req.hashCode; @@ -72,7 +71,6 @@ class SpanDataController { userL2: choreographer.l2LangCode!, enableIGC: choreographer.igcEnabled, enableIT: choreographer.itEnabled, - prevMessages: choreographer.prevMessages(), span: span, ), ); diff --git a/lib/pangea/repo/span_data_repo.dart b/lib/pangea/repo/span_data_repo.dart index 360ddd421..7073581a1 100644 --- a/lib/pangea/repo/span_data_repo.dart +++ b/lib/pangea/repo/span_data_repo.dart @@ -5,7 +5,6 @@ import 'package:fluffychat/pangea/config/environment.dart'; import 'package:fluffychat/pangea/enum/span_choice_type.dart'; import 'package:fluffychat/pangea/enum/span_data_type.dart'; import 'package:fluffychat/pangea/models/span_data.dart'; -import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:http/http.dart'; import '../constants/model_keys.dart'; @@ -48,7 +47,6 @@ class SpanDetailsRepoReqAndRes { String userL2; bool enableIT; bool enableIGC; - List prevMessages; SpanData span; SpanDetailsRepoReqAndRes({ @@ -56,7 +54,6 @@ class SpanDetailsRepoReqAndRes { required this.userL2, required this.enableIGC, required this.enableIT, - required this.prevMessages, required this.span, }); @@ -65,7 +62,6 @@ class SpanDetailsRepoReqAndRes { ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, - "prev_messages": prevMessages, 'span': span.toJson(), }; @@ -75,7 +71,6 @@ class SpanDetailsRepoReqAndRes { userL2: json['user_l2'] as String, enableIT: json['enable_it'] as bool, enableIGC: json['enable_igc'] as bool, - prevMessages: json['prev_messages'] as List, span: SpanData.fromJson(json['span']), ); @@ -90,7 +85,6 @@ class SpanDetailsRepoReqAndRes { if (other.userL2 != userL2) return false; if (other.enableIT != enableIT) return false; if (other.enableIGC != enableIGC) return false; - if (!other.prevMessages.equals(prevMessages)) return false; if (const ListEquality().equals( other.span.choices?.sorted((a, b) => b.value.compareTo(a.value)), span.choices?.sorted((a, b) => b.value.compareTo(a.value)), @@ -110,7 +104,6 @@ class SpanDetailsRepoReqAndRes { userL2.hashCode, enableIT.hashCode, enableIGC.hashCode, - prevMessages.hashCode, if (span.choices != null) Object.hashAll( span.choices! @@ -139,7 +132,6 @@ final mockRequest = SpanDetailsRepoReqAndRes( userL2: "en", enableIGC: true, enableIT: true, - prevMessages: [], span: spanDataRepomockSpan, ); From 54d63b1a4f133c3fb5acf7bd52e47d82774456ad Mon Sep 17 00:00:00 2001 From: Kelrap Date: Tue, 13 Aug 2024 10:39:30 -0400 Subject: [PATCH 8/9] Fix audio errors --- .../controllers/choreographer.dart | 58 +++++++++++-------- lib/pangea/repo/igc_repo.dart | 6 +- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index c462ff3b4..bff53890a 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -19,6 +19,7 @@ import 'package:fluffychat/pangea/utils/any_state_holder.dart'; import 'package:fluffychat/pangea/utils/error_handler.dart'; import 'package:fluffychat/pangea/utils/overlay.dart'; import 'package:fluffychat/pangea/widgets/igc/paywall_card.dart'; +import 'package:fluffychat/utils/matrix_sdk_extensions/filtered_timeline_extension.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:matrix/matrix.dart'; @@ -71,38 +72,47 @@ class Choreographer { List prevMessages() { const int howFarBack = 5; - final List events = chatController.timeline?.events + final List eventList = chatController.timeline?.events .where( (e) => - e.messageType == MessageTypes.Text || - e.messageType == MessageTypes.Audio, + e.isVisibleInGui && + (e.messageType == MessageTypes.Text || + e.messageType == MessageTypes.Audio) && + e.status.isSent && + e.type.equals('m.room.message'), ) .toList() ?? []; - // Sort from most recent to least - events.sort( - (a, b) => b.originServerTs.compareTo(a.originServerTs), - ); final List messages = []; - for (final Event event in events) { - final Map? content = - (event.messageType == MessageTypes.Text) - ? event.content - : (event as PangeaMessageEvent) - .getSpeechToTextLocalOnly(l1LangCode, l2LangCode) - ?.toJson(); - if (content != null) { - messages.add( - PreviousMessage( - content: event.content, - sender: event.senderId, - timestamp: event.originServerTs, - ), - ); - if (messages.length >= howFarBack) { - return messages; + try { + for (final Event event in eventList) { + final String? content = (event.messageType == MessageTypes.Text) + ? event.content.toString() + : PangeaMessageEvent( + event: event, + // eventList will be empty if the timeline is null + timeline: chatController.timeline!, + ownMessage: event.senderId == + pangeaController.matrixState.client.userID, + ) + .getSpeechToTextLocalOnly(l1LangCode, l2LangCode) + ?.transcript + .text; + if (content != null) { + messages.add( + PreviousMessage( + content: content, + sender: event.senderId, + timestamp: event.originServerTs, + ), + ); + if (messages.length >= howFarBack) { + return messages; + } } } + } catch (err, stack) { + ErrorHandler.logError(e: err, s: stack); } return messages; } diff --git a/lib/pangea/repo/igc_repo.dart b/lib/pangea/repo/igc_repo.dart index 50dba9c2a..861932812 100644 --- a/lib/pangea/repo/igc_repo.dart +++ b/lib/pangea/repo/igc_repo.dart @@ -98,7 +98,7 @@ class IgcRepo { /// Previous text/audio message sent in chat /// Contain message content, sender, and timestamp class PreviousMessage { - Map content; + String content; String sender; DateTime timestamp; @@ -110,7 +110,7 @@ class PreviousMessage { factory PreviousMessage.fromJson(Map json) => PreviousMessage( - content: jsonDecode(json[ModelKey.prevContent]) ?? {}, + content: json[ModelKey.prevContent] ?? "", sender: json[ModelKey.prevSender] ?? "", timestamp: json[ModelKey.prevTimestamp] == null ? DateTime.now() @@ -118,7 +118,7 @@ class PreviousMessage { ); Map toJson() => { - ModelKey.prevContent: jsonEncode(content), + ModelKey.prevContent: content, ModelKey.prevSender: sender, ModelKey.prevTimestamp: timestamp.toIso8601String(), }; From b5abef7d1aa37f5eb4c956adb8f2d71eae32e2e2 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 20 Aug 2024 15:13:46 -0400 Subject: [PATCH 9/9] moved previous message function to igc controller --- .../controllers/choreographer.dart | 51 ------------------- .../controllers/igc_controller.dart | 47 ++++++++++++++++- .../pangea_message_event.dart | 2 +- lib/pangea/models/igc_text_data_model.dart | 5 -- lib/pangea/repo/igc_repo.dart | 1 - 5 files changed, 47 insertions(+), 59 deletions(-) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index bff53890a..1a11de0e3 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -9,20 +9,16 @@ import 'package:fluffychat/pangea/controllers/pangea_controller.dart'; import 'package:fluffychat/pangea/controllers/subscription_controller.dart'; import 'package:fluffychat/pangea/enum/assistance_state_enum.dart'; import 'package:fluffychat/pangea/enum/edit_type.dart'; -import 'package:fluffychat/pangea/matrix_event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/models/it_step.dart'; import 'package:fluffychat/pangea/models/representation_content_model.dart'; import 'package:fluffychat/pangea/models/space_model.dart'; import 'package:fluffychat/pangea/models/tokens_event_content_model.dart'; -import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:fluffychat/pangea/utils/any_state_holder.dart'; import 'package:fluffychat/pangea/utils/error_handler.dart'; import 'package:fluffychat/pangea/utils/overlay.dart'; import 'package:fluffychat/pangea/widgets/igc/paywall_card.dart'; -import 'package:fluffychat/utils/matrix_sdk_extensions/filtered_timeline_extension.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; -import 'package:matrix/matrix.dart'; import 'package:sentry_flutter/sentry_flutter.dart'; import '../../../widgets/matrix.dart'; @@ -70,53 +66,6 @@ class Choreographer { clear(); } - List prevMessages() { - const int howFarBack = 5; - final List eventList = chatController.timeline?.events - .where( - (e) => - e.isVisibleInGui && - (e.messageType == MessageTypes.Text || - e.messageType == MessageTypes.Audio) && - e.status.isSent && - e.type.equals('m.room.message'), - ) - .toList() ?? - []; - final List messages = []; - try { - for (final Event event in eventList) { - final String? content = (event.messageType == MessageTypes.Text) - ? event.content.toString() - : PangeaMessageEvent( - event: event, - // eventList will be empty if the timeline is null - timeline: chatController.timeline!, - ownMessage: event.senderId == - pangeaController.matrixState.client.userID, - ) - .getSpeechToTextLocalOnly(l1LangCode, l2LangCode) - ?.transcript - .text; - if (content != null) { - messages.add( - PreviousMessage( - content: content, - sender: event.senderId, - timestamp: event.originServerTs, - ), - ); - if (messages.length >= howFarBack) { - return messages; - } - } - } - } catch (err, stack) { - ErrorHandler.logError(e: err, s: stack); - } - return messages; - } - void send(BuildContext context) { if (isFetching) return; diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index 89941ccec..237697b40 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -4,12 +4,14 @@ import 'dart:developer'; import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/controllers/error_service.dart'; import 'package:fluffychat/pangea/choreographer/controllers/span_data_controller.dart'; +import 'package:fluffychat/pangea/matrix_event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/models/igc_text_data_model.dart'; import 'package:fluffychat/pangea/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:fluffychat/pangea/widgets/igc/span_card.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:matrix/matrix.dart'; import '../../models/span_card_model.dart'; import '../../utils/error_handler.dart'; @@ -43,7 +45,7 @@ class IgcController { userL2: choreographer.l2LangCode!, enableIGC: choreographer.igcEnabled && !onlyTokensAndLanguageDetection, enableIT: choreographer.itEnabled && !onlyTokensAndLanguageDetection, - prevMessages: choreographer.prevMessages(), + prevMessages: prevMessages(), ); final IGCTextData igcTextDataResponse = await IgcRepo.getIGC( @@ -126,6 +128,49 @@ class IgcController { ); } + /// Get the content of previous text and audio messages in chat. + /// Passed to IGC request to add context. + List prevMessages({int numMessages = 5}) { + final List events = choreographer.chatController.visibleEvents + .where( + (e) => + e.type == EventTypes.Message && + (e.messageType == MessageTypes.Text || + e.messageType == MessageTypes.Audio), + ) + .toList(); + + final List messages = []; + for (final Event event in events) { + final String? content = event.messageType == MessageTypes.Text + ? event.content.toString() + : PangeaMessageEvent( + event: event, + timeline: choreographer.chatController.timeline!, + ownMessage: event.senderId == + choreographer.pangeaController.matrixState.client.userID, + ) + .getSpeechToTextLocal( + choreographer.l1LangCode, + choreographer.l2LangCode, + ) + ?.transcript + .text; + if (content == null) continue; + messages.add( + PreviousMessage( + content: content, + sender: event.senderId, + timestamp: event.originServerTs, + ), + ); + if (messages.length >= numMessages) { + return messages; + } + } + return messages; + } + bool get hasRelevantIGCTextData { if (igcTextData == null) return false; diff --git a/lib/pangea/matrix_event_wrappers/pangea_message_event.dart b/lib/pangea/matrix_event_wrappers/pangea_message_event.dart index edc4abf0e..cd4ce5e76 100644 --- a/lib/pangea/matrix_event_wrappers/pangea_message_event.dart +++ b/lib/pangea/matrix_event_wrappers/pangea_message_event.dart @@ -269,7 +269,7 @@ class PangeaMessageEvent { null; }).toSet(); - SpeechToTextModel? getSpeechToTextLocalOnly( + SpeechToTextModel? getSpeechToTextLocal( String? l1Code, String? l2Code, ) { diff --git a/lib/pangea/models/igc_text_data_model.dart b/lib/pangea/models/igc_text_data_model.dart index eba1c1476..5f32f92d1 100644 --- a/lib/pangea/models/igc_text_data_model.dart +++ b/lib/pangea/models/igc_text_data_model.dart @@ -5,7 +5,6 @@ import 'package:fluffychat/pangea/controllers/language_detection_controller.dart import 'package:fluffychat/pangea/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/models/span_card_model.dart'; -import 'package:fluffychat/pangea/repo/igc_repo.dart'; import 'package:fluffychat/pangea/utils/error_handler.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -27,7 +26,6 @@ class IGCTextData { bool enableIT; bool enableIGC; bool loading = false; - List prevMessages; IGCTextData({ required this.detections, @@ -39,7 +37,6 @@ class IGCTextData { required this.userL2, required this.enableIT, required this.enableIGC, - required this.prevMessages, }); factory IGCTextData.fromJson(Map json) { @@ -79,7 +76,6 @@ class IGCTextData { userL2: json[ModelKey.userL2], enableIT: json["enable_it"], enableIGC: json["enable_igc"], - prevMessages: json["prev_messages"], ); } @@ -97,7 +93,6 @@ class IGCTextData { ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, - "prev_messages": prevMessages, }; /// if we haven't run IGC or IT or there are no matches, we use the highest validated detection diff --git a/lib/pangea/repo/igc_repo.dart b/lib/pangea/repo/igc_repo.dart index 861932812..33abb4deb 100644 --- a/lib/pangea/repo/igc_repo.dart +++ b/lib/pangea/repo/igc_repo.dart @@ -88,7 +88,6 @@ class IgcRepo { userL2: "en", enableIT: true, enableIGC: true, - prevMessages: [], ); return igcTextData;