import 'dart:developer'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:collection/collection.dart'; import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart'; import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart'; import 'package:fluffychat/pangea/choreographer/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/choreographer/repo/language_detection_repo.dart'; import 'package:fluffychat/pangea/common/constants/model_keys.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart'; import 'package:fluffychat/widgets/matrix.dart'; // import 'package:language_tool/language_tool.dart'; class IGCTextData { LanguageDetectionResponse detections; String originalInput; String? fullTextCorrection; List tokens; List matches; String userL1; String userL2; bool enableIT; bool enableIGC; bool loading = false; IGCTextData({ required this.detections, required this.originalInput, required this.fullTextCorrection, required this.tokens, required this.matches, required this.userL1, required this.userL2, required this.enableIT, required this.enableIGC, }); factory IGCTextData.fromJson(Map json) { // changing this to allow for use of the LanguageDetectionResponse methods // TODO - change API after we're sure all clients are updated. not urgent. final LanguageDetectionResponse detections = json[_detectionsKey] is Iterable ? LanguageDetectionResponse.fromJson({ "detections": json[_detectionsKey], "full_text": json["original_input"], }) : LanguageDetectionResponse.fromJson( json[_detectionsKey] as Map, ); return IGCTextData( tokens: (json[_tokensKey] as Iterable) .map( (e) => PangeaToken.fromJson(e as Map), ) .toList() .cast(), matches: json[_matchesKey] != null ? (json[_matchesKey] as Iterable) .map( (e) { return PangeaMatch.fromJson(e as Map); }, ) .toList() .cast() : [], detections: detections, originalInput: json["original_input"], fullTextCorrection: json["full_text_correction"], userL1: json[ModelKey.userL1], userL2: json[ModelKey.userL2], enableIT: json["enable_it"], enableIGC: json["enable_igc"], ); } factory IGCTextData.fromRepresentationEvent( RepresentationEvent event, String userL1, String userL2, ) { final PangeaRepresentation content = event.content; final List tokens = event.tokens ?? []; final List matches = event.choreo?.choreoSteps .map((step) => step.acceptedOrIgnoredMatch) .whereType() .toList() ?? []; String originalInput = content.text; if (matches.isNotEmpty) { originalInput = matches.first.match.fullText; } final defaultDetections = LanguageDetectionResponse( detections: [ LanguageDetection(langCode: content.langCode, confidence: 1), ], fullText: content.text, ); LanguageDetectionResponse detections = defaultDetections; if (event.detections != null) { try { detections = LanguageDetectionResponse.fromJson({ "detections": event.detections, "full_text": content.text, }); } catch (e, s) { ErrorHandler.logError( e: e, s: s, m: "Error parsing detections in IGCTextData.fromRepresentationEvent", data: { "detections": event.detections, "full_text": content.text, }, ); } } return IGCTextData( detections: detections, originalInput: originalInput, fullTextCorrection: content.text, tokens: tokens, matches: matches, userL1: userL1, userL2: userL2, enableIT: true, enableIGC: true, ); } static const String _tokensKey = "tokens"; static const String _matchesKey = "matches"; static const String _detectionsKey = "detections"; Map toJson() => { _detectionsKey: detections.toJson(), "original_input": originalInput, "full_text_correction": fullTextCorrection, _tokensKey: tokens.map((e) => e.toJson()).toList(), _matchesKey: matches.map((e) => e.toJson()).toList(), ModelKey.userL1: userL1, ModelKey.userL2: userL2, "enable_it": enableIT, "enable_igc": enableIGC, }; /// if we haven't run IGC or IT or there are no matches, we use the highest validated detection /// from [LanguageDetectionResponse.highestValidatedDetection] /// if we have run igc/it and there are no matches, we can relax the threshold /// and use the highest confidence detection String get detectedLanguage { return detections.detections.firstOrNull?.langCode ?? LanguageKeys.unknownLanguage; } // reconstruct fullText based on accepted match //update offsets in existing matches to reflect the change //if existing matches overlap with the accepted one, remove them?? void acceptReplacement( int matchIndex, int choiceIndex, ) async { //should be already added to choreoRecord //TODO - that should be done in the same function to avoid error potential final PangeaMatch pangeaMatch = matches[matchIndex]; if (pangeaMatch.match.choices == null) { debugger(when: kDebugMode); ErrorHandler.logError( m: "pangeaMatch.match.choices is null in acceptReplacement", data: { "match": pangeaMatch.match.toJson(), }, ); return; } final SpanChoice replacement = pangeaMatch.match.choices![choiceIndex]; final newStart = originalInput.characters.take(pangeaMatch.match.offset); final newEnd = originalInput.characters .skip(pangeaMatch.match.offset + pangeaMatch.match.length); final fullText = newStart + replacement.value.characters + newEnd; originalInput = fullText.toString(); int startIndex; int endIndex; // replace the tokens that are part of the match // with the tokens in the replacement // start is inclusive try { startIndex = tokenIndexByOffset(pangeaMatch.match.offset); // end is exclusive, hence the +1 // use pangeaMatch.matchContent.trim().length instead of pangeaMatch.match.length since pangeaMatch.match.length may include leading/trailing spaces endIndex = tokenIndexByOffset( pangeaMatch.match.offset + pangeaMatch.matchContent.trim().length, ) + 1; } catch (err, s) { matches.removeAt(matchIndex); for (final match in matches) { match.match.fullText = originalInput; if (match.match.offset > pangeaMatch.match.offset) { match.match.offset += replacement.value.length - pangeaMatch.match.length; } } ErrorHandler.logError( e: err, s: s, data: { "cursorOffset": pangeaMatch.match.offset, "match": pangeaMatch.match.toJson(), "tokens": tokens.map((e) => e.toJson()).toString(), }, ); return; } // for all tokens after the replacement, update their offsets for (int i = endIndex; i < tokens.length; i++) { tokens[i].text.offset += replacement.value.length - pangeaMatch.match.length; } // clone the list for debugging purposes final List newTokens = List.from(tokens); // replace the tokens in the list newTokens.replaceRange(startIndex, endIndex, replacement.tokens); final String newFullText = PangeaToken.reconstructText(newTokens); if (newFullText.trim() != originalInput.trim() && kDebugMode) { PangeaToken.reconstructText(newTokens, debugWalkThrough: true); ErrorHandler.logError( m: "reconstructed text not working", s: StackTrace.current, data: { "originalInput": originalInput, "newFullText": newFullText, "match": pangeaMatch.match.toJson(), }, ); } tokens = newTokens; //update offsets in existing matches to reflect the change //Question - remove matches that overlap with the accepted one? // see case of "quiero ver un fix" matches.removeAt(matchIndex); for (final match in matches) { match.match.fullText = originalInput; if (match.match.offset > pangeaMatch.match.offset) { match.match.offset += replacement.value.length - pangeaMatch.match.length; } } } void removeMatchByOffset(int offset) { final int index = getTopMatchIndexForOffset(offset); if (index != -1) { matches.removeAt(index); } } int tokenIndexByOffset(int cursorOffset) { final tokenIndex = tokens.indexWhere( (token) => token.start <= cursorOffset && cursorOffset <= token.end, ); if (tokenIndex < 0) { throw "No token found for cursor offset"; } return tokenIndex; } List matchIndicesByOffset(int offset) { final List matchesForOffset = []; for (final (index, match) in matches.indexed) { if (match.isOffsetInMatchSpan(offset)) { matchesForOffset.add(index); } } return matchesForOffset; } int getTopMatchIndexForOffset(int offset) { final List matchesForToken = matchIndicesByOffset(offset); final int matchIndex = matchesForToken.indexWhere((matchIndex) { final match = matches[matchIndex]; return (enableIT && (match.isITStart || match.isl1SpanMatch)) || (enableIGC && match.isGrammarMatch); }); if (matchIndex == -1) return -1; return matchesForToken[matchIndex]; } PangeaMatch? getTopMatchForToken(PangeaToken token) { final int topMatchIndex = getTopMatchIndexForOffset(token.text.offset); if (topMatchIndex == -1) return null; return matches[topMatchIndex]; } int getAfterTokenSpacingByIndex(int tokenIndex) { final int endOfToken = tokens[tokenIndex].end; if (tokenIndex + 1 < tokens.length) { final spaceBetween = tokens[tokenIndex + 1].text.offset - endOfToken; if (spaceBetween < 0) { ErrorHandler.logError( m: "weird token lengths for ${tokens[tokenIndex].text.content} and ${tokens[tokenIndex + 1].text.content}", data: { "fullText": originalInput, "tokens": tokens.map((e) => e.toJson()).toString(), }, ); return 0; } return spaceBetween; } else { return originalInput.length - endOfToken; } } static TextStyle underlineStyle(Color color) => TextStyle( decoration: TextDecoration.underline, decorationColor: color, decorationThickness: 5, ); TextSpan getSpanItem({ required int start, required int end, TextStyle? style, }) { return TextSpan( text: originalInput.characters.getRange(start, end).toString(), style: style, ); } int? get _openMatchIndex { final RegExp pattern = RegExp(r'span_card_overlay_\d+'); final String? matchingKeys = MatrixState.pAnyState.getMatchingOverlayKeys(pattern).firstOrNull; if (matchingKeys == null) return null; final int? index = int.tryParse(matchingKeys.split("_").last); if (index == null || matches.length <= index || matches[index].status != PangeaMatchStatus.open) { return null; } return index; } //PTODO - handle multitoken spans /// Returns a list of [TextSpan]s used to display the text in the input field /// with the appropriate styling for each error match. List constructTokenSpan({ ChoreoRecordStep? choreoStep, TextStyle? defaultStyle, }) { final stepMatch = choreoStep?.acceptedOrIgnoredMatch; final List textSpanMatches = List.from(matches); if (stepMatch != null && stepMatch.status == PangeaMatchStatus.automatic) { textSpanMatches.add(stepMatch); } final List items = []; if (loading) { return [ TextSpan( text: originalInput, style: defaultStyle, ), ]; } textSpanMatches.sort((a, b) => a.match.offset.compareTo(b.match.offset)); final List> matchRanges = textSpanMatches .map( (match) => [ match.match.offset, match.match.length + match.match.offset, ], ) .toList(); // create a pointer to the current index in the original input // and iterate until the pointer has reached the end of the input int currentIndex = 0; while (currentIndex < originalInput.characters.length) { // check if the pointer is at a match, and if so, get the index of the match final int matchIndex = matchRanges.indexWhere( (range) => currentIndex >= range[0] && currentIndex < range[1], ); final bool inMatch = matchIndex != -1; if (inMatch) { // if the pointer is in a match, then add that match to items // and then move the pointer to the end of the match range final PangeaMatch match = textSpanMatches[matchIndex]; items.add( getSpanItem( start: match.match.offset, end: match.match.offset + match.match.length, style: match.textStyle( matchIndex, _openMatchIndex, defaultStyle, ), ), ); currentIndex = match.match.offset + match.match.length; } else { // otherwise, if the pointer is not at a match, then add all the text // until the next match (or, if there is not next match, the end of the // text) to items and move the pointer to the start of the next match final int nextIndex = matchRanges .firstWhereOrNull( (range) => range[0] > currentIndex, ) ?.first ?? originalInput.characters.length; items.add( getSpanItem( start: currentIndex, end: nextIndex, style: defaultStyle, ), ); currentIndex = nextIndex; } } return items; } List matchTokens(int matchIndex) { if (matchIndex >= matches.length) { return []; } final PangeaMatch match = matches[matchIndex]; final List tokensForMatch = []; for (final token in tokens) { if (match.isOffsetInMatchSpan(token.text.offset)) { tokensForMatch.add(token); } } return tokensForMatch; } }