feat: leave analytics rooms after extracting data, use generator function to batch rooms in download (#1478)

lib/pangea/analytics/extensions/room_analytics_extension.dart

@@ -1,100 +1,176 @@
 part of "../../extensions/pangea_room_extension.dart";
 
 extension AnalyticsRoomExtension on Room {
-  Future<List<SpaceRoomsChunk>> _getFullSpaceHierarchy() async {
+  /// Get next n analytics rooms via the space hierarchy
+  ///     If joined
+  ///       If not in target language
+  ///           If not created by user, leave
+  ///       Else, add to list
+  ///     Else
+  ///       If room name does not match L2, skip
+  ///       Join and wait for room in sync.
+  ///     Repeat the same procedure as above.
+  ///
+  /// If not n analytics rooms in list, and nextBatch != null, repeat the above
+  /// procedure with nextBatch until n analytics rooms are found or nextBatch == null
+  ///
+  /// Yield this list of rooms.
+  /// Once analytics have been retrieved, leave analytics rooms not created by self.
+  Stream<List<Room>> getNextAnalyticsRoomBatch(String userL2) async* {
     final List<SpaceRoomsChunk> rooms = [];
     String? nextBatch;
+    int spaceHierarchyCalls = 0;
+    int callsToServer = 0;
+
+    while (spaceHierarchyCalls <= 5 &&
+        (nextBatch != null || spaceHierarchyCalls == 0)) {
+      spaceHierarchyCalls++;
+      final resp = await _getNextBatch(nextBatch);
+      callsToServer++;
+      if (resp == null) return;
 
-    try {
-      final resp = await client.getSpaceHierarchy(
-        id,
-        limit: 100,
-        maxDepth: 1,
-      );
       rooms.addAll(resp.rooms);
       nextBatch = resp.nextBatch;
+
+      final List<Room> roomsBatch = [];
+      while (rooms.isNotEmpty) {
+        // prevent rate-limiting
+        if (callsToServer >= 5) {
+          callsToServer = 0;
+          await Future.delayed(const Duration(milliseconds: 7500));
+        }
+
+        final nextRoomChunk = rooms.removeAt(0);
+        if (nextRoomChunk.roomType != PangeaRoomTypes.analytics) {
+          continue;
+        }
+
+        final matchingRoom = client.rooms.firstWhereOrNull(
+          (r) => r.id == nextRoomChunk.roomId,
+        );
+
+        final (analyticsRoom, calls) = matchingRoom != null
+            ? await _handleJoinedAnalyticsRoom(matchingRoom, userL2)
+            : await _handleUnjoinedAnalyticsRoom(nextRoomChunk, userL2);
+
+        callsToServer += calls;
+        if (analyticsRoom == null) continue;
+        roomsBatch.add(analyticsRoom);
+
+        if (roomsBatch.length >= 5) {
+          final roomsBatchCopy = List<Room>.from(roomsBatch);
+          roomsBatch.clear();
+          yield roomsBatchCopy;
+        }
+      }
+
+      yield roomsBatch;
+    }
+  }
+
+  /// Return analytics room, given unjoined member of space hierarchy,
+  /// if should get analytics for that room, and number of call made
+  /// to the server to help prevent rate-limiting
+  Future<(Room?, int)> _handleUnjoinedAnalyticsRoom(
+    SpaceRoomsChunk chunk,
+    String l2,
+  ) async {
+    int callsToServer = 0;
+    final nameParts = chunk.name?.split(" ");
+    if (nameParts != null && nameParts.length >= 2) {
+      final roomLangCode = nameParts[1];
+      if (roomLangCode != l2) return (null, callsToServer);
+    }
+
+    Room? analyticsRoom = await _joinAnalyticsRoomChunk(chunk);
+    callsToServer++;
+
+    if (analyticsRoom == null) return (null, callsToServer);
+    final (room, calls) = await _handleJoinedAnalyticsRoom(analyticsRoom, l2);
+    analyticsRoom = room;
+    callsToServer += calls;
+
+    return (analyticsRoom, callsToServer);
+  }
+
+  /// Return analytics room if should add to returned list
+  /// and the number of calls made to the server (used to prevent rate-limiting)
+  Future<(Room?, int)> _handleJoinedAnalyticsRoom(
+    Room analyticsRoom,
+    String l2,
+  ) async {
+    if (client.userID == null) return (null, 0);
+    if (analyticsRoom.madeForLang != l2) {
+      await _leaveNonTargetAnalyticsRoom(analyticsRoom, l2);
+      return (null, 1);
+    }
+    return (analyticsRoom, 0);
+  }
+
+  Future<Room?> _joinAnalyticsRoomChunk(
+    SpaceRoomsChunk chunk,
+  ) async {
+    final matchingRoom = client.rooms.firstWhereOrNull(
+      (r) => r.id == chunk.roomId,
+    );
+    if (matchingRoom != null) return matchingRoom;
+
+    try {
+      final syncFuture = client.waitForRoomInSync(chunk.roomId, join: true);
+      await client.joinRoom(chunk.roomId);
+      await syncFuture;
+      return client.getRoomById(chunk.roomId);
     } catch (e, s) {
       ErrorHandler.logError(
         e: e,
         s: s,
         data: {
-          "spaceID": id,
+          "roomID": chunk.roomId,
         },
       );
-      return rooms;
+      return null;
     }
+  }
 
-    int tries = 0;
+  Future<void> _leaveNonTargetAnalyticsRoom(Room room, String userL2) async {
-    while (nextBatch != null && tries <= 5) {
+    if (client.userID == null ||
-      GetSpaceHierarchyResponse nextResp;
+        room.isMadeByUser(client.userID!) ||
-      try {
+        room.madeForLang == userL2) {
-        nextResp = await client.getSpaceHierarchy(
+      return;
-          id,
-          from: nextBatch,
-          limit: 100,
-          maxDepth: 1,
-        );
-        rooms.addAll(nextResp.rooms);
-      } catch (e, s) {
-        ErrorHandler.logError(
-          e: e,
-          s: s,
-          data: {
-            "spaceID": id,
-          },
-        );
-        break;
-      }
-
-      nextBatch = nextResp.nextBatch;
-      tries++;
     }
 
-    return rooms;
+    try {
+      await room.leave();
+    } catch (e, s) {
+      ErrorHandler.logError(
+        e: e,
+        s: s,
+        data: {
+          "roomID": room.id,
+        },
+      );
+    }
   }
 
-  Future<void> _joinAnalyticsRooms() async {
+  Future<GetSpaceHierarchyResponse?> _getNextBatch(String? nextBatch) async {
-    final List<SpaceRoomsChunk> rooms = await _getFullSpaceHierarchy();
+    try {
-
+      final resp = await client.getSpaceHierarchy(
-    final unjoinedAnalyticsRooms = rooms.where(
+        id,
-      (room) {
+        from: nextBatch,
-        if (room.roomType != PangeaRoomTypes.analytics) return false;
+        limit: 100,
-        final matchingRoom = client.rooms.firstWhereOrNull(
+        maxDepth: 1,
-          (r) => r.id == room.roomId,
+      );
-        );
+      return resp;
-        return matchingRoom == null ||
+    } catch (e, s) {
-            matchingRoom.membership != Membership.join;
+      ErrorHandler.logError(
-      },
+        e: e,
-    ).toList();
+        s: s,
-
+        data: {
-    const batchSize = 5;
+          "spaceID": id,
-    int batchNum = 0;
+          "nextBatch": nextBatch,
-    while (batchSize * batchNum < unjoinedAnalyticsRooms.length) {
+        },
-      final batch =
+      );
-          unjoinedAnalyticsRooms.sublist(batchSize * batchNum).take(batchSize);
+      return null;
-
-      batchNum++;
-      for (final analyticsRoom in batch) {
-        try {
-          final syncFuture =
-              client.waitForRoomInSync(analyticsRoom.roomId, join: true);
-          await client.joinRoom(analyticsRoom.roomId);
-          await syncFuture;
-        } catch (e, s) {
-          ErrorHandler.logError(
-            e: e,
-            s: s,
-            data: {
-              "spaceID": id,
-              "roomID": analyticsRoom.roomId,
-            },
-          );
-        }
-      }
-
-      if (batchSize * batchNum < unjoinedAnalyticsRooms.length) {
-        await Future.delayed(const Duration(milliseconds: 7500));
-      }
     }
   }
 

lib/pangea/extensions/pangea_room_extension.dart

@@ -44,8 +44,6 @@ part "room_user_permissions_extension.dart";
 extension PangeaRoom on Room {
 // analytics
 
-  Future<void> joinAnalyticsRooms() async => await _joinAnalyticsRooms();
-
   Future<DateTime?> analyticsLastUpdated(String userId) async {
     return await _analyticsLastUpdated(userId);
   }

lib/pangea/spaces/widgets/download_analytics_dialog.dart

@@ -1,6 +1,5 @@
 import 'package:flutter/material.dart';
 
-import 'package:collection/collection.dart';
 import 'package:csv/csv.dart';
 import 'package:excel/excel.dart';
 import 'package:flutter_gen/gen_l10n/l10n.dart';
@@ -34,10 +33,9 @@ class DownloadAnalyticsDialog extends StatefulWidget {
 class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
   bool _initialized = false;
   bool _downloaded = false;
-  bool _joiningRooms = false;
   bool _downloading = false;
 
-  bool get _loading => _joiningRooms || _downloading || !_initialized;
+  bool get _loading => _downloading || !_initialized;
 
   String? _error;
 
@@ -65,6 +63,9 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
         },
       );
     } finally {
+      _downloadStatuses = Map.fromEntries(
+        _usersToDownload.map((user) => MapEntry(user.id, 0)),
+      );
       if (mounted) setState(() => _initialized = true);
     }
   }
@@ -78,7 +79,6 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
 
   void _clean() {
     _error = null;
-    _joiningRooms = false;
     _downloading = false;
     _downloaded = false;
     _downloadStatuses = Map.fromEntries(
@@ -88,7 +88,11 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
 
   List<User> get _usersToDownload => widget.space
       .getParticipants()
-      .where((member) => member.id != BotName.byEnvironment)
+      .where(
+        (member) =>
+            member.id != BotName.byEnvironment &&
+            member.membership == Membership.join,
+      )
       .toList();
 
   Color _downloadStatusColor(String userID) {
@@ -100,38 +104,41 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
   }
 
   String? get _statusText {
-    if (_joiningRooms) return L10n.of(context).accessingMemberAnalytics;
     if (_downloading) return L10n.of(context).downloading;
     if (_downloaded) return L10n.of(context).downloadComplete;
     return null;
   }
 
-  Room? _userAnalyticsRoom(String userID) {
+  String? get userL2 =>
-    final rooms = widget.space.client.rooms;
+      MatrixState.pangeaController.languageController.userL2?.langCode;
-    final l2 = MatrixState.pangeaController.languageController.userL2?.langCode;
-    if (l2 == null) return null;
-    return rooms.firstWhereOrNull((room) {
-      return room.isAnalyticsRoomOfUser(userID) && room.isMadeForLang(l2);
-    });
-  }
 
   Future<void> _runDownload() async {
     try {
-      if (!mounted) return;
+      if (!mounted || userL2 == null) return;
       setState(() {
         _error = null;
-        _joiningRooms = true;
+        _downloading = true;
       });
 
-      await widget.space.joinAnalyticsRooms();
+      final List<AnalyticsSummaryModel> summaries = [];
-      if (mounted) {
+      await for (final batch
-        setState(() {
+          in widget.space.getNextAnalyticsRoomBatch(userL2!)) {
-          _joiningRooms = false;
+        if (batch.isEmpty) continue;
-          _downloading = true;
+        final List<AnalyticsSummaryModel?> batchSummaries = await Future.wait(
-        });
+          batch.map((r) => _getAnalyticsModel(r)),
+        );
+        summaries.addAll(batchSummaries.whereType<AnalyticsSummaryModel>());
+      }
+
+      for (final userID in _downloadStatuses.keys) {
+        if (_downloadStatuses[userID] == 0) {
+          _downloadStatuses[userID] = -1;
+          summaries.add(AnalyticsSummaryModel.emptyModel(userID));
+        }
       }
 
-      await _downloadSpaceAnalytics();
+      await _downloadSpaceAnalytics(summaries);
+
       if (mounted) {
         setState(() {
           _downloading = false;
@@ -153,18 +160,12 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
     }
   }
 
-  Future<void> _downloadSpaceAnalytics() async {
+  Future<void> _downloadSpaceAnalytics(
-    final l2 = MatrixState.pangeaController.languageController.userL2?.langCode;
+    List<AnalyticsSummaryModel> summaries,
-    if (l2 == null) return;
+  ) async {
-
-    final List<AnalyticsSummaryModel?> summaries = await Future.wait(
-      _usersToDownload.map((user) => _getUserAnalyticsModel(user.id)),
-    );
-
-    final allSummaries = summaries.whereType<AnalyticsSummaryModel>().toList();
     final content = _downloadType == DownloadType.xlsx
-        ? _getExcelFileContent(allSummaries)
+        ? _getExcelFileContent(summaries)
-        : _getCSVFileContent(allSummaries);
+        : _getCSVFileContent(summaries);
 
     final fileName =
         "analytics_${widget.space.name}_${DateTime.now().toIso8601String()}.${_downloadType == DownloadType.xlsx ? 'xlsx' : 'csv'}";
@@ -176,13 +177,16 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
     );
   }
 
-  Future<AnalyticsSummaryModel?> _getUserAnalyticsModel(String userID) async {
+  Future<AnalyticsSummaryModel?> _getAnalyticsModel(Room analyticsRoom) async {
+    final String? userID = analyticsRoom.creatorId;
+    if (userID == null) return null;
+
+    AnalyticsSummaryModel? summary;
     try {
-      final userAnalyticsRoom = _userAnalyticsRoom(userID);
+      _downloadStatuses[userID] = 1;
-      _downloadStatuses[userID] = userAnalyticsRoom != null ? 1 : -1;
       if (mounted) setState(() {});
 
-      final constructEvents = await userAnalyticsRoom?.getAnalyticsEvents(
+      final constructEvents = await analyticsRoom.getAnalyticsEvents(
         userId: userID,
       );
 
@@ -197,14 +201,13 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
       }
 
       final constructs = ConstructListModel(uses: uses);
-      final summary = AnalyticsSummaryModel.fromConstructListModel(
+      summary = AnalyticsSummaryModel.fromConstructListModel(
         userID,
         constructs,
         getCopy,
         context,
       );
       if (mounted) setState(() => _downloadStatuses[userID] = 2);
-      return summary;
     } catch (e, s) {
       ErrorHandler.logError(
         e: e,
@@ -215,8 +218,23 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
         },
       );
       if (mounted) setState(() => _downloadStatuses[userID] = -2);
+    } finally {
+      if (userID != widget.space.client.userID) {
+        try {
+          await analyticsRoom.leave();
+        } catch (e, s) {
+          ErrorHandler.logError(
+            e: e,
+            s: s,
+            data: {
+              "spaceID": widget.space.id,
+              "userID": userID,
+            },
+          );
+        }
+      }
     }
-    return null;
+    return summary;
   }
 
   List<CellValue> _formatExcelRow(

pubspec.lock

@@ -1523,7 +1523,7 @@ packages:
     description:
       path: "."
       ref: main
-      resolved-ref: f03945433cbcf7ffc33b521a9190630d8bb54513
+      resolved-ref: "16a089ad229671f2367c8927b9edae776004a3ae"
       url: "https://github.com/pangeachat/matrix-dart-sdk.git"
     source: git
     version: "0.36.0"