From 66b70d630d31990d936b77f23385166368ee220c Mon Sep 17 00:00:00 2001
From: Sebastian Wilke <sebastian@wilke.id>
Date: Sat, 5 Oct 2024 23:28:53 +0200
Subject: [PATCH] dev: started implementation of batch upload tasks for
 background sync

---
 mobile/lib/services/backup.service.dart | 175 ++++++++++++++----------
 1 file changed, 100 insertions(+), 75 deletions(-)

diff --git a/mobile/lib/services/backup.service.dart b/mobile/lib/services/backup.service.dart
index 9109f9368e..0a0e276b77 100644
--- a/mobile/lib/services/backup.service.dart
+++ b/mobile/lib/services/backup.service.dart
@@ -12,6 +12,7 @@ import 'package:immich_mobile/entities/asset.entity.dart';
 import 'package:immich_mobile/entities/backup_album.entity.dart';
 import 'package:immich_mobile/entities/duplicated_asset.entity.dart';
 import 'package:immich_mobile/entities/store.entity.dart';
+import 'package:immich_mobile/extensions/string_extensions.dart';
 import 'package:immich_mobile/interfaces/album_media.interface.dart';
 import 'package:immich_mobile/interfaces/file_media.interface.dart';
 import 'package:immich_mobile/models/backup/backup_candidate.model.dart';
@@ -277,11 +278,13 @@ class BackupService {
     }
 
     List<BackupCandidate> candidates = assets.toList();
+    List<UploadTask> backgroundTasks = [];
+
     if (isBackground) {
       candidates = _sortPhotosFirst(candidates);
     }
 
-    for (final candidate in candidates) {
+    for (final (i, candidate) in candidates.indexed) {
       final Asset asset = candidate.asset;
       File? file;
       File? livePhotoFile;
@@ -361,9 +364,7 @@ class BackupService {
             onProgress: onProgress,
           );
 
-          int statusCode;
           final fileLength = file.lengthSync();
-          dynamic body;
 
           String? livePhotoVideoId;
           if (asset.local!.isLivePhoto && livePhotoFile != null) {
@@ -379,7 +380,20 @@ class BackupService {
             baseRequest.fields['livePhotoVideoId'] = livePhotoVideoId;
           }
 
-          if (!isBackground) {
+          onCurrentAsset(
+            CurrentUploadAsset(
+              id: asset.localId!,
+              fileCreatedAt: asset.fileCreatedAt.year == 1970
+                  ? asset.fileModifiedAt
+                  : asset.fileCreatedAt,
+              fileName: originalFileName,
+              fileType: _getAssetType(asset.type),
+              fileSize: file.lengthSync(),
+              iCloudAsset: false,
+            ),
+          );
+
+          if (isBackground) {
             print("BACKGROUND SYNC");
             final (baseDir, dir, filename) = await Task.split(file: file);
 
@@ -398,22 +412,10 @@ class BackupService {
               fields: baseRequest.fields,
               headers: baseRequest.headers,
               updates: Updates.statusAndProgress,
+              metaData: i.toString(),
             );
 
-            final response = await FileDownloader().upload(
-              backgroundRequest,
-              onProgress: (percentage) => {
-                // onProgress returns a double in [0.0;1.0] for percentage
-                if (percentage > 0)
-                  baseRequest.onProgress(
-                    (percentage * fileLength).toInt(),
-                    fileLength,
-                  ),
-              },
-            );
-
-            body = jsonDecode(response.responseBody ?? "{}");
-            statusCode = response.responseStatusCode ?? 500;
+            backgroundTasks.add(backgroundRequest);
           } else {
             print("FOREGROUND SYNC");
             final fileStream = file.openRead();
@@ -440,68 +442,34 @@ class BackupService {
               cancellationToken: cancelToken,
             );
 
-            body = jsonDecode(await response.stream.bytesToString());
-            statusCode = response.statusCode;
-          }
-
-          onCurrentAsset(
-            CurrentUploadAsset(
-              id: asset.localId!,
-              fileCreatedAt: asset.fileCreatedAt.year == 1970
-                  ? asset.fileModifiedAt
-                  : asset.fileCreatedAt,
-              fileName: originalFileName,
-              fileType: _getAssetType(asset.type),
-              fileSize: file.lengthSync(),
-              iCloudAsset: false,
-            ),
-          );
-
-          if (![200, 201].contains(statusCode)) {
-            final error = body;
-
-            // debugPrint(
-            //   "Error(${error['statusCode']}) uploading ${asset.localId} | $originalFileName | Created on ${asset.fileCreatedAt} | ${error['error']}",
-            // );
-
-            onError(
-              ErrorUploadAsset(
-                asset: asset,
-                id: asset.localId!,
-                fileCreatedAt: asset.fileCreatedAt,
-                fileName: originalFileName,
-                fileType: _getAssetType(candidate.asset.type),
-                errorMessage: error['error'],
-              ),
-            );
-
-            if (error == "Quota has been exceeded!") {
-              anyErrors = true;
+            dynamic body = jsonDecode(await response.stream.bytesToString());
+            int statusCode = response.statusCode;
+            if ((anyErrors =
+                    !_handleUploadError(asset, body, statusCode, onError)) ==
+                true) {
               break;
             }
 
-            continue;
-          }
+            bool isDuplicate = false;
+            if (statusCode == 200) {
+              isDuplicate = true;
+              duplicatedAssetIds.add(asset.localId!);
+            }
 
-          bool isDuplicate = false;
-          if (statusCode == 200) {
-            isDuplicate = true;
-            duplicatedAssetIds.add(asset.localId!);
-          }
-
-          onSuccess(
-            SuccessUploadAsset(
-              candidate: candidate,
-              remoteAssetId: body['id'] as String,
-              isDuplicate: isDuplicate,
-            ),
-          );
-
-          if (shouldSyncAlbums) {
-            await _albumService.syncUploadAlbums(
-              candidate.albumNames,
-              [body['id'] as String],
+            onSuccess(
+              SuccessUploadAsset(
+                candidate: candidate,
+                remoteAssetId: body['id'] as String,
+                isDuplicate: isDuplicate,
+              ),
             );
+
+            if (shouldSyncAlbums) {
+              await _albumService.syncUploadAlbums(
+                candidate.albumNames,
+                [body['id'] as String],
+              );
+            }
           }
         }
       } on http.CancelledException {
@@ -524,6 +492,32 @@ class BackupService {
       }
     }
 
+    if (isBackground) {
+      final response = await FileDownloader().uploadBatch(
+        backgroundTasks,
+        taskProgressCallback: (update) {
+          onProgress((update.progress * update.expectedFileSize).toInt(),
+              update.expectedFileSize);
+        },
+        taskStatusCallback: (update) {
+          // BackupCandidate index is stored in task metadata
+          int i = update.task.metaData.toInt();
+          if (update.status.isFinalState) {
+            dynamic body = jsonDecode(update.responseBody ?? "{}");
+            int statusCode = update.responseStatusCode ?? 500;
+            if (_handleUploadError(
+              candidates[i].asset,
+              body,
+              statusCode,
+              onError,
+            )) {
+              // TODO: Cancel queue because quota exceeded
+            }
+          }
+        },
+      );
+    }
+
     if (duplicatedAssetIds.isNotEmpty) {
       await _saveDuplicatedAssetIds(duplicatedAssetIds);
     }
@@ -531,6 +525,37 @@ class BackupService {
     return !anyErrors;
   }
 
+  bool _handleUploadError(
+    Asset asset,
+    dynamic body,
+    int statusCode,
+    void Function(ErrorUploadAsset error) onError,
+  ) {
+    if (![200, 201].contains(statusCode)) {
+      final error = body;
+
+      debugPrint(
+        "Error(${error['statusCode']}) uploading ${asset.localId} | ${asset.fileName} | Created on ${asset.fileCreatedAt} | ${error['error']}",
+      );
+
+      onError(
+        ErrorUploadAsset(
+          asset: asset,
+          id: asset.localId!,
+          fileCreatedAt: asset.fileCreatedAt,
+          fileName: asset.fileName,
+          fileType: _getAssetType(asset.type),
+          errorMessage: error['error'],
+        ),
+      );
+
+      if (error == "Quota has been exceeded!") {
+        return false;
+      }
+    }
+    return true;
+  }
+
   Future<String?> uploadLivePhotoVideo(
     String originalFileName,
     File? livePhotoVideoFile,