Merge pull request #13 from emu-sync/dev

Rework upload/download to cater for larger folders

Author: emu-sync

CHANGELOG.md

@@ -72,4 +72,11 @@ This is the same release as 1.0.2, but with a fix for Windows not correctly iden
     - Useful for troubleshooting and understanding exactly when AutoSync occurs.
 
 ## Other
-- Tweaked the layout of several sections to provide clearer visual structure (hopefully!).
+- Tweaked the layout of several sections to provide clearer visual structure (hopefully!).
+
+# v1.0.7
+
+- Added a progress indicator when syncing game files
+- Reworked how EmuSync uploads/downloads files 
+    - This is mostly for people who are uploading larger folders and may have experienced issues where they'd fail
+- Various minor bugixes related to the Dropbox storage provider

NEWS.md

@@ -7,3 +7,6 @@ The status of your game syncs can now be checked via the Decky plugin, as well a
 The latest version of EmuSync will now keep local backups whenever it downloads new files, so if something goes horribly wrong, there is a now a disaster recovery option!
 
 Read more about game backups [here](https://github.com/emu-sync/EmuSync/wiki/Local-game-backups).
+
+## Features I plan to add
+- I'm planning to add options to override backup settings per game, such as changing the amount of backups to keep

src/EmuSync.Agent/Background/SyncTaskWorker.cs

@@ -1,17 +1,25 @@
-namespace EmuSync.Agent.Background;
+using EmuSync.Domain;
+using EmuSync.Domain.Services.Interfaces;
+
+namespace EmuSync.Agent.Background;
 
 public class SyncTaskWorker(
     ILogger<SyncTaskWorker> logger,
     IServiceProvider serviceProvider,
-    ISyncTasks syncTasks
+    ISyncTasks syncTasks,
+    ILocalDataAccessor localDataAccessor
 ) : BackgroundService
 {
     private readonly ILogger<SyncTaskWorker> _logger = logger;
     private readonly IServiceProvider _serviceProvider = serviceProvider;
     private readonly ISyncTasks _syncTasks = syncTasks;
+    private readonly ILocalDataAccessor _localDataAccessor = localDataAccessor;
 
     protected override async Task ExecuteAsync(CancellationToken cancellationToken)
     {
+        //keep temporary zip folder tidy
+        DeleteTemporaryZipFolder();
+
         while (!cancellationToken.IsCancellationRequested)
         {
 
@@ -37,6 +45,24 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken)
 
     }
 
+    private void DeleteTemporaryZipFolder()
+    {
+        try
+        {
+            string path = _localDataAccessor.GetLocalFilePath(DomainConstants.LocalDataGameTempZipsFolder);
+
+            if (Directory.Exists(path))
+            {
+                Directory.Delete(path, true);
+            }
+
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to delete temp zip folder on startup");
+        }
+    }
+
     private async Task TryProcessTasksAsync(CancellationToken cancellationToken)
     {
         try

src/EmuSync.Agent/Controllers/GameSyncController.cs

@@ -1,7 +1,10 @@
 using EmuSync.Agent.Dto.GameSync;
 using EmuSync.Domain.Enums;
+using EmuSync.Domain.Objects;
+using EmuSync.Domain.Services.Interfaces;
 using EmuSync.Services.Managers.Interfaces;
 using EmuSync.Services.Managers.Results;
+using System.Diagnostics.CodeAnalysis;
 
 namespace EmuSync.Agent.Controllers;
 
@@ -13,13 +16,15 @@ public class GameSyncController(
     IGameSyncManager manager,
     IGameManager gameManager,
     ISyncSourceManager syncSourceManager,
-    IGameSyncStatusCache gameSyncStatusCache
+    IGameSyncStatusCache gameSyncStatusCache,
+    ISyncProgressTracker syncProgressTracker
 ) : CustomControllerBase(logger, validator)
 {
     private readonly IGameSyncManager _manager = manager;
     private readonly IGameManager _gameManager = gameManager;
     private readonly ISyncSourceManager _syncSourceManager = syncSourceManager;
     private readonly IGameSyncStatusCache _gameSyncStatusCache = gameSyncStatusCache;
+    private readonly ISyncProgressTracker _syncProgressTracker = syncProgressTracker;
 
     [HttpGet("{id}")]
     public async Task<IActionResult> GetSyncStatus([FromRoute] string id, CancellationToken cancellationToken = default)
@@ -166,4 +171,21 @@ public async Task<IActionResult> RestoreFromBackup([FromRoute] string id, [FromR
 
         return Ok();
     }
+
+    [HttpGet("{id}/SyncProgress")]
+    public async Task<IActionResult> SyncProgress([FromRoute] string id, CancellationToken cancellationToken = default)
+    {
+        SyncProgress? syncProgress = _syncProgressTracker.Get(id);
+
+        SyncProgressDto response = new()
+        {
+            InProgress = syncProgress != null,
+            OverallCompletionPercent = syncProgress?.OverallCompletionPercent is double val
+                ? Math.Round(val, 2)
+                : null,
+            CurrentStage = syncProgress?.CurrentStage,
+        };
+
+        return Ok(response);
+    }
 }

src/EmuSync.Agent/Dto/GameSync/SyncProgressDto.cs

@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace EmuSync.Agent.Dto.GameSync;
+
+public record SyncProgressDto
+{
+    [JsonPropertyName("inProgress")]
+    public bool InProgress { get; set; }
+
+    [JsonPropertyName("overallCompletionPercent")]
+    public double? OverallCompletionPercent { get; set; }
+
+    [JsonPropertyName("currentStage")]
+    public string? CurrentStage { get; set; }
+}

src/EmuSync.Agent/Program.cs

@@ -6,6 +6,8 @@
 using EmuSync.Domain.Enums;
 using EmuSync.Domain.Extensions;
 using EmuSync.Domain.Helpers;
+using EmuSync.Domain.Services;
+using EmuSync.Domain.Services.Interfaces;
 using EmuSync.Services.LudusaviImporter;
 using EmuSync.Services.LudusaviImporter.Interfaces;
 using EmuSync.Services.Managers.Extensions;
@@ -134,6 +136,7 @@ private static void ConfigureServices(WebApplicationBuilder builder)
         #endregion
 
         builder.Services.AddSingleton<ISyncTasks, SyncTasks>();
+        builder.Services.AddSingleton<ISyncProgressTracker, SyncProgressTracker>();
 
         builder.Services.AddSingleton<IApiCache, ApiCache>();
         builder.Services.AddSingleton<IGameSyncStatusCache, GameSyncStatusCache>();

src/EmuSync.Domain/DomainConstants.cs

@@ -6,6 +6,7 @@ public class DomainConstants
     public const string LocalDataFolder = ".emusync-data";
 
     public const string LocalDataGameBackupFolder = "game-backups";
+    public const string LocalDataGameTempZipsFolder = "temp-zips";
     public const string LocalDataGameBackupFileNameFormat = "backup_{0}.zip";
     public const string LocalDataGameBackupManifestFile = "manifest.json";
 

src/EmuSync.Domain/Helpers/ZipHelper.cs

@@ -5,40 +5,60 @@ namespace EmuSync.Domain.Helpers;
 public static class ZipHelper
 {
     /// <summary>
-    /// Creates an in memory zip of all files and folders found at <paramref name="folderPath"/>
+    /// Creates a zip of all files and folders found at <paramref name="folderPath"/>
+    /// and writes it to <paramref name="zipPath"/>
     /// </summary>
-    /// <param name="folderPath"></param>
-    /// <returns></returns>
-    public static MemoryStream CreateZipFromFolder(string folderPath)
-
+    public static void CreateZipFromFolder(
+        string folderPath,
+        string zipPath,
+        Action<double>? onProgressChange = null
+    )
     {
-        var memoryStream = new MemoryStream();
+        var files = Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories);
+        int totalFiles = files.Length;
+        int processedFiles = 0;
+
+        Directory.CreateDirectory(Path.GetDirectoryName(zipPath)!);
+
+        using var fileStream = new FileStream(
+            zipPath,
+            FileMode.Create,
+            FileAccess.Write,
+            FileShare.None
+        );
 
-        using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, leaveOpen: true))
+        using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create);
+
+        foreach (var filePath in files)
         {
-            foreach (var filePath in Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories))
-            {
-                var relativePath = Path.GetRelativePath(folderPath, filePath);
-                var entry = archive.CreateEntry(relativePath, CompressionLevel.Optimal);
+            var relativePath = Path.GetRelativePath(folderPath, filePath);
+            var entry = archive.CreateEntry(relativePath, CompressionLevel.Optimal);
 
-                using var entryStream = entry.Open();
+            using var entryStream = entry.Open();
+            using var input = File.OpenRead(filePath);
+            input.CopyTo(entryStream);
 
-                using var fileStream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-                fileStream.CopyTo(entryStream);
-            }
+            processedFiles++;
+            onProgressChange?.Invoke(
+                totalFiles == 0 ? 100 : (processedFiles / (double)totalFiles) * 100
+            );
         }
-
-        memoryStream.Position = 0; // reset for reading
-        return memoryStream;
     }
 
+
     /// <summary>
     /// Extracts the in-memory zip to <paramref name="outputDirectory"/>
     /// </summary>
     /// <param name="zipStream"></param>
     /// <param name="outputDirectory"></param>
     /// <param name="forceLastWriteTime"></param>
-    public static void ExtractToDirectory(MemoryStream zipStream, string outputDirectory, DateTime? forceLastWriteTime = null)
+    /// <param name="onProgressChange"></param>
+    public static void ExtractToDirectory(
+        Stream zipStream,
+        string outputDirectory,
+        DateTime? forceLastWriteTime = null,
+        Action<double>? onProgressChange = null
+    )
     {
         string? cleanOutputDirectory = GetOsSafePath(outputDirectory);
         if (string.IsNullOrEmpty(cleanOutputDirectory)) return;
@@ -53,7 +73,11 @@ public static void ExtractToDirectory(MemoryStream zipStream, string outputDirec
 
         Directory.CreateDirectory(cleanOutputDirectory);
 
-        foreach (var entry in archive.Entries)
+        var entries = archive.Entries.Where(e => !string.IsNullOrEmpty(e.Name)).ToList();
+        int totalEntries = entries.Count;
+        int processedEntries = 0;
+
+        foreach (var entry in entries)
         {
             var filePath = GetOsSafePath(
                 Path.Combine(cleanOutputDirectory, entry.FullName)
@@ -85,6 +109,9 @@ public static void ExtractToDirectory(MemoryStream zipStream, string outputDirec
             {
                 File.SetLastWriteTimeUtc(filePath, forceLastWriteTime.Value);
             }
+
+            processedEntries++;
+            onProgressChange?.Invoke((processedEntries / (double)totalEntries) * 100);
         }
 
         //stop false positives and ensure we keep the last write time on the local directory the same

src/EmuSync.Domain/Objects/ProgressStream.cs

@@ -0,0 +1,67 @@
+namespace EmuSync.Domain.Objects;
+
+public class ProgressStream : Stream
+{
+    private readonly Stream _inner;
+    private readonly Action<double>? _reportProgress;
+    private readonly ulong _totalSize; // Explicit total size
+    private long _totalRead;
+
+    public ProgressStream(Stream inner, Action<double>? reportProgress, ulong totalSize = 0)
+    {
+        _inner = inner ?? throw new ArgumentNullException(nameof(inner));
+        _reportProgress = reportProgress;
+
+        _totalSize = totalSize > 0 ? totalSize : (inner.CanSeek ? (ulong)inner.Length : 0);
+    }
+
+    public override bool CanRead => _inner.CanRead;
+    public override bool CanSeek => _inner.CanSeek;
+    public override bool CanWrite => _inner.CanWrite;
+    public override long Length => _inner.Length;
+    public override long Position
+    {
+        get => _inner.Position;
+        set => _inner.Position = value;
+    }
+
+    public override void Flush() => _inner.Flush();
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        int read = _inner.Read(buffer, offset, count);
+        Report(read);
+        return read;
+    }
+
+    public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        int read = await _inner.ReadAsync(buffer, offset, count, cancellationToken);
+        Report(read);
+        return read;
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        _inner.Write(buffer, offset, count);
+        Report(count);
+    }
+
+    public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+    {
+        await _inner.WriteAsync(buffer, offset, count, cancellationToken);
+        Report(count);
+    }
+
+    private void Report(int count)
+    {
+        if (_reportProgress == null || count <= 0 || _totalSize <= 0) return;
+
+        _totalRead += count;
+        double percent = (_totalRead / (double)_totalSize) * 100;
+        _reportProgress(percent);
+    }
+
+    public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin);
+    public override void SetLength(long value) => _inner.SetLength(value);
+}

src/EmuSync.Domain/Objects/SyncProgress.cs

@@ -0,0 +1,7 @@
+namespace EmuSync.Domain.Objects;
+
+public record SyncProgress
+{
+    public double OverallCompletionPercent { get; set; }
+    public string CurrentStage { get; set; }
+}