Skip to content

Commit 5689c8d

Browse files
PathoschildPathoschild
committed
switch to zip download for fetching mod dataset
Azure App Services doesn't have Git installed on the servers by default, and alternatives like LibGit2Sharp are slower than just redownloading the latest branch when it changes.
1 parent a444b2b commit 5689c8d

5 files changed

Lines changed: 181 additions & 81 deletions

File tree

‎src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ internal interface IModDatasetRepository
1111
*********/
1212
/// <summary>Fetch the latest mod dataset, if it changed or hasn't been fetched yet.</summary>
1313
/// <param name="log">A callback which should receive progress messages for logging.</param>
14+
/// <exception cref="InvalidOperationException">The downloaded archive isn't a valid mod dataset.</exception>
1415
Task UpdateAsync(Action<string>? log = null);
1516

1617
/// <summary>Get the full path to a file in the mod dataset.</summary>

‎src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs‎

Lines changed: 138 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
using System;
2-
using System.Diagnostics;
32
using System.IO;
3+
using System.IO.Compression;
4+
using System.Net;
5+
using System.Net.Http;
6+
using System.Net.Http.Headers;
7+
using System.Text.Json;
48
using System.Threading.Tasks;
59

610
namespace StardewModdingAPI.Web.Framework.Caching.ModDataset;
@@ -11,75 +15,171 @@ internal class ModDatasetRepository : IModDatasetRepository
1115
/*********
1216
** Fields
1317
*********/
14-
/// <summary>The HTTPS URL of the mod dataset Git repo.</summary>
15-
private readonly string RepoUrl;
18+
/// <summary>The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL).</summary>
19+
private readonly string DownloadZipUrl;
1620

17-
/// <summary>The full path to the mod dataset repo.</summary>
18-
private readonly string LocalRepoPath;
21+
/// <summary>The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted.</summary>
22+
private readonly string LocalRootPath;
23+
24+
/// <summary>The HTTP client with which to fetch the mod dataset archive.</summary>
25+
private readonly HttpClient HttpClient;
26+
27+
/// <summary>The name of the file within the <see cref="LocalRootPath"/> in which to cache data about the last fetched dataset.</summary>
28+
private const string CacheFileName = ".cache.json";
29+
30+
/// <summary>The path to the actual 'dataset' folder.</summary>
31+
private string? DatasetPath;
1932

2033

2134
/*********
2235
** Public methods
2336
*********/
2437
/// <summary>Construct an instance.</summary>
25-
/// <param name="repoUrl"><inheritdoc cref="RepoUrl" path="/summary" /></param>
26-
/// <param name="localRepoPath">The path to the mod dataset repo.</param>
27-
public ModDatasetRepository(string repoUrl, string localRepoPath)
38+
/// <param name="downloadZipUrl"><inheritdoc cref="DownloadZipUrl" path="/summary" /></param>
39+
/// <param name="localPath"><inheritdoc cref="LocalRootPath" path="/summary" /></param>
40+
/// <param name="userAgent">The user agent to use when fetching the archive URL.</param>
41+
public ModDatasetRepository(string downloadZipUrl, string localPath, string userAgent)
2842
{
29-
this.RepoUrl = repoUrl;
30-
this.LocalRepoPath = Path.GetFullPath(
31-
Environment.ExpandEnvironmentVariables(localRepoPath)
32-
);
43+
this.DownloadZipUrl = downloadZipUrl;
44+
this.LocalRootPath = Path.GetFullPath(this.ExpandEnvironmentVariables(localPath));
45+
46+
this.HttpClient = new HttpClient();
47+
this.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
3348
}
3449

3550
/// <inheritdoc />
3651
public async Task UpdateAsync(Action<string>? log = null)
3752
{
38-
if (!Directory.Exists(Path.Combine(this.LocalRepoPath, ".git")))
53+
// check for new archive
54+
log?.Invoke(" Checking for newer dataset...");
55+
LastDownload? cached = await this.GetCacheInfoAsync();
56+
using HttpRequestMessage request = this.BuildDownloadRequest(cached?.ETag);
57+
using HttpResponseMessage response = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
58+
59+
// download new archive
60+
if (cached is null || response.StatusCode is not HttpStatusCode.NotModified)
3961
{
40-
log?.Invoke("Cloning mod dataset repo...");
41-
Directory.CreateDirectory(Path.GetDirectoryName(this.LocalRepoPath)!);
42-
await this.RunGitAsync("clone", "--depth", "1", this.RepoUrl, this.LocalRepoPath);
62+
// read response data
63+
if (response.StatusCode is not HttpStatusCode.NotModified)
64+
response.EnsureSuccessStatusCode();
65+
string? newEtag = response.Headers.ETag?.ToString();
66+
67+
// download & unzip dataset
68+
log?.Invoke(" Downloading dataset...");
69+
string newFolderName = Guid.NewGuid().ToString("N");
70+
string newRootPath = Path.Combine(this.LocalRootPath, newFolderName);
71+
Directory.CreateDirectory(newRootPath);
72+
await using (Stream downloadStream = await response.Content.ReadAsStreamAsync())
73+
await using (ZipArchive archive = new(downloadStream, ZipArchiveMode.Read))
74+
await archive.ExtractToDirectoryAsync(newRootPath);
75+
76+
// locate 'dataset' folder
77+
log?.Invoke(" Locating 'dataset' folder...");
78+
string? newDatasetPath = null;
79+
foreach (DirectoryInfo entry in new DirectoryInfo(newRootPath).EnumerateDirectories("*", SearchOption.AllDirectories))
80+
{
81+
if (entry.Name == "dataset")
82+
{
83+
newDatasetPath = Path.GetRelativePath(newRootPath, entry.FullName);
84+
break;
85+
}
86+
}
87+
if (newDatasetPath is null)
88+
throw new InvalidOperationException("No 'dataset' folder found in the downloaded archive.");
89+
90+
// save cache info
91+
cached = new LastDownload(newFolderName, newDatasetPath, newEtag);
92+
await this.SaveCacheInfoAsync(cached);
93+
log?.Invoke($" Dataset saved to {newRootPath} with {(newEtag != null ? $"ETag header {newEtag}" : "no ETag header")}.");
4394
}
44-
else
95+
96+
// track dataset path
97+
this.DatasetPath = Path.Combine(this.LocalRootPath, cached.FolderName, cached.RelativePathToDataset);
98+
99+
// clear previous datasets if possible
100+
foreach (DirectoryInfo directory in new DirectoryInfo(this.LocalRootPath).EnumerateDirectories())
45101
{
46-
log?.Invoke("Fetching latest changes from mod dataset repo...");
47-
await this.RunGitAsync("-C", this.LocalRepoPath, "fetch", "--depth", "1", "origin");
48-
await this.RunGitAsync("-C", this.LocalRepoPath, "reset", "--hard", "FETCH_HEAD");
102+
if (directory.Name == cached.FolderName)
103+
continue;
104+
105+
log?.Invoke($" Deleting previous dataset download at {directory.FullName}...");
106+
try
107+
{
108+
directory.Delete(recursive: true);
109+
}
110+
catch (Exception ex)
111+
{
112+
log?.Invoke($" Deletion failed: {ex}");
113+
}
49114
}
50115
}
51116

52117
/// <inheritdoc />
53118
public string GetFilePath(string relativePath)
54119
{
55-
return Path.Combine(this.LocalRepoPath, "dataset", relativePath);
120+
return this.DatasetPath is not null
121+
? Path.Combine(this.DatasetPath, relativePath)
122+
: throw new InvalidOperationException($"Must call '{nameof(this.UpdateAsync)}' before '{nameof(this.GetFilePath)}'.");
56123
}
57124

58125

59126
/*********
60127
** Private methods
61128
*********/
62-
/// <summary>Run a Git command and wait for it to complete.</summary>
63-
/// <param name="args">The git arguments.</param>
64-
/// <exception cref="InvalidOperationException">The Git command exited with a non-zero exit code.</exception>
65-
private async Task RunGitAsync(params string[] args)
129+
/// <summary>Build the HTTP request to download the archive from the server if it changed.</summary>
130+
/// <param name="etag">The ETag value for the last successful dataset download, if any.</param>
131+
private HttpRequestMessage BuildDownloadRequest(string? etag)
66132
{
67-
ProcessStartInfo startInfo = new("git", args)
133+
HttpRequestMessage request = new(HttpMethod.Get, this.DownloadZipUrl);
134+
try
135+
{
136+
if (etag != null)
137+
request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(etag));
138+
}
139+
catch
68140
{
69-
RedirectStandardOutput = true,
70-
RedirectStandardError = true,
71-
UseShellExecute = false,
72-
CreateNoWindow = true
73-
};
141+
request.Dispose();
142+
throw;
143+
}
144+
145+
return request;
146+
}
147+
148+
/// <summary>Get the cached info about the last dataset download, if it was previously downloaded.</summary>
149+
private async Task<LastDownload?> GetCacheInfoAsync()
150+
{
151+
string path = Path.Combine(this.LocalRootPath, CacheFileName);
152+
if (!File.Exists(path))
153+
return null;
154+
155+
await using FileStream stream = File.OpenRead(path);
156+
return await JsonSerializer.DeserializeAsync<LastDownload>(stream);
157+
}
158+
159+
/// <summary>Save the cache info about a dataset download.</summary>
160+
/// <param name="cacheInfo">The cache info to save.</param>
161+
private async Task SaveCacheInfoAsync(LastDownload cacheInfo)
162+
{
163+
string path = Path.Combine(this.LocalRootPath, CacheFileName);
74164

75-
using Process process = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start Git process.");
165+
await using FileStream stream = File.Create(path);
166+
await JsonSerializer.SerializeAsync(stream, cacheInfo);
167+
}
76168

77-
Task readOutput = process.StandardOutput.ReadToEndAsync();
78-
Task<string> readError = process.StandardError.ReadToEndAsync();
79-
await process.WaitForExitAsync();
80-
await Task.WhenAll(readOutput, readError);
169+
/// <summary>Expand environment variables in a path.</summary>
170+
/// <param name="path">The path to parse.</param>
171+
/// <returns>Returns the path with any environment variables replaced.</returns>
172+
private string ExpandEnvironmentVariables(string path)
173+
{
174+
if (OperatingSystem.IsLinux())
175+
path = path.Replace("%TEMP%", Path.GetTempPath(), StringComparison.OrdinalIgnoreCase);
81176

82-
if (process.ExitCode != 0)
83-
throw new InvalidOperationException($"Git command failed with exit code {process.ExitCode}: {await readError}.");
177+
return Environment.ExpandEnvironmentVariables(path);
84178
}
179+
180+
/// <summary>The cached metadata about the last dataset download.</summary>
181+
/// <param name="FolderName">The folder name within the root folder.</param>
182+
/// <param name="RelativePathToDataset">The relative path to the 'dataset' folder within the <see cref="FolderName"/>.</param>
183+
/// <param name="ETag">The ETag value for the downloaded archive, if available.</param>
184+
private record LastDownload(string FolderName, string RelativePathToDataset, string? ETag);
85185
}

‎src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ internal class ModDatasetConfig
66
/*********
77
** Accessors
88
*********/
9-
/// <summary>The HTTPS URL of the mod dataset Git repo.</summary>
10-
public string RepoUrl { get; set; } = null!;
9+
/// <summary>The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL).</summary>
10+
public string DownloadZipUrl { get; set; } = null!;
1111

12-
/// <summary>The local path into which to clone the mod dataset repo.</summary>
13-
public string LocalPath { get; set; } = null!;
12+
/// <summary>The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted.</summary>
13+
public string LocalRootPath { get; set; } = null!;
1414
}

‎src/SMAPI.Web/Startup.cs‎

Lines changed: 36 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,15 @@ public Startup(IWebHostEnvironment env)
8383
/// <param name="services">The service injection container.</param>
8484
public void ConfigureServices(IServiceCollection services)
8585
{
86+
// load config
87+
ApiClientsConfig apiConfig = this.Configuration.GetRequiredSection("ApiClients").Get<ApiClientsConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded.");
88+
BackgroundServicesConfig backgroundServicesConfig = this.Configuration.GetRequiredSection("BackgroundServices").Get<BackgroundServicesConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'BackgroundServices' config section couldn't be loaded.");
89+
ModDatasetConfig datasetConfig = this.Configuration.GetRequiredSection("ModDataset").Get<ModDatasetConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded.");
90+
91+
// init user agent
92+
string userAgentVersion = this.GetType().Assembly.GetName().Version!.ToString(3);
93+
string userAgent = string.Format(apiConfig.UserAgent, userAgentVersion);
94+
8695
// init basic services
8796
services
8897
.Configure<ApiClientsConfig>(this.Configuration.GetRequiredSection("ApiClients"))
@@ -110,10 +119,7 @@ public void ConfigureServices(IServiceCollection services)
110119
services.AddSingleton<INexusExportCacheRepository>(new NexusExportCacheMemoryRepository());
111120

112121
// init mod dataset
113-
{
114-
ModDatasetConfig config = this.Configuration.GetRequiredSection("ModDataset").Get<ModDatasetConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded.");
115-
services.AddSingleton<IModDatasetRepository>(new ModDatasetRepository(config.RepoUrl, config.LocalPath));
116-
}
122+
services.AddSingleton<IModDatasetRepository>(new ModDatasetRepository(datasetConfig.DownloadZipUrl, datasetConfig.LocalRootPath, userAgent));
117123

118124
// init Hangfire
119125
services
@@ -128,29 +134,22 @@ public void ConfigureServices(IServiceCollection services)
128134
});
129135

130136
// init background service
131-
{
132-
BackgroundServicesConfig config = this.Configuration.GetRequiredSection("BackgroundServices").Get<BackgroundServicesConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded.");
133-
if (config.Enabled)
134-
services.AddHostedService<BackgroundService>();
135-
}
137+
if (backgroundServicesConfig.Enabled)
138+
services.AddHostedService<BackgroundService>();
136139

137140
// init API clients
138141
{
139-
ApiClientsConfig api = this.Configuration.GetRequiredSection("ApiClients").Get<ApiClientsConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded.");
140-
string version = this.GetType().Assembly.GetName().Version!.ToString(3);
141-
string userAgent = string.Format(api.UserAgent, version);
142-
143142
services.AddSingleton<IChucklefishClient>(new ChucklefishClient(
144143
userAgent: userAgent,
145-
baseUrl: api.ChucklefishBaseUrl,
146-
modPageUrlFormat: api.ChucklefishModPageUrlFormat
144+
baseUrl: apiConfig.ChucklefishBaseUrl,
145+
modPageUrlFormat: apiConfig.ChucklefishModPageUrlFormat
147146
));
148147

149-
if (!string.IsNullOrWhiteSpace(api.CurseForgeExportUrl))
148+
if (!string.IsNullOrWhiteSpace(apiConfig.CurseForgeExportUrl))
150149
{
151150
services.AddSingleton<ICurseForgeExportApiClient>(new CurseForgeExportApiClient(
152151
userAgent: userAgent,
153-
baseUrl: api.CurseForgeExportUrl
152+
baseUrl: apiConfig.CurseForgeExportUrl
154153
));
155154
}
156155
else
@@ -159,26 +158,26 @@ public void ConfigureServices(IServiceCollection services)
159158
services.AddSingleton<ICurseForgeClient>(
160159
provider => new CurseForgeClient(
161160
userAgent: userAgent,
162-
apiUrl: api.CurseForgeBaseUrl,
163-
apiKey: api.CurseForgeApiKey,
164-
webModUrl: api.CurseForgeWebPageUrl,
161+
apiUrl: apiConfig.CurseForgeBaseUrl,
162+
apiKey: apiConfig.CurseForgeApiKey,
163+
webModUrl: apiConfig.CurseForgeWebPageUrl,
165164
exportCache: provider.GetRequiredService<ICurseForgeExportCacheRepository>()
166165
)
167166
);
168167

169168
services.AddSingleton<IGitHubClient>(new GitHubClient(
170-
baseUrl: api.GitHubBaseUrl,
169+
baseUrl: apiConfig.GitHubBaseUrl,
171170
userAgent: userAgent,
172-
acceptHeader: api.GitHubAcceptHeader,
173-
username: api.GitHubUsername,
174-
password: api.GitHubPassword
171+
acceptHeader: apiConfig.GitHubAcceptHeader,
172+
username: apiConfig.GitHubUsername,
173+
password: apiConfig.GitHubPassword
175174
));
176175

177-
if (!string.IsNullOrWhiteSpace(api.ModDropExportUrl))
176+
if (!string.IsNullOrWhiteSpace(apiConfig.ModDropExportUrl))
178177
{
179178
services.AddSingleton<IModDropExportApiClient>(new ModDropExportApiClient(
180179
userAgent: userAgent,
181-
baseUrl: api.ModDropExportUrl
180+
baseUrl: apiConfig.ModDropExportUrl
182181
));
183182
}
184183
else
@@ -187,42 +186,42 @@ public void ConfigureServices(IServiceCollection services)
187186
services.AddSingleton<IModDropClient>(
188187
provider => new ModDropClient(
189188
userAgent: userAgent,
190-
apiUrl: api.ModDropApiUrl,
191-
modUrlFormat: api.ModDropModPageUrl,
189+
apiUrl: apiConfig.ModDropApiUrl,
190+
modUrlFormat: apiConfig.ModDropModPageUrl,
192191
exportCache: provider.GetRequiredService<IModDropExportCacheRepository>()
193192
)
194193
);
195194

196-
if (!string.IsNullOrWhiteSpace(api.NexusExportUrl))
195+
if (!string.IsNullOrWhiteSpace(apiConfig.NexusExportUrl))
197196
{
198197
services.AddSingleton<INexusExportApiClient>(
199198
new NexusExportApiClient(
200199
userAgent: userAgent,
201-
baseUrl: api.NexusExportUrl
200+
baseUrl: apiConfig.NexusExportUrl
202201
)
203202
);
204203
}
205204
else
206205
services.AddSingleton<INexusExportApiClient>(new DisabledNexusExportApiClient());
207206

208-
if (!string.IsNullOrWhiteSpace(api.NexusApiKey))
207+
if (!string.IsNullOrWhiteSpace(apiConfig.NexusApiKey))
209208
{
210209
services.AddSingleton<INexusClient>(
211210
provider => new NexusClient(
212211
webUserAgent: userAgent,
213-
webBaseUrl: api.NexusBaseUrl,
214-
webModUrlFormat: api.NexusModUrlFormat,
215-
webModScrapeUrlFormat: api.NexusModScrapeUrlFormat,
216-
apiAppVersion: version,
217-
apiKey: api.NexusApiKey,
212+
webBaseUrl: apiConfig.NexusBaseUrl,
213+
webModUrlFormat: apiConfig.NexusModUrlFormat,
214+
webModScrapeUrlFormat: apiConfig.NexusModScrapeUrlFormat,
215+
apiAppVersion: userAgentVersion,
216+
apiKey: apiConfig.NexusApiKey,
218217
exportCache: provider.GetRequiredService<INexusExportCacheRepository>()
219218
));
220219
}
221220
else
222221
services.AddSingleton<INexusClient>(new DisabledNexusClient());
223222

224223
services.AddSingleton<IPastebinClient>(new PastebinClient(
225-
baseUrl: api.PastebinBaseUrl,
224+
baseUrl: apiConfig.PastebinBaseUrl,
226225
userAgent: userAgent
227226
));
228227

0 commit comments

Comments
 (0)