Skip to content

Commit 6516f73

Browse files
AlexCSDevAlexCSDev
authored
Fixes for issues 87, 88, 89 (#97)
* Added filename truncation with maximum length configured by --max-filename-length, added unit tests for truncation testing (issue #87) * Fixed PostSubdirectoryHelper exception on posts without title, added unit tests for that (issue #88) * Increased MaxFilenameLength to 100, updated UniversalDownloaderPlatform to fix #89, post subdirectories are now created in PatreonPageCrawler only if description/embed saving is enabled * Changed version
1 parent 7f07561 commit 6516f73

14 files changed

Lines changed: 267 additions & 8 deletions

‎PatreonDownloader.App/Models/CommandLineOptions.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,7 @@ class CommandLineOptions
3737

3838
[Option("sub-directory-pattern", Required = false, HelpText = "Pattern which will be used to create a name for the sub directories if --use-sub-directories is used. Supported parameters: %PostId%, %PublishedAt%, %PostTitle%.", Default = "[%PostId%] %PublishedAt% %PostTitle%")]
3939
public string SubDirectoryPattern { get; set; }
40+
[Option("max-filename-length", Required = false, HelpText = "All names of downloaded files will be truncated so their length won't be more than specified value (excluding file extension)", Default = 100)]
41+
public int MaxFilenameLength { get; set; }
4042
}
4143
}

‎PatreonDownloader.App/Program.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,8 @@ private static async Task<PatreonDownloaderSettings> InitializeSettings(CommandL
190190
DownloadDirectory = commandLineOptions.DownloadDirectory,
191191
RemoteFileSizeNotAvailableAction = commandLineOptions.NoRemoteSizeAction,
192192
UseSubDirectories = commandLineOptions.UseSubDirectories,
193-
SubDirectoryPattern = commandLineOptions.SubDirectoryPattern
193+
SubDirectoryPattern = commandLineOptions.SubDirectoryPattern,
194+
MaxFilenameLength = commandLineOptions.MaxFilenameLength
194195
};
195196

196197
return settings;

‎PatreonDownloader.App/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,5 @@
2929
// Build Number
3030
// Revision
3131
//
32-
[assembly: AssemblyVersion("0.10.1.0")]
33-
[assembly: AssemblyFileVersion("0.10.1.0")]
32+
[assembly: AssemblyVersion("0.10.1.1")]
33+
[assembly: AssemblyFileVersion("0.10.1.1")]

‎PatreonDownloader.Implementation/Helpers/PostSubdirectoryHelper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ internal class PostSubdirectoryHelper
2020
/// <returns></returns>
2121
public static string CreateNameFromPattern(PatreonCrawledUrl crawledUrl, string pattern)
2222
{
23-
string postTitle = crawledUrl.Title.Trim();
23+
string postTitle = crawledUrl.Title?.Trim() ?? "No Title";
2424
while (postTitle.Length > 1 && postTitle[^1] == '.')
2525
postTitle = postTitle.Remove(postTitle.Length - 1).Trim();
2626

‎PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public class PatreonDownloaderSettings : UniversalDownloaderPlatformSettings
1717
private string _downloadDirectory;
1818
private bool _useSubDirectories;
1919
private string _subDirectoryPattern;
20+
private int _maxFilenameLength;
2021

2122
public bool SaveDescriptions
2223
{
@@ -69,6 +70,15 @@ public string SubDirectoryPattern
6970
set => ConsumableSetter.Set(Consumed, ref _subDirectoryPattern, value);
7071
}
7172

73+
/// <summary>
74+
/// Filenames will be truncated to this length
75+
/// </summary>
76+
public int MaxFilenameLength
77+
{
78+
get => _maxFilenameLength;
79+
set => ConsumableSetter.Set(Consumed, ref _maxFilenameLength, value);
80+
}
81+
7282
public PatreonDownloaderSettings()
7383
{
7484
_saveDescriptions = true;
@@ -78,11 +88,12 @@ public PatreonDownloaderSettings()
7888
_downloadDirectory = null;
7989
_useSubDirectories = false;
8090
_subDirectoryPattern = "[%PostId%] %PublishedAt% %PostTitle%";
91+
_maxFilenameLength = 100;
8192
}
8293

8394
public override string ToString()
8495
{
85-
return $"SaveDescriptions={_saveDescriptions},SaveEmbeds={_saveEmbeds},SaveJson={_saveJson},SaveAvatarAndCover={_saveAvatarAndCover},DownloadDirectory={_downloadDirectory},OverwriteFiles={base.OverwriteFiles},UseSubDirectories={_useSubDirectories}";
96+
return $"SaveDescriptions={_saveDescriptions},SaveEmbeds={_saveEmbeds},SaveJson={_saveJson},SaveAvatarAndCover={_saveAvatarAndCover},DownloadDirectory={_downloadDirectory},OverwriteFiles={base.OverwriteFiles},UseSubDirectories={_useSubDirectories},MaxFilenameLength={_maxFilenameLength}";
8697
}
8798
}
8899
}

‎PatreonDownloader.Implementation/PatreonCrawledUrlProcessor.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,18 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl, string down
152152
filename = PathSanitizer.SanitizePath(filename);
153153
_logger.Debug($"Sanitized filename: {filename}");
154154

155+
if (filename.Length > _patreonDownloaderSettings.MaxFilenameLength)
156+
{
157+
_logger.Debug($"Filename is too long, will be truncated: {filename}");
158+
string extension = Path.GetExtension(filename);
159+
if (extension.Length > 4)
160+
{
161+
_logger.Warn($"File extension for file {filename} is longer 4 characters and won't be appended to truncated filename!");
162+
extension = "";
163+
}
164+
filename = filename.Substring(0, _patreonDownloaderSettings.MaxFilenameLength) + extension;
165+
_logger.Debug($"Truncated filename: {filename}");
166+
}
155167

156168
string key = $"{crawledUrl.PostId}_{filename.ToLowerInvariant()}";
157169
if (!_fileCountDict.ContainsKey(key))

‎PatreonDownloader.Implementation/PatreonDownloader.Implementation.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<TargetFramework>net5.0</TargetFramework>
5+
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
56
</PropertyGroup>
67

78
<ItemGroup>

‎PatreonDownloader.Implementation/PatreonPageCrawler.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,11 @@ private async Task<ParsingResult> ParsePage(string json, string downloadDirector
150150
};
151151

152152
string additionalFilesSaveDirectory = downloadDirectory;
153-
if (_patreonDownloaderSettings.UseSubDirectories)
153+
if (_patreonDownloaderSettings.UseSubDirectories &&
154+
(_patreonDownloaderSettings.SaveDescriptions ||
155+
(jsonEntry.Attributes.Embed != null && _patreonDownloaderSettings.SaveEmbeds)
156+
)
157+
)
154158
{
155159
additionalFilesSaveDirectory = Path.Combine(downloadDirectory,
156160
PostSubdirectoryHelper.CreateNameFromPattern(entry, _patreonDownloaderSettings.SubDirectoryPattern));

‎PatreonDownloader.Implementation/Properties/AssemblyInfo.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
using System.Reflection;
2+
using System.Runtime.CompilerServices;
3+
using System.Runtime.InteropServices;
4+
5+
// In SDK-style projects such as this one, several assembly attributes that were historically
6+
// defined in this file are now automatically added during build and populated with
7+
// values defined in project properties. For details of which attributes are included
8+
// and how to customise this process see: https://aka.ms/assembly-info-properties
9+
10+
// General Information about an assembly is controlled through the following
11+
// set of attributes. Change these attribute values to modify the information
12+
// associated with an assembly.
13+
[assembly: AssemblyTitle("Patreon Downloader Implementation Library")]
14+
[assembly: AssemblyDescription("")]
15+
[assembly: AssemblyConfiguration("")]
16+
[assembly: AssemblyCompany("")]
17+
[assembly: AssemblyProduct("Patreon Downloader")]
18+
[assembly: AssemblyCopyright("Copyright 2019-2021 Aleksey Tsutsey & Contributors")]
19+
[assembly: AssemblyTrademark("")]
20+
[assembly: AssemblyCulture("")]
21+
22+
23+
// Setting ComVisible to false makes the types in this assembly not visible to COM
24+
// components. If you need to access a type in this assembly from COM, set the ComVisible
25+
// attribute to true on that type.
26+
27+
[assembly: ComVisible(false)]
28+
29+
// The following GUID is for the ID of the typelib if this project is exposed to COM.
30+
31+
[assembly: Guid("11fe4289-dd12-4f48-a571-938e4261f26d")]
32+
33+
[assembly: InternalsVisibleTo("PatreonDownloader.Tests")]

‎PatreonDownloader.Tests/PatreonCrawledUrlProcessorTests.cs‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
using PatreonDownloader.Implementation;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.IO;
5+
using System.Linq;
6+
using System.Net;
7+
using System.Text;
8+
using System.Threading.Tasks;
9+
using PatreonDownloader.Implementation.Enums;
10+
using PatreonDownloader.Implementation.Models;
11+
using UniversalDownloaderPlatform.Common.Enums;
12+
using Xunit;
13+
14+
namespace PatreonDownloader.Tests
15+
{
16+
public class PatreonCrawledUrlProcessorTests
17+
{
18+
[Fact]
19+
public async Task ProcessCrawledUrl_MediaFileNameIsUrl_IsTruncatedAndNoExtension()
20+
{
21+
PatreonDownloaderSettings settings = new PatreonDownloaderSettings
22+
{
23+
CookieContainer = new CookieContainer(),
24+
DownloadDirectory = "c:\\downloads",
25+
MaxDownloadRetries = 10,
26+
OverwriteFiles = false,
27+
RemoteFileSizeNotAvailableAction = RemoteFileSizeNotAvailableAction.KeepExisting,
28+
RetryMultiplier = 1,
29+
SaveAvatarAndCover = true,
30+
SaveDescriptions = true,
31+
SaveEmbeds = true,
32+
SaveJson = true,
33+
UseSubDirectories = true,
34+
SubDirectoryPattern = "[%PostId%] %PublishedAt% %PostTitle%",
35+
MaxFilenameLength = 50
36+
};
37+
38+
settings.Consumed = true;
39+
40+
PatreonCrawledUrl crawledUrl = new PatreonCrawledUrl
41+
{
42+
PostId = "123456",
43+
Title = "Test Post",
44+
PublishedAt = DateTime.Parse("07.07.2020 20:00:15"),
45+
Url = "https://www.patreon.com/media-u/Z0FBQUFBQmhXZDd3LXMwN0lJUFdVYTVIMEY1OGxzZTgwaFpQcW5TMk5WQVgxd2JVRFZvRXhjMjQ2V09oTW51eUpLQzIyOW1TdHRzYkY2Uk4yclAwX0VsSXBPMFZsNTBTcmZoaGx4OXJkR1Zham1CYl9fOWNVb3AzZGN1Wl9FMmNzcmIxc3hDek4xcHNuRV92LUVqQ0JESE4tcVBNYzlxYkRnWQ1=",
46+
Filename = "https://www.patreon.com/media-u/Z0FBQUFBQmhXZDd3a0xfckdEWmFrU0tjZHFUUkZfaDZ1OW92TjFVWFVDNk02c2FvS2FNczZxMS1rSVlaNUotX095dUNhdzJBSmYzMVpDV1luR1BYSXR6OVlZelpFOFFVektEcnpJT1plbElua2kwT1N2ZUMyU1NWaHV0eHQydWhnUXlmVWVLVDFYclBsSDBRaVJ3MDA5d2tzdDRZR3dtb3dBWQ1=",
47+
UrlType = PatreonCrawledUrlType.PostMedia
48+
};
49+
50+
PatreonCrawledUrlProcessor crawledUrlProcessor = new PatreonCrawledUrlProcessor(new PatreonRemoteFilenameRetriever());
51+
await crawledUrlProcessor.BeforeStart(settings);
52+
await crawledUrlProcessor.ProcessCrawledUrl(crawledUrl,
53+
Path.Combine(settings.DownloadDirectory, "UnitTesting"));
54+
55+
Assert.Equal(@"c:\downloads\UnitTesting\[123456] 2020-07-07 Test Post\media_https___www.patreon.com_media-u_Z0FBQUFBQmhX", crawledUrl.DownloadPath);
56+
}
57+
58+
[Fact]
59+
public async Task ProcessCrawledUrl_MediaFileNameTooLong_IsTruncatedWithExtension()
60+
{
61+
PatreonDownloaderSettings settings = new PatreonDownloaderSettings
62+
{
63+
CookieContainer = new CookieContainer(),
64+
DownloadDirectory = "c:\\downloads",
65+
MaxDownloadRetries = 10,
66+
OverwriteFiles = false,
67+
RemoteFileSizeNotAvailableAction = RemoteFileSizeNotAvailableAction.KeepExisting,
68+
RetryMultiplier = 1,
69+
SaveAvatarAndCover = true,
70+
SaveDescriptions = true,
71+
SaveEmbeds = true,
72+
SaveJson = true,
73+
UseSubDirectories = true,
74+
SubDirectoryPattern = "[%PostId%] %PublishedAt% %PostTitle%",
75+
MaxFilenameLength = 50
76+
};
77+
78+
settings.Consumed = true;
79+
80+
PatreonCrawledUrl crawledUrl = new PatreonCrawledUrl
81+
{
82+
PostId = "123456",
83+
Title = "Test Post",
84+
PublishedAt = DateTime.Parse("07.07.2020 20:00:15"),
85+
Url = "https://www.patreon.com/media-u/Z0FBQUFBQmhXZDd3LXMwN0lJUFdVYTVIMEY1OGxzZTgwaFpQcW5TMk5WQVgxd2JVRFZvRXhjMjQ2V09oTW51eUpLQzIyOW1TdHRzYkY2Uk4yclAwX0VsSXBPMFZsNTBTcmZoaGx4OXJkR1Zham1CYl9fOWNVb3AzZGN1Wl9FMmNzcmIxc3hDek4xcHNuRV92LUVqQ0JESE4tcVBNYzlxYkRnWQ1=",
86+
Filename = "E0OarAVlc0iipzgUC7JdvBCf9fgSmbwk3xRDjRGByTM24SuMl6HkY1DIdGfcvnZhbTb978AHonnwqWNzMPEWBRQp007ateP9ByhB.png",
87+
UrlType = PatreonCrawledUrlType.PostFile
88+
};
89+
90+
PatreonCrawledUrlProcessor crawledUrlProcessor = new PatreonCrawledUrlProcessor(new PatreonRemoteFilenameRetriever());
91+
await crawledUrlProcessor.BeforeStart(settings);
92+
await crawledUrlProcessor.ProcessCrawledUrl(crawledUrl,
93+
Path.Combine(settings.DownloadDirectory, "UnitTesting"));
94+
95+
Assert.Equal(@"c:\downloads\UnitTesting\[123456] 2020-07-07 Test Post\post_E0OarAVlc0iipzgUC7JdvBCf9fgSmbwk3xRDjRGByTM24.png", crawledUrl.DownloadPath);
96+
}
97+
}
98+
}

0 commit comments

Comments
 (0)