Skip to content

Commit abd91f6

Browse files
AlexCSDevAlexCSDev
committed
Updated version to 0.10.3.0, added --filenames-fallback-to-content-type command line option, fixed youtube and imgur urls not being ignored correctly, cleaned up the code, updated UDP to improve browser mimicking
1 parent 3b0e571 commit abd91f6

15 files changed

Lines changed: 104 additions & 55 deletions

File tree

‎PatreonDownloader.App/Models/CommandLineOptions.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,7 @@ class CommandLineOptions
3939
public string SubDirectoryPattern { get; set; }
4040
[Option("max-filename-length", Required = false, HelpText = "All names of downloaded files will be truncated so their length won't be more than specified value (excluding file extension)", Default = 100)]
4141
public int MaxFilenameLength { get; set; }
42+
[Option("filenames-fallback-to-content-type", Required = false, HelpText = "Fallback to using filename generated from url hash if the server returns file content type (extension) and all other methods have failed. Use with caution, this might result in unwanted files being created or the same files being downloaded on every run under different names.", Default = false)]
43+
public bool FilenamesFallbackToContentType { get; set; }
4244
}
4345
}

‎PatreonDownloader.App/Program.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,8 @@ private static async Task<PatreonDownloaderSettings> InitializeSettings(CommandL
191191
RemoteFileSizeNotAvailableAction = commandLineOptions.NoRemoteSizeAction,
192192
UseSubDirectories = commandLineOptions.UseSubDirectories,
193193
SubDirectoryPattern = commandLineOptions.SubDirectoryPattern,
194-
MaxFilenameLength = commandLineOptions.MaxFilenameLength
194+
MaxFilenameLength = commandLineOptions.MaxFilenameLength,
195+
FallbackToContentTypeFilenames = commandLineOptions.FilenamesFallbackToContentType
195196
};
196197

197198
return settings;

‎PatreonDownloader.App/Properties/AssemblyInfo.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
[assembly: AssemblyConfiguration("")]
1111
[assembly: AssemblyCompany("")]
1212
[assembly: AssemblyProduct("Patreon Downloader")]
13-
[assembly: AssemblyCopyright("Copyright 2019-2021 Aleksey Tsutsey & Contributors")]
13+
[assembly: AssemblyCopyright("Copyright 2019-2022 Aleksey Tsutsey & Contributors")]
1414
[assembly: AssemblyTrademark("")]
1515
[assembly: AssemblyCulture("")]
1616

@@ -29,5 +29,5 @@
2929
// Build Number
3030
// Revision
3131
//
32-
[assembly: AssemblyVersion("0.10.2.0")]
33-
[assembly: AssemblyFileVersion("0.10.2.0")]
32+
[assembly: AssemblyVersion("0.10.3.0")]
33+
[assembly: AssemblyFileVersion("0.10.3.0")]

‎PatreonDownloader.App/settings.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"UrlBlackList": "patreon.com/posts/|tmblr.co/|t.umblr.com/redirect|mailto:|postybirb.com|picarto.tv|deviantart.com|https://twitter.com|https://steamcommunity.com|http://www.furaffinity.net|https://e621.net/post/show|https://e621.net/posts/|trello.com|https://smutba.se|https://sfmlab.com|http://fav.me|https://inkbunny.net|https://www.pixiv.net/"
2+
"UrlBlackList": "patreon.com/posts/|tmblr.co/|t.umblr.com/redirect|mailto:|postybirb.com|picarto.tv|deviantart.com|https://twitter.com|https://steamcommunity.com|http://www.furaffinity.net|https://e621.net/post/show|https://e621.net/posts/|trello.com|https://smutba.se|https://sfmlab.com|http://fav.me|https://inkbunny.net|https://www.pixiv.net/|pixiv.me"
33
}

‎PatreonDownloader.Common/Properties/AssemblyInfo.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
[assembly: AssemblyConfiguration("")]
1111
[assembly: AssemblyCompany("")]
1212
[assembly: AssemblyProduct("Patreon Downloader")]
13-
[assembly: AssemblyCopyright("Copyright 2019-2021 Aleksey Tsutsey & Contributors")]
13+
[assembly: AssemblyCopyright("Copyright 2019-2022 Aleksey Tsutsey & Contributors")]
1414
[assembly: AssemblyTrademark("")]
1515
[assembly: AssemblyCulture("")]
1616

‎PatreonDownloader.Implementation/Helpers/HashHelper.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Security.Cryptography;
5+
using System.Text;
6+
using System.Threading.Tasks;
7+
8+
namespace PatreonDownloader.Implementation.Helpers
9+
{
10+
internal static class HashHelper
11+
{
12+
//https://www.c-sharpcorner.com/article/compute-sha256-hash-in-c-sharp/
13+
public static string ComputeSha256Hash(string rawData)
14+
{
15+
// Create a SHA256
16+
using (SHA256 sha256Hash = SHA256.Create())
17+
{
18+
// ComputeHash - returns byte array
19+
byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(rawData));
20+
21+
// Convert byte array to a string
22+
StringBuilder builder = new StringBuilder();
23+
for (int i = 0; i < bytes.Length; i++)
24+
{
25+
builder.Append(bytes[i].ToString("x2"));
26+
}
27+
return builder.ToString();
28+
}
29+
}
30+
}
31+
}

‎PatreonDownloader.Implementation/Interfaces/IRemoteFilenameRetriever.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
using System.Threading.Tasks;
2+
using UniversalDownloaderPlatform.Common.Interfaces.Models;
23

34
namespace PatreonDownloader.Implementation.Interfaces
45
{
56
interface IRemoteFilenameRetriever
67
{
8+
/// <summary>
9+
/// Initialization function, called on every PatreonDownloader.Download call
10+
/// </summary>
11+
/// <returns></returns>
12+
Task BeforeStart(IUniversalDownloaderPlatformSettings settings);
713
Task<string> RetrieveRemoteFileName(string url);
814
}
915
}

‎PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class PatreonDownloaderSettings : UniversalDownloaderPlatformSettings
1818
private bool _useSubDirectories;
1919
private string _subDirectoryPattern;
2020
private int _maxFilenameLength;
21+
private bool _fallbackToContentTypeFilenames;
2122

2223
public bool SaveDescriptions
2324
{
@@ -79,6 +80,15 @@ public int MaxFilenameLength
7980
set => ConsumableSetter.Set(Consumed, ref _maxFilenameLength, value);
8081
}
8182

83+
/// <summary>
84+
/// Fallback to using sha256 hash and Content-Type for filenames if Content-Disposition fails
85+
/// </summary>
86+
public bool FallbackToContentTypeFilenames
87+
{
88+
get => _fallbackToContentTypeFilenames;
89+
set => ConsumableSetter.Set(Consumed, ref _fallbackToContentTypeFilenames, value);
90+
}
91+
8292
public PatreonDownloaderSettings()
8393
{
8494
_saveDescriptions = true;
@@ -88,12 +98,13 @@ public PatreonDownloaderSettings()
8898
_downloadDirectory = null;
8999
_useSubDirectories = false;
90100
_subDirectoryPattern = "[%PostId%] %PublishedAt% %PostTitle%";
101+
_fallbackToContentTypeFilenames = false;
91102
_maxFilenameLength = 100;
92103
}
93104

94105
public override string ToString()
95106
{
96-
return $"SaveDescriptions={_saveDescriptions},SaveEmbeds={_saveEmbeds},SaveJson={_saveJson},SaveAvatarAndCover={_saveAvatarAndCover},DownloadDirectory={_downloadDirectory},OverwriteFiles={base.OverwriteFiles},UseSubDirectories={_useSubDirectories},MaxFilenameLength={_maxFilenameLength}";
107+
return $"SaveDescriptions={_saveDescriptions},SaveEmbeds={_saveEmbeds},SaveJson={_saveJson},SaveAvatarAndCover={_saveAvatarAndCover},DownloadDirectory={_downloadDirectory},OverwriteFiles={base.OverwriteFiles},UseSubDirectories={_useSubDirectories},MaxFilenameLength={_maxFilenameLength},FallbackToContentTypeFilenames={_fallbackToContentTypeFilenames}";
97108
}
98109
}
99110
}

‎PatreonDownloader.Implementation/PatreonCrawledUrlProcessor.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ public async Task BeforeStart(IUniversalDownloaderPlatformSettings settings)
5353
{
5454
_fileCountDict = new Dictionary<string, int>();
5555
_patreonDownloaderSettings = (PatreonDownloaderSettings) settings;
56+
await _remoteFilenameRetriever.BeforeStart(settings);
5657
}
5758

5859
public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl, string downloadDirectory)
@@ -87,11 +88,13 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl, string down
8788
{
8889
//TODO: YOUTUBE SUPPORT?
8990
_logger.Fatal($"[{crawledUrl.PostId}] [NOT SUPPORTED] YOUTUBE link found: {crawledUrl.Url}");
91+
return false;
9092
}
9193
else if (crawledUrl.Url.IndexOf("imgur.com/", StringComparison.Ordinal) != -1)
9294
{
9395
//TODO: IMGUR SUPPORT
9496
_logger.Fatal($"[{crawledUrl.PostId}] [NOT SUPPORTED] IMGUR link found: {crawledUrl.Url}");
97+
return false;
9598
}
9699

97100
string filename = crawledUrl.Filename;

‎PatreonDownloader.Implementation/PatreonDefaultPlugin.cs‎

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -23,32 +23,18 @@ namespace PatreonDownloader.Engine
2323
internal sealed class PatreonDefaultPlugin : IPlugin
2424
{
2525
private IWebDownloader _webDownloader;
26-
private IRemoteFilenameRetriever _remoteFilenameRetriever;
2726

2827
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
29-
private Dictionary<string, int> _fileCountDict; //file counter for duplicate check
3028
private bool _overwriteFiles;
3129

32-
private static readonly Regex _fileIdRegex; //Regex used to retrieve file id from its url
33-
3430
public string Name => "Default plugin";
3531

3632
public string Author => "Aleksey Tsutsey";
3733
public string ContactInformation => "https://github.com/Megalan/PatreonDownloader";
3834

39-
static PatreonDefaultPlugin()
40-
{
41-
_fileIdRegex =
42-
new Regex(
43-
"https:\\/\\/(.+)\\.patreonusercontent\\.com\\/(.+)\\/(.+)\\/patreon-media\\/p\\/post\\/([0-9]+)\\/([a-z0-9]+)",
44-
RegexOptions.Compiled | RegexOptions.IgnoreCase);
45-
}
46-
47-
public PatreonDefaultPlugin(IWebDownloader webDownloader, IRemoteFilenameRetriever remoteFilenameRetriever)
35+
public PatreonDefaultPlugin(IWebDownloader webDownloader)
4836
{
4937
_webDownloader = webDownloader ?? throw new ArgumentNullException(nameof(webDownloader));
50-
_remoteFilenameRetriever = remoteFilenameRetriever ??
51-
throw new ArgumentNullException(nameof(remoteFilenameRetriever));
5238
}
5339

5440
public async Task<bool> IsSupportedUrl(string url)
@@ -72,7 +58,6 @@ public async Task Download(ICrawledUrl crawledUrl, string downloadDirectory)
7258
public async Task BeforeStart(bool overwriteFiles)
7359
{
7460
_overwriteFiles = overwriteFiles;
75-
_fileCountDict = new Dictionary<string, int>();
7661
}
7762

7863
public async Task<List<string>> ExtractSupportedUrls(string htmlContents)
@@ -133,21 +118,6 @@ private bool IsAllowedUrl(string url)
133118
return false;
134119
}
135120

136-
if (url.IndexOf("youtube.com/watch?v=", StringComparison.Ordinal) != -1 ||
137-
url.IndexOf("youtu.be/", StringComparison.Ordinal) != -1)
138-
{
139-
//TODO: YOUTUBE SUPPORT?
140-
_logger.Fatal($"[NOT SUPPORTED] YOUTUBE link found: {url}");
141-
return false;
142-
}
143-
144-
if (url.IndexOf("imgur.com/", StringComparison.Ordinal) != -1)
145-
{
146-
//TODO: IMGUR SUPPORT
147-
_logger.Fatal($"[NOT SUPPORTED] IMGUR link found: {url}");
148-
return false;
149-
}
150-
151121
return true;
152122
}
153123
}

0 commit comments

Comments
 (0)