Skip to content

Commit cacc3c8

Browse files
AlexCSDevAlexCSDev
authored
Added captcha check, the browser will be now shown if captcha is detected. After captcha is solved the new cookies are written back into the WebDownloader, updated user agent for browser, updated PuppeteerSharp, browser viewport is no longer forced to 800x600 resolution, updated version to 0.10.2.0 (#113)
1 parent 3fd86e0 commit cacc3c8

10 files changed

Lines changed: 254 additions & 6 deletions

File tree

‎PatreonDownloader.App/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,5 @@
2929
// Build Number
3030
// Revision
3131
//
32-
[assembly: AssemblyVersion("0.10.1.2")]
33-
[assembly: AssemblyFileVersion("0.10.1.2")]
32+
[assembly: AssemblyVersion("0.10.2.0")]
33+
[assembly: AssemblyFileVersion("0.10.2.0")]

‎PatreonDownloader.Common/Interfaces/ICaptchaSolver.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Net;
5+
using System.Text;
6+
using System.Threading.Tasks;
7+
8+
namespace PatreonDownloader.Common.Interfaces
9+
{
10+
public interface ICaptchaSolver
11+
{
12+
Task<CookieContainer> SolveCaptcha(string url);
13+
}
14+
}

‎PatreonDownloader.Implementation/PatreonDownloader.Implementation.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
</ItemGroup>
1414

1515
<ItemGroup>
16+
<ProjectReference Include="..\PatreonDownloader.Common\PatreonDownloader.Common.csproj" />
17+
<ProjectReference Include="..\PatreonDownloader.PuppeteerEngine\PatreonDownloader.PuppeteerEngine.csproj" />
1618
<ProjectReference Include="..\submodules\UniversalDownloaderPlatform\UniversalDownloaderPlatform.DefaultImplementations\UniversalDownloaderPlatform.DefaultImplementations.csproj" />
1719
</ItemGroup>
1820

‎PatreonDownloader.Implementation/PatreonDownloaderModule.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Text;
4+
using Ninject;
45
using Ninject.Modules;
56
using PatreonDownloader.Engine;
67
using PatreonDownloader.Implementation.Interfaces;
78
using PatreonDownloader.Implementation.Models;
9+
using PatreonDownloader.PuppeteerEngine;
810
using UniversalDownloaderPlatform.Common.Interfaces;
911
using UniversalDownloaderPlatform.Common.Interfaces.Models;
1012
using UniversalDownloaderPlatform.Common.Interfaces.Plugins;
@@ -19,7 +21,7 @@ public class PatreonDownloaderModule : NinjectModule
1921
public override void Load()
2022
{
2123
Bind<IRemoteFileSizeChecker>().To<RemoteFileSizeChecker>().InSingletonScope();
22-
Bind<IWebDownloader>().To<WebDownloader>().InSingletonScope();
24+
Bind<IWebDownloader>().To<PatreonWebDownloader>().InSingletonScope();
2325
Bind<IRemoteFilenameRetriever>().To<PatreonRemoteFilenameRetriever>().InSingletonScope();
2426
Bind<ICrawlTargetInfoRetriever>().To<PatreonCrawlTargetInfoRetriever>().InSingletonScope();
2527
Bind<ICrawledUrlProcessor>().To<PatreonCrawledUrlProcessor>().InSingletonScope();

‎PatreonDownloader.Implementation/PatreonWebDownloader.cs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Net;
5+
using System.Text;
6+
using System.Threading.Tasks;
7+
using NLog;
8+
using PatreonDownloader.Common.Interfaces;
9+
using PatreonDownloader.PuppeteerEngine;
10+
using UniversalDownloaderPlatform.Common.Exceptions;
11+
using UniversalDownloaderPlatform.Common.Interfaces;
12+
using UniversalDownloaderPlatform.Common.Interfaces.Models;
13+
using UniversalDownloaderPlatform.DefaultImplementations;
14+
using UniversalDownloaderPlatform.DefaultImplementations.Interfaces;
15+
16+
namespace PatreonDownloader.Implementation
17+
{
18+
internal class PatreonWebDownloader : WebDownloader
19+
{
20+
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
21+
public PatreonWebDownloader(IRemoteFileSizeChecker remoteFileSizeChecker) : base(remoteFileSizeChecker)
22+
{
23+
24+
}
25+
26+
public override async Task DownloadFile(string url, string path, bool overwrite = false)
27+
{
28+
try
29+
{
30+
await base.DownloadFile(url, path, overwrite);
31+
}
32+
catch (DownloadException ex)
33+
{
34+
if (ex.StatusCode == HttpStatusCode.Forbidden && !string.IsNullOrWhiteSpace(ex.Response))
35+
{
36+
if (ex.Response.ToLowerInvariant().Contains("https://ct.captcha-delivery.com/c.js"))
37+
{
38+
if (! await SolveCaptchaAndUpdateCookies(url))
39+
throw;
40+
41+
await this.DownloadFile(url, path, overwrite);
42+
}
43+
}
44+
else
45+
{
46+
throw;
47+
}
48+
}
49+
}
50+
51+
public override async Task<string> DownloadString(string url)
52+
{
53+
try
54+
{
55+
return await base.DownloadString(url);
56+
}
57+
catch (DownloadException ex)
58+
{
59+
if (ex.StatusCode == HttpStatusCode.Forbidden && !string.IsNullOrWhiteSpace(ex.Response))
60+
{
61+
if (ex.Response.ToLowerInvariant().Contains("https://ct.captcha-delivery.com/c.js"))
62+
{
63+
if (!await SolveCaptchaAndUpdateCookies(url))
64+
throw;
65+
66+
return await this.DownloadString(url);
67+
}
68+
69+
throw;
70+
}
71+
else
72+
{
73+
throw;
74+
}
75+
}
76+
}
77+
78+
private async Task<bool> SolveCaptchaAndUpdateCookies(string url)
79+
{
80+
_logger.Warn("Captcha has been triggered, the browser window will be opened now. Please solve the captcha there.");
81+
82+
PuppeteerCaptchaSolver captchaSolver = new PuppeteerCaptchaSolver();
83+
CookieContainer cookies = await captchaSolver.SolveCaptcha(url);
84+
captchaSolver.Dispose();
85+
captchaSolver = null;
86+
87+
if (cookies == null)
88+
return false;
89+
90+
UpdateCookies(cookies);
91+
92+
return true;
93+
}
94+
}
95+
}

‎PatreonDownloader.PuppeteerEngine/PatreonDownloader.PuppeteerEngine.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<PackageReference Include="Ninject" Version="3.3.4" />
1010
<PackageReference Include="ninject.extensions.conventions" Version="3.3.0" />
1111
<PackageReference Include="NLog" Version="4.7.7" />
12-
<PackageReference Include="PuppeteerSharp" Version="2.0.4" />
12+
<PackageReference Include="PuppeteerSharp" Version="6.2.0" />
1313
</ItemGroup>
1414

1515
<ItemGroup>

‎PatreonDownloader.PuppeteerEngine/PuppeteerCaptchaSolver.cs‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
using System;
2+
using System.Diagnostics;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Net;
6+
using System.Threading.Tasks;
7+
using NLog;
8+
using PatreonDownloader.Common.Interfaces;
9+
using PatreonDownloader.PuppeteerEngine.Wrappers.Browser;
10+
using PuppeteerSharp;
11+
12+
namespace PatreonDownloader.PuppeteerEngine
13+
{
14+
public class PuppeteerCaptchaSolver : ICaptchaSolver, IDisposable
15+
{
16+
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
17+
private IPuppeteerEngine _puppeteerEngine;
18+
19+
/// <summary>
20+
/// Create new instance of PuppeteerCaptchaSolver using internal browser
21+
/// </summary>
22+
public PuppeteerCaptchaSolver()
23+
{
24+
_puppeteerEngine = new PuppeteerEngine();
25+
}
26+
27+
private async Task<IWebBrowser> RestartBrowser(bool headless)
28+
{
29+
await _puppeteerEngine.CloseBrowser();
30+
await Task.Delay(1000); //safety first
31+
32+
_puppeteerEngine = new PuppeteerEngine(headless);
33+
return await _puppeteerEngine.GetBrowser();
34+
}
35+
36+
public async Task<CookieContainer> SolveCaptcha(string url)
37+
{
38+
try
39+
{
40+
CookieContainer cookieContainer = new CookieContainer();
41+
42+
43+
_logger.Debug("Calling captcha check");
44+
try
45+
{
46+
await CaptchaCheck(url);
47+
}
48+
catch (Exception ex)
49+
{
50+
_logger.Fatal($"Captcha solver error: {ex.Message}");
51+
return null;
52+
}
53+
54+
_logger.Debug("Retrieving browser");
55+
IWebBrowser browser = await _puppeteerEngine.GetBrowser();
56+
57+
_logger.Debug("Retrieving cookies");
58+
IWebPage page = await browser.NewPageAsync();
59+
await page.GoToAsync("https://www.patreon.com/home");
60+
61+
CookieParam[] browserCookies = await page.GetCookiesAsync();
62+
63+
if (browserCookies != null && browserCookies.Length > 0)
64+
{
65+
foreach (CookieParam browserCookie in browserCookies)
66+
{
67+
_logger.Debug($"Adding cookie: {browserCookie.Name}");
68+
Cookie cookie = new Cookie(browserCookie.Name, browserCookie.Value, browserCookie.Path, browserCookie.Domain);
69+
cookieContainer.Add(cookie);
70+
}
71+
}
72+
else
73+
{
74+
_logger.Fatal("No cookies were extracted from browser");
75+
return null;
76+
}
77+
78+
await page.CloseAsync();
79+
80+
return cookieContainer;
81+
}
82+
catch (TimeoutException ex)
83+
{
84+
_logger.Fatal($"Internal operation timed out. Exception: {ex}");
85+
return null;
86+
}
87+
}
88+
89+
private async Task CaptchaCheck(string url)
90+
{
91+
_logger.Debug("Retrieving browser");
92+
IWebBrowser browser = await _puppeteerEngine.GetBrowser();
93+
94+
IWebPage page = null;
95+
bool passedCaptcha = false;
96+
do
97+
{
98+
if (page == null || page.IsClosed)
99+
page = await browser.NewPageAsync();
100+
101+
_logger.Debug("Checking if captcha is returned");
102+
IWebResponse response = await page.GoToAsync(url);
103+
if (response.Status == HttpStatusCode.Forbidden)
104+
{
105+
_logger.Debug("Got 403 forbidden (possibly captcha), opening page");
106+
if (_puppeteerEngine.IsHeadless)
107+
{
108+
_logger.Debug("Puppeteer is in headless mode, restarting in full mode");
109+
browser = await RestartBrowser(false);
110+
page = await browser.NewPageAsync();
111+
}
112+
113+
await page.GoToAsync(url);
114+
115+
await page.WaitForRequestAsync(request => request.Url == url);
116+
}
117+
else
118+
{
119+
_logger.Debug("Captcha was not returned, done");
120+
121+
passedCaptcha = true;
122+
}
123+
} while (!passedCaptcha);
124+
125+
await page.CloseAsync();
126+
}
127+
128+
public void Dispose()
129+
{
130+
_puppeteerEngine?.Dispose();
131+
}
132+
}
133+
}

‎PatreonDownloader.PuppeteerEngine/PuppeteerEngine.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,9 @@ private async Task<IWebBrowser> StartLocalBrowser()
134134
//Devtools = true,
135135
Headless = _headless,
136136
UserDataDir = Path.Combine(Environment.CurrentDirectory, "chromedata"),
137+
DefaultViewport = null,
137138
//Headless mode changes user agent so we need to force it to use "real" user agent
138-
Args = new[] { "--user-agent=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3882.0 Safari/537.36\"" }
139+
Args = new[] { "--user-agent=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.0 Safari/537.36\"" }
139140
});
140141

141142
_logger.Debug("Opening new page");

‎PatreonDownloader.PuppeteerEngine/PuppeteerEngineModule.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public override void Load()
1010
{
1111
Bind<IPuppeteerEngine>().To<PuppeteerEngine>().InSingletonScope();
1212
Bind<ICookieRetriever>().To<PuppeteerCookieRetriever>();
13+
Bind<ICaptchaSolver>().To<PuppeteerCaptchaSolver>();
1314
Bind<IWebBrowser>().To<WebBrowser>();
1415
Bind<IWebPage>().To<WebPage>();
1516
Bind<IWebRequest>().To<WebRequest>();

‎submodules/UniversalDownloaderPlatform‎

0 commit comments

Comments
 (0)