From d4db47311d023a8a5f4b697eb016c8982dcc7c32 Mon Sep 17 00:00:00 2001 From: Ivan Kara Date: Thu, 29 Jun 2023 14:33:36 +0700 Subject: [PATCH 1/2] Added flaresolverr response storage --- src/FlareSolverrSharp/ClearanceHandler.cs | 83 +++++++++++++++---- .../FlaresolverrResponseStorage.cs | 24 ++++++ 2 files changed, 91 insertions(+), 16 deletions(-) create mode 100644 src/FlareSolverrSharp/FlaresolverrResponseStorage.cs diff --git a/src/FlareSolverrSharp/ClearanceHandler.cs b/src/FlareSolverrSharp/ClearanceHandler.cs index e166383..62a2c43 100644 --- a/src/FlareSolverrSharp/ClearanceHandler.cs +++ b/src/FlareSolverrSharp/ClearanceHandler.cs @@ -20,6 +20,7 @@ public class ClearanceHandler : DelegatingHandler { private readonly HttpClient _client; private readonly string _flareSolverrApiUrl; + private readonly IFlaresolverrResponseStorage _responseStorage; private FlareSolverr _flareSolverr; private string _userAgent; @@ -42,6 +43,17 @@ public class ClearanceHandler : DelegatingHandler /// FlareSolverr API URL. If null or empty it will detect the challenges, but /// they will not be solved. Example: "http://localhost:8191/" public ClearanceHandler(string flareSolverrApiUrl) + : this(flareSolverrApiUrl, new DefaultFlaresolverrResponseStorage()) + { + } + + /// + /// Creates a new instance of the . + /// + /// FlareSolverr API URL. If null or empty it will detect the challenges, but + /// they will not be solved. Example: "http://localhost:8191/" + /// Storage to persist challenge responses. + public ClearanceHandler(string flareSolverrApiUrl, IFlaresolverrResponseStorage responseStorage) : base(new HttpClientHandler()) { // Validate URI @@ -50,6 +62,7 @@ public ClearanceHandler(string flareSolverrApiUrl) throw new FlareSolverrException("FlareSolverr URL is malformed: " + flareSolverrApiUrl); _flareSolverrApiUrl = flareSolverrApiUrl; + _responseStorage = responseStorage; _client = new HttpClient(new HttpClientHandler { @@ -84,34 +97,72 @@ protected override async Task SendAsync(HttpRequestMessage var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); // Detect if there is a challenge in the response - if (ChallengeDetector.IsClearanceRequired(response)) + if (!ChallengeDetector.IsClearanceRequired(response)) { - if (_flareSolverr == null) - throw new FlareSolverrException("Challenge detected but FlareSolverr is not configured"); + return response; + } - // Resolve the challenge using FlareSolverr API - var flareSolverrResponse = await _flareSolverr.Solve(request); + if (_flareSolverr == null) + throw new FlareSolverrException("Challenge detected but FlareSolverr is not configured"); + + // Check if saved response exists. + var flareSolverrResponse = await _responseStorage.LoadAsync(); + bool isNewResponse = true; - // Save the FlareSolverr User-Agent for the following requests - var flareSolverUserAgent = flareSolverrResponse.Solution.UserAgent; - if (flareSolverUserAgent != null && !flareSolverUserAgent.Equals(request.Headers.UserAgent.ToString())) + if (flareSolverrResponse == null) + { + // Resolve the challenge using FlareSolverr API + flareSolverrResponse = await _flareSolverr.Solve(request); + } + else + { + // Set user agent + if (flareSolverrResponse.Solution.UserAgent != null && + !flareSolverrResponse.Solution.UserAgent.Equals(request.Headers.UserAgent.ToString())) { - _userAgent = flareSolverUserAgent; - - // Set the User-Agent if required + _userAgent = flareSolverrResponse.Solution.UserAgent; SetUserAgentHeader(request); } - // Change the cookies in the original request with the cookies provided by FlareSolverr + // Retry request with saved response InjectCookies(request, flareSolverrResponse); response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); - // Detect if there is a challenge in the response if (ChallengeDetector.IsClearanceRequired(response)) - throw new FlareSolverrException("The cookies provided by FlareSolverr are not valid"); + { + // Resolve the challenge using FlareSolverr API + flareSolverrResponse = await _flareSolverr.Solve(request); + } + else + { + isNewResponse = false; + } + } + + // Save the FlareSolverr User-Agent for the following requests + var flareSolverUserAgent = flareSolverrResponse.Solution.UserAgent; + if (flareSolverUserAgent != null && !flareSolverUserAgent.Equals(request.Headers.UserAgent.ToString())) + { + _userAgent = flareSolverUserAgent; - // Add the "Set-Cookie" header in the response with the cookies provided by FlareSolverr - InjectSetCookieHeader(response, flareSolverrResponse); + // Set the User-Agent if required + SetUserAgentHeader(request); + } + + // Change the cookies in the original request with the cookies provided by FlareSolverr + InjectCookies(request, flareSolverrResponse); + response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + + // Detect if there is a challenge in the response + if (ChallengeDetector.IsClearanceRequired(response)) + throw new FlareSolverrException("The cookies provided by FlareSolverr are not valid"); + + // Add the "Set-Cookie" header in the response with the cookies provided by FlareSolverr + InjectSetCookieHeader(response, flareSolverrResponse); + + if (isNewResponse) + { + await _responseStorage.SaveAsync(flareSolverrResponse); } return response; diff --git a/src/FlareSolverrSharp/FlaresolverrResponseStorage.cs b/src/FlareSolverrSharp/FlaresolverrResponseStorage.cs new file mode 100644 index 0000000..90703c2 --- /dev/null +++ b/src/FlareSolverrSharp/FlaresolverrResponseStorage.cs @@ -0,0 +1,24 @@ +using System.Threading.Tasks; +using FlareSolverrSharp.Types; + +namespace FlareSolverrSharp +{ + public interface IFlaresolverrResponseStorage + { + Task SaveAsync(FlareSolverrResponse result); + Task LoadAsync(); + } + + public class DefaultFlaresolverrResponseStorage : IFlaresolverrResponseStorage + { + public Task LoadAsync() + { + return Task.FromResult(null); + } + + public Task SaveAsync(FlareSolverrResponse result) + { + return Task.CompletedTask; + } + } +} From 02c9fb012293d3802e09c526665e0ab2a8294986 Mon Sep 17 00:00:00 2001 From: Ivan Kara Date: Thu, 29 Jun 2023 16:19:30 +0700 Subject: [PATCH 2/2] Better code --- src/FlareSolverrSharp/ClearanceHandler.cs | 28 ++++++++--------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/src/FlareSolverrSharp/ClearanceHandler.cs b/src/FlareSolverrSharp/ClearanceHandler.cs index 62a2c43..a1e37f8 100644 --- a/src/FlareSolverrSharp/ClearanceHandler.cs +++ b/src/FlareSolverrSharp/ClearanceHandler.cs @@ -107,14 +107,8 @@ protected override async Task SendAsync(HttpRequestMessage // Check if saved response exists. var flareSolverrResponse = await _responseStorage.LoadAsync(); - bool isNewResponse = true; - if (flareSolverrResponse == null) - { - // Resolve the challenge using FlareSolverr API - flareSolverrResponse = await _flareSolverr.Solve(request); - } - else + if (flareSolverrResponse != null) { // Set user agent if (flareSolverrResponse.Solution.UserAgent != null && @@ -128,17 +122,17 @@ protected override async Task SendAsync(HttpRequestMessage InjectCookies(request, flareSolverrResponse); response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); - if (ChallengeDetector.IsClearanceRequired(response)) - { - // Resolve the challenge using FlareSolverr API - flareSolverrResponse = await _flareSolverr.Solve(request); - } - else + if (!ChallengeDetector.IsClearanceRequired(response)) { - isNewResponse = false; + // Success with saved response. + InjectSetCookieHeader(response, flareSolverrResponse); + return response; } } + // Resolve the challenge using FlareSolverr API + flareSolverrResponse = await _flareSolverr.Solve(request); + // Save the FlareSolverr User-Agent for the following requests var flareSolverUserAgent = flareSolverrResponse.Solution.UserAgent; if (flareSolverUserAgent != null && !flareSolverUserAgent.Equals(request.Headers.UserAgent.ToString())) @@ -159,11 +153,7 @@ protected override async Task SendAsync(HttpRequestMessage // Add the "Set-Cookie" header in the response with the cookies provided by FlareSolverr InjectSetCookieHeader(response, flareSolverrResponse); - - if (isNewResponse) - { - await _responseStorage.SaveAsync(flareSolverrResponse); - } + await _responseStorage.SaveAsync(flareSolverrResponse); return response; }