diff --git a/src/FlareSolverrSharp/ClearanceHandler.cs b/src/FlareSolverrSharp/ClearanceHandler.cs index e166383..a1e37f8 100644 --- a/src/FlareSolverrSharp/ClearanceHandler.cs +++ b/src/FlareSolverrSharp/ClearanceHandler.cs @@ -20,6 +20,7 @@ public class ClearanceHandler : DelegatingHandler { private readonly HttpClient _client; private readonly string _flareSolverrApiUrl; + private readonly IFlaresolverrResponseStorage _responseStorage; private FlareSolverr _flareSolverr; private string _userAgent; @@ -42,6 +43,17 @@ public class ClearanceHandler : DelegatingHandler /// FlareSolverr API URL. If null or empty it will detect the challenges, but /// they will not be solved. Example: "http://localhost:8191/" public ClearanceHandler(string flareSolverrApiUrl) + : this(flareSolverrApiUrl, new DefaultFlaresolverrResponseStorage()) + { + } + + /// + /// Creates a new instance of the . + /// + /// FlareSolverr API URL. If null or empty it will detect the challenges, but + /// they will not be solved. Example: "http://localhost:8191/" + /// Storage to persist challenge responses. + public ClearanceHandler(string flareSolverrApiUrl, IFlaresolverrResponseStorage responseStorage) : base(new HttpClientHandler()) { // Validate URI @@ -50,6 +62,7 @@ public ClearanceHandler(string flareSolverrApiUrl) throw new FlareSolverrException("FlareSolverr URL is malformed: " + flareSolverrApiUrl); _flareSolverrApiUrl = flareSolverrApiUrl; + _responseStorage = responseStorage; _client = new HttpClient(new HttpClientHandler { @@ -84,36 +97,64 @@ protected override async Task SendAsync(HttpRequestMessage var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); // Detect if there is a challenge in the response - if (ChallengeDetector.IsClearanceRequired(response)) + if (!ChallengeDetector.IsClearanceRequired(response)) { - if (_flareSolverr == null) - throw new FlareSolverrException("Challenge detected but FlareSolverr is not configured"); + return response; + } - // Resolve the challenge using FlareSolverr API - var flareSolverrResponse = await _flareSolverr.Solve(request); + if (_flareSolverr == null) + throw new FlareSolverrException("Challenge detected but FlareSolverr is not configured"); + + // Check if saved response exists. + var flareSolverrResponse = await _responseStorage.LoadAsync(); - // Save the FlareSolverr User-Agent for the following requests - var flareSolverUserAgent = flareSolverrResponse.Solution.UserAgent; - if (flareSolverUserAgent != null && !flareSolverUserAgent.Equals(request.Headers.UserAgent.ToString())) + if (flareSolverrResponse != null) + { + // Set user agent + if (flareSolverrResponse.Solution.UserAgent != null && + !flareSolverrResponse.Solution.UserAgent.Equals(request.Headers.UserAgent.ToString())) { - _userAgent = flareSolverUserAgent; - - // Set the User-Agent if required + _userAgent = flareSolverrResponse.Solution.UserAgent; SetUserAgentHeader(request); } - // Change the cookies in the original request with the cookies provided by FlareSolverr + // Retry request with saved response InjectCookies(request, flareSolverrResponse); response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); - // Detect if there is a challenge in the response - if (ChallengeDetector.IsClearanceRequired(response)) - throw new FlareSolverrException("The cookies provided by FlareSolverr are not valid"); + if (!ChallengeDetector.IsClearanceRequired(response)) + { + // Success with saved response. + InjectSetCookieHeader(response, flareSolverrResponse); + return response; + } + } + + // Resolve the challenge using FlareSolverr API + flareSolverrResponse = await _flareSolverr.Solve(request); + + // Save the FlareSolverr User-Agent for the following requests + var flareSolverUserAgent = flareSolverrResponse.Solution.UserAgent; + if (flareSolverUserAgent != null && !flareSolverUserAgent.Equals(request.Headers.UserAgent.ToString())) + { + _userAgent = flareSolverUserAgent; - // Add the "Set-Cookie" header in the response with the cookies provided by FlareSolverr - InjectSetCookieHeader(response, flareSolverrResponse); + // Set the User-Agent if required + SetUserAgentHeader(request); } + // Change the cookies in the original request with the cookies provided by FlareSolverr + InjectCookies(request, flareSolverrResponse); + response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + + // Detect if there is a challenge in the response + if (ChallengeDetector.IsClearanceRequired(response)) + throw new FlareSolverrException("The cookies provided by FlareSolverr are not valid"); + + // Add the "Set-Cookie" header in the response with the cookies provided by FlareSolverr + InjectSetCookieHeader(response, flareSolverrResponse); + await _responseStorage.SaveAsync(flareSolverrResponse); + return response; } diff --git a/src/FlareSolverrSharp/FlareSolverrSharp.csproj b/src/FlareSolverrSharp/FlareSolverrSharp.csproj index c51af3f..8f98d59 100644 --- a/src/FlareSolverrSharp/FlareSolverrSharp.csproj +++ b/src/FlareSolverrSharp/FlareSolverrSharp.csproj @@ -1,7 +1,9 @@  - netstandard1.3 + netstandard2.0 + latest + enable FlareSolverrSharp FlareSolverrSharp 3.0.5 @@ -16,7 +18,7 @@ - + diff --git a/src/FlareSolverrSharp/FlaresolverrResponseStorage.cs b/src/FlareSolverrSharp/FlaresolverrResponseStorage.cs new file mode 100644 index 0000000..1850222 --- /dev/null +++ b/src/FlareSolverrSharp/FlaresolverrResponseStorage.cs @@ -0,0 +1,24 @@ +using System.Threading.Tasks; +using FlareSolverrSharp.Types; + +namespace FlareSolverrSharp +{ + public interface IFlaresolverrResponseStorage + { + Task SaveAsync(FlareSolverrResponse result); + Task LoadAsync(); + } + + public class DefaultFlaresolverrResponseStorage : IFlaresolverrResponseStorage + { + public Task LoadAsync() + { + return Task.FromResult(null); + } + + public Task SaveAsync(FlareSolverrResponse result) + { + return Task.CompletedTask; + } + } +} diff --git a/src/FlareSolverrSharp/Solvers/FlareSolverr.cs b/src/FlareSolverrSharp/Solvers/FlareSolverr.cs index f4e6628..3716ecb 100644 --- a/src/FlareSolverrSharp/Solvers/FlareSolverr.cs +++ b/src/FlareSolverrSharp/Solvers/FlareSolverr.cs @@ -2,11 +2,11 @@ using System.Net; using System.Net.Http; using System.Text; +using System.Text.Json; using System.Threading.Tasks; using FlareSolverrSharp.Exceptions; using FlareSolverrSharp.Types; using FlareSolverrSharp.Utilities; -using Newtonsoft.Json; namespace FlareSolverrSharp.Solvers { @@ -102,7 +102,9 @@ await Locker.LockAsync(async () => var resContent = await response.Content.ReadAsStringAsync(); try { - result = JsonConvert.DeserializeObject(resContent); + result = JsonSerializer.Deserialize( + resContent, + FlareSolverrContext.Default.FlareSolverrResponse); } catch (Exception) { @@ -160,10 +162,9 @@ private FlareSolverrRequestProxy GetProxy() private HttpContent GetSolverRequestContent(FlareSolverrRequest request) { - var payload = JsonConvert.SerializeObject(request, new JsonSerializerSettings - { - NullValueHandling = NullValueHandling.Ignore - }); + var payload = JsonSerializer.Serialize( + request, + FlareSolverrContext.Default.FlareSolverrRequest); HttpContent content = new StringContent(payload, Encoding.UTF8, "application/json"); return content; } diff --git a/src/FlareSolverrSharp/Solvers/FlareSolverrContext.cs b/src/FlareSolverrSharp/Solvers/FlareSolverrContext.cs new file mode 100644 index 0000000..d798168 --- /dev/null +++ b/src/FlareSolverrSharp/Solvers/FlareSolverrContext.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; +using FlareSolverrSharp.Types; + +namespace FlareSolverrSharp.Solvers +{ + [JsonSerializable(typeof(FlareSolverrRequest))] + [JsonSerializable(typeof(FlareSolverrResponse))] + [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] + public partial class FlareSolverrContext : JsonSerializerContext + { + } +} \ No newline at end of file diff --git a/src/FlareSolverrSharp/Types/FlareSolverrRequest.cs b/src/FlareSolverrSharp/Types/FlareSolverrRequest.cs index 49bd18b..a292382 100644 --- a/src/FlareSolverrSharp/Types/FlareSolverrRequest.cs +++ b/src/FlareSolverrSharp/Types/FlareSolverrRequest.cs @@ -1,19 +1,21 @@ -using Newtonsoft.Json; +using System.Text.Json.Serialization; namespace FlareSolverrSharp.Types { + [JsonDerivedType(typeof(FlareSolverrRequestGet))] + [JsonDerivedType(typeof(FlareSolverrRequestPost))] public class FlareSolverrRequest { - [JsonProperty("cmd")] - public string Cmd; + [JsonPropertyName("cmd")] + public string Cmd { get; set; } - [JsonProperty("url")] - public string Url; + [JsonPropertyName("url")] + public string Url { get; set; } - [JsonProperty("session")] - public string Session; + [JsonPropertyName("session")] + public string Session { get; set; } - [JsonProperty("proxy")] - public FlareSolverrRequestProxy Proxy; + [JsonPropertyName("proxy")] + public FlareSolverrRequestProxy Proxy { get; set; } } } diff --git a/src/FlareSolverrSharp/Types/FlareSolverrRequestGet.cs b/src/FlareSolverrSharp/Types/FlareSolverrRequestGet.cs index 41b520b..b73d33c 100644 --- a/src/FlareSolverrSharp/Types/FlareSolverrRequestGet.cs +++ b/src/FlareSolverrSharp/Types/FlareSolverrRequestGet.cs @@ -1,10 +1,10 @@ -using Newtonsoft.Json; +using System.Text.Json.Serialization; namespace FlareSolverrSharp.Types { public class FlareSolverrRequestGet : FlareSolverrRequest { - [JsonProperty("maxTimeout")] - public int MaxTimeout; + [JsonPropertyName("maxTimeout")] + public int MaxTimeout { get; set; } } } \ No newline at end of file diff --git a/src/FlareSolverrSharp/Types/FlareSolverrRequestPost.cs b/src/FlareSolverrSharp/Types/FlareSolverrRequestPost.cs index 932c5f1..b64cd20 100644 --- a/src/FlareSolverrSharp/Types/FlareSolverrRequestPost.cs +++ b/src/FlareSolverrSharp/Types/FlareSolverrRequestPost.cs @@ -1,13 +1,13 @@ -using Newtonsoft.Json; +using System.Text.Json.Serialization; namespace FlareSolverrSharp.Types { public class FlareSolverrRequestPost : FlareSolverrRequest { - [JsonProperty("postData")] - public string PostData; + [JsonPropertyName("postData")] + public string PostData { get; set; } - [JsonProperty("maxTimeout")] - public int MaxTimeout; + [JsonPropertyName("maxTimeout")] + public int MaxTimeout { get; set; } } } \ No newline at end of file diff --git a/src/FlareSolverrSharp/Types/FlareSolverrRequestProxy.cs b/src/FlareSolverrSharp/Types/FlareSolverrRequestProxy.cs index d8e3135..8dde077 100644 --- a/src/FlareSolverrSharp/Types/FlareSolverrRequestProxy.cs +++ b/src/FlareSolverrSharp/Types/FlareSolverrRequestProxy.cs @@ -1,10 +1,10 @@ -using Newtonsoft.Json; +using System.Text.Json.Serialization; namespace FlareSolverrSharp.Types { public class FlareSolverrRequestProxy { - [JsonProperty("url")] - public string Url; + [JsonPropertyName("url")] + public string Url { get; set; } } } diff --git a/src/FlareSolverrSharp/Types/FlareSolverrResponse.cs b/src/FlareSolverrSharp/Types/FlareSolverrResponse.cs index e852ef7..8a2a3ce 100644 --- a/src/FlareSolverrSharp/Types/FlareSolverrResponse.cs +++ b/src/FlareSolverrSharp/Types/FlareSolverrResponse.cs @@ -1,4 +1,4 @@ -using Newtonsoft.Json; +using System.Text.Json.Serialization; // ReSharper disable UnusedMember.Global // ReSharper disable UnassignedField.Global @@ -7,38 +7,77 @@ namespace FlareSolverrSharp.Types { public class FlareSolverrResponse { - public string Status; - public string Message; - public long StartTimestamp; - public long EndTimestamp; - public string Version; - public Solution Solution; - public string Session; - public string[] Sessions; + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("message")] + public string Message { get; set; } + + [JsonPropertyName("startTimestamp")] + public long StartTimestamp { get; set; } + + [JsonPropertyName("endTimestamp")] + public long EndTimestamp { get; set; } + + [JsonPropertyName("version")] + public string Version { get; set; } + + [JsonPropertyName("solution")] + public Solution Solution { get; set; } } public class Solution { - public string Url; - public string Status; - public Headers Headers; - public string Response; - public Cookie[] Cookies; - public string UserAgent; + [JsonPropertyName("url")] + public string Url { get; set; } + + [JsonPropertyName("status")] + public int Status { get; set; } + + [JsonPropertyName("headers")] + public Headers Headers { get; set; } + + [JsonPropertyName("response")] + public string Response { get; set; } + + [JsonPropertyName("cookies")] + public Cookie[] Cookies { get; set; } + + [JsonPropertyName("userAgent")] + public string UserAgent { get; set; } } public class Cookie { - public string Name; - public string Value; - public string Domain; - public string Path; - public double Expires; - public int Size; - public bool HttpOnly; - public bool Secure; - public bool Session; - public string SameSite; + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } + + [JsonPropertyName("domain")] + public string Domain { get; set; } + + [JsonPropertyName("path")] + public string Path { get; set; } + + [JsonPropertyName("expiry")] + public double Expires { get; set; } + + [JsonPropertyName("size")] + public int Size { get; set; } + + [JsonPropertyName("httpOnly")] + public bool HttpOnly { get; set; } + + [JsonPropertyName("secure")] + public bool Secure { get; set; } + + [JsonPropertyName("session")] + public bool Session { get; set; } + + [JsonPropertyName("sameSite")] + public string SameSite { get; set; } public string ToHeaderValue() => $"{Name}={Value}"; public System.Net.Cookie ToCookieObj() => new System.Net.Cookie(Name, Value); @@ -47,9 +86,13 @@ public class Cookie public class Headers { - public string Status; - public string Date; - [JsonProperty(PropertyName = "content-type")] - public string ContentType; + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("date")] + public string Date { get; set; } + + [JsonPropertyName("content-type")] + public string ContentType { get; set; } } } \ No newline at end of file