diff --git a/DevBase.Net/Batch/Batch.cs b/DevBase.Net/Batch/Batch.cs new file mode 100644 index 0000000..76941d4 --- /dev/null +++ b/DevBase.Net/Batch/Batch.cs @@ -0,0 +1,82 @@ +using System.Collections.Concurrent; +using DevBase.Net.Core; + +namespace DevBase.Net.Batch; + +public sealed class Batch +{ + private readonly ConcurrentQueue _queue = new(); + private readonly BatchRequests _parent; + + public string Name { get; } + public int QueueCount => _queue.Count; + + internal Batch(string name, BatchRequests parent) + { + Name = name; + _parent = parent; + } + + public Batch Add(Request request) + { + ArgumentNullException.ThrowIfNull(request); + _queue.Enqueue(request); + return this; + } + + public Batch Add(IEnumerable requests) + { + foreach (Request request in requests) + Add(request); + return this; + } + + public Batch Add(string url) + { + return Add(new Request(url)); + } + + public Batch Add(IEnumerable urls) + { + foreach (string url in urls) + Add(url); + return this; + } + + public Batch Enqueue(Request request) => Add(request); + public Batch Enqueue(string url) => Add(url); + public Batch Enqueue(IEnumerable requests) => Add(requests); + public Batch Enqueue(IEnumerable urls) => Add(urls); + + public Batch Enqueue(string url, Action configure) + { + Request request = new Request(url); + configure(request); + return Add(request); + } + + public Batch Enqueue(Func requestFactory) + { + return Add(requestFactory()); + } + + public bool TryDequeue(out Request? request) + { + return _queue.TryDequeue(out request); + } + + internal List DequeueAll() + { + List requests = new List(_queue.Count); + while (_queue.TryDequeue(out Request? request)) + requests.Add(request); + return requests; + } + + public void Clear() + { + while (_queue.TryDequeue(out _)) { } + } + + public BatchRequests EndBatch() => _parent; +} diff --git a/DevBase.Net/Batch/BatchProgressInfo.cs b/DevBase.Net/Batch/BatchProgressInfo.cs new file mode 100644 index 0000000..17c8fca --- /dev/null +++ b/DevBase.Net/Batch/BatchProgressInfo.cs @@ -0,0 +1,12 @@ +namespace DevBase.Net.Batch; + +public sealed record BatchProgressInfo( + string BatchName, + int Completed, + int Total, + int Errors +) +{ + public double PercentComplete => Total > 0 ? (double)Completed / Total * 100 : 0; + public int Remaining => Total - Completed; +} diff --git a/DevBase.Net/Core/BatchRequests.cs b/DevBase.Net/Batch/BatchRequests.cs similarity index 89% rename from DevBase.Net/Core/BatchRequests.cs rename to DevBase.Net/Batch/BatchRequests.cs index ec6ab82..fd4e5c5 100644 --- a/DevBase.Net/Core/BatchRequests.cs +++ b/DevBase.Net/Batch/BatchRequests.cs @@ -1,9 +1,10 @@ using System.Collections.Concurrent; using System.Net; using System.Text; +using DevBase.Net.Core; using DevBase.Net.Utils; -namespace DevBase.Net.Core; +namespace DevBase.Net.Batch; public sealed class BatchRequests : IDisposable, IAsyncDisposable { @@ -777,121 +778,3 @@ public async ValueTask DisposeAsync() #endregion } - -public sealed class Batch -{ - private readonly ConcurrentQueue _queue = new(); - private readonly BatchRequests _parent; - - public string Name { get; } - public int QueueCount => _queue.Count; - - internal Batch(string name, BatchRequests parent) - { - Name = name; - _parent = parent; - } - - public Batch Add(Request request) - { - ArgumentNullException.ThrowIfNull(request); - _queue.Enqueue(request); - return this; - } - - public Batch Add(IEnumerable requests) - { - foreach (Request request in requests) - Add(request); - return this; - } - - public Batch Add(string url) - { - return Add(new Request(url)); - } - - public Batch Add(IEnumerable urls) - { - foreach (string url in urls) - Add(url); - return this; - } - - public Batch Enqueue(Request request) => Add(request); - public Batch Enqueue(string url) => Add(url); - public Batch Enqueue(IEnumerable requests) => Add(requests); - public Batch Enqueue(IEnumerable urls) => Add(urls); - - public Batch Enqueue(string url, Action configure) - { - Request request = new Request(url); - configure(request); - return Add(request); - } - - public Batch Enqueue(Func requestFactory) - { - return Add(requestFactory()); - } - - public bool TryDequeue(out Request? request) - { - return _queue.TryDequeue(out request); - } - - internal List DequeueAll() - { - List requests = new List(_queue.Count); - while (_queue.TryDequeue(out Request? request)) - requests.Add(request); - return requests; - } - - public void Clear() - { - while (_queue.TryDequeue(out _)) { } - } - - public BatchRequests EndBatch() => _parent; -} - -public sealed record BatchProgressInfo( - string BatchName, - int Completed, - int Total, - int Errors -) -{ - public double PercentComplete => Total > 0 ? (double)Completed / Total * 100 : 0; - public int Remaining => Total - Completed; -} - -public sealed record BatchStatistics( - int BatchCount, - int TotalQueuedRequests, - int ProcessedRequests, - int ErrorCount, - Dictionary RequestsPerBatch -) -{ - public double SuccessRate => ProcessedRequests > 0 - ? (double)(ProcessedRequests - ErrorCount) / ProcessedRequests * 100 - : 0; -} - -public readonly struct RequeueDecision -{ - public bool ShouldRequeue { get; } - public Request? ModifiedRequest { get; } - - private RequeueDecision(bool shouldRequeue, Request? modifiedRequest = null) - { - ShouldRequeue = shouldRequeue; - ModifiedRequest = modifiedRequest; - } - - public static RequeueDecision NoRequeue => new(false); - public static RequeueDecision Requeue() => new(true); - public static RequeueDecision RequeueWith(Request modifiedRequest) => new(true, modifiedRequest); -} diff --git a/DevBase.Net/Batch/BatchStatistics.cs b/DevBase.Net/Batch/BatchStatistics.cs new file mode 100644 index 0000000..304c238 --- /dev/null +++ b/DevBase.Net/Batch/BatchStatistics.cs @@ -0,0 +1,14 @@ +namespace DevBase.Net.Batch; + +public sealed record BatchStatistics( + int BatchCount, + int TotalQueuedRequests, + int ProcessedRequests, + int ErrorCount, + Dictionary RequestsPerBatch +) +{ + public double SuccessRate => ProcessedRequests > 0 + ? (double)(ProcessedRequests - ErrorCount) / ProcessedRequests * 100 + : 0; +} diff --git a/DevBase.Net/Batch/Proxied/ProxiedBatch.cs b/DevBase.Net/Batch/Proxied/ProxiedBatch.cs new file mode 100644 index 0000000..a41a60c --- /dev/null +++ b/DevBase.Net/Batch/Proxied/ProxiedBatch.cs @@ -0,0 +1,82 @@ +using System.Collections.Concurrent; +using DevBase.Net.Core; + +namespace DevBase.Net.Batch.Proxied; + +public sealed class ProxiedBatch +{ + private readonly ConcurrentQueue _queue = new(); + private readonly ProxiedBatchRequests _parent; + + public string Name { get; } + public int QueueCount => _queue.Count; + + internal ProxiedBatch(string name, ProxiedBatchRequests parent) + { + Name = name; + _parent = parent; + } + + public ProxiedBatch Add(Request request) + { + ArgumentNullException.ThrowIfNull(request); + _queue.Enqueue(request); + return this; + } + + public ProxiedBatch Add(IEnumerable requests) + { + foreach (Request request in requests) + Add(request); + return this; + } + + public ProxiedBatch Add(string url) + { + return Add(new Request(url)); + } + + public ProxiedBatch Add(IEnumerable urls) + { + foreach (string url in urls) + Add(url); + return this; + } + + public ProxiedBatch Enqueue(Request request) => Add(request); + public ProxiedBatch Enqueue(string url) => Add(url); + public ProxiedBatch Enqueue(IEnumerable requests) => Add(requests); + public ProxiedBatch Enqueue(IEnumerable urls) => Add(urls); + + public ProxiedBatch Enqueue(string url, Action configure) + { + Request request = new Request(url); + configure(request); + return Add(request); + } + + public ProxiedBatch Enqueue(Func requestFactory) + { + return Add(requestFactory()); + } + + public bool TryDequeue(out Request? request) + { + return _queue.TryDequeue(out request); + } + + internal List DequeueAll() + { + List requests = new List(_queue.Count); + while (_queue.TryDequeue(out Request? request)) + requests.Add(request); + return requests; + } + + public void Clear() + { + while (_queue.TryDequeue(out _)) { } + } + + public ProxiedBatchRequests EndBatch() => _parent; +} diff --git a/DevBase.Net/Core/ProxiedBatchRequests.cs b/DevBase.Net/Batch/Proxied/ProxiedBatchRequests.cs similarity index 85% rename from DevBase.Net/Core/ProxiedBatchRequests.cs rename to DevBase.Net/Batch/Proxied/ProxiedBatchRequests.cs index cc89179..4cab3d3 100644 --- a/DevBase.Net/Core/ProxiedBatchRequests.cs +++ b/DevBase.Net/Batch/Proxied/ProxiedBatchRequests.cs @@ -1,10 +1,12 @@ using System.Collections.Concurrent; using System.Net; using System.Text; +using DevBase.Net.Batch.Strategies; +using DevBase.Net.Core; using DevBase.Net.Proxy; using DevBase.Net.Utils; -namespace DevBase.Net.Core; +namespace DevBase.Net.Batch.Proxied; public sealed class ProxiedBatchRequests : IDisposable, IAsyncDisposable { @@ -965,196 +967,3 @@ public async ValueTask DisposeAsync() #endregion } - -public sealed class ProxiedBatch -{ - private readonly ConcurrentQueue _queue = new(); - private readonly ProxiedBatchRequests _parent; - - public string Name { get; } - public int QueueCount => _queue.Count; - - internal ProxiedBatch(string name, ProxiedBatchRequests parent) - { - Name = name; - _parent = parent; - } - - public ProxiedBatch Add(Request request) - { - ArgumentNullException.ThrowIfNull(request); - _queue.Enqueue(request); - return this; - } - - public ProxiedBatch Add(IEnumerable requests) - { - foreach (Request request in requests) - Add(request); - return this; - } - - public ProxiedBatch Add(string url) - { - return Add(new Request(url)); - } - - public ProxiedBatch Add(IEnumerable urls) - { - foreach (string url in urls) - Add(url); - return this; - } - - public ProxiedBatch Enqueue(Request request) => Add(request); - public ProxiedBatch Enqueue(string url) => Add(url); - public ProxiedBatch Enqueue(IEnumerable requests) => Add(requests); - public ProxiedBatch Enqueue(IEnumerable urls) => Add(urls); - - public ProxiedBatch Enqueue(string url, Action configure) - { - Request request = new Request(url); - configure(request); - return Add(request); - } - - public ProxiedBatch Enqueue(Func requestFactory) - { - return Add(requestFactory()); - } - - public bool TryDequeue(out Request? request) - { - return _queue.TryDequeue(out request); - } - - internal List DequeueAll() - { - List requests = new List(_queue.Count); - while (_queue.TryDequeue(out Request? request)) - requests.Add(request); - return requests; - } - - public void Clear() - { - while (_queue.TryDequeue(out _)) { } - } - - public ProxiedBatchRequests EndBatch() => _parent; -} - -public sealed record ProxiedBatchStatistics( - int BatchCount, - int TotalQueuedRequests, - int ProcessedRequests, - int ErrorCount, - int ProxyFailureCount, - int TotalProxies, - int AvailableProxies, - Dictionary RequestsPerBatch -) -{ - public double SuccessRate => ProcessedRequests > 0 - ? (double)(ProcessedRequests - ErrorCount) / ProcessedRequests * 100 - : 0; - - public double ProxyAvailabilityRate => TotalProxies > 0 - ? (double)AvailableProxies / TotalProxies * 100 - : 0; -} - -public sealed record ProxyFailureContext( - TrackedProxyInfo Proxy, - System.Exception Exception, - Request FailedRequest, - bool ProxyTimedOut, - int CurrentFailureCount, - int RemainingAvailableProxies -); - -public interface IProxyRotationStrategy -{ - TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex); -} - -public sealed class RoundRobinStrategy : IProxyRotationStrategy -{ - public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) - { - if (proxies.Count == 0) - return null; - - int attempts = 0; - while (attempts < proxies.Count) - { - currentIndex = (currentIndex + 1) % proxies.Count; - var proxy = proxies[currentIndex]; - - if (proxy.IsAvailable()) - return proxy; - - attempts++; - } - - return null; - } -} - -public sealed class RandomStrategy : IProxyRotationStrategy -{ - private static readonly Random Random = new(); - - public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) - { - var available = proxies.Where(p => p.IsAvailable()).ToList(); - if (available.Count == 0) - return null; - - int index = Random.Next(available.Count); - var selected = available[index]; - currentIndex = proxies.IndexOf(selected); - return selected; - } -} - -public sealed class LeastFailuresStrategy : IProxyRotationStrategy -{ - public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) - { - var available = proxies.Where(p => p.IsAvailable()).ToList(); - if (available.Count == 0) - return null; - - var selected = available.OrderBy(p => p.FailureCount).ThenBy(p => p.TotalTimeouts).First(); - currentIndex = proxies.IndexOf(selected); - return selected; - } -} - -public sealed class StickyStrategy : IProxyRotationStrategy -{ - public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) - { - if (proxies.Count == 0) - return null; - - if (currentIndex >= 0 && currentIndex < proxies.Count) - { - var current = proxies[currentIndex]; - if (current.IsAvailable()) - return current; - } - - for (int i = 0; i < proxies.Count; i++) - { - if (proxies[i].IsAvailable()) - { - currentIndex = i; - return proxies[i]; - } - } - - return null; - } -} diff --git a/DevBase.Net/Batch/Proxied/ProxiedBatchStatistics.cs b/DevBase.Net/Batch/Proxied/ProxiedBatchStatistics.cs new file mode 100644 index 0000000..dbad211 --- /dev/null +++ b/DevBase.Net/Batch/Proxied/ProxiedBatchStatistics.cs @@ -0,0 +1,21 @@ +namespace DevBase.Net.Batch.Proxied; + +public sealed record ProxiedBatchStatistics( + int BatchCount, + int TotalQueuedRequests, + int ProcessedRequests, + int ErrorCount, + int ProxyFailureCount, + int TotalProxies, + int AvailableProxies, + Dictionary RequestsPerBatch +) +{ + public double SuccessRate => ProcessedRequests > 0 + ? (double)(ProcessedRequests - ErrorCount) / ProcessedRequests * 100 + : 0; + + public double ProxyAvailabilityRate => TotalProxies > 0 + ? (double)AvailableProxies / TotalProxies * 100 + : 0; +} diff --git a/DevBase.Net/Batch/Proxied/ProxyFailureContext.cs b/DevBase.Net/Batch/Proxied/ProxyFailureContext.cs new file mode 100644 index 0000000..980a0d3 --- /dev/null +++ b/DevBase.Net/Batch/Proxied/ProxyFailureContext.cs @@ -0,0 +1,13 @@ +using DevBase.Net.Core; +using DevBase.Net.Proxy; + +namespace DevBase.Net.Batch.Proxied; + +public sealed record ProxyFailureContext( + TrackedProxyInfo Proxy, + System.Exception Exception, + Request FailedRequest, + bool ProxyTimedOut, + int CurrentFailureCount, + int RemainingAvailableProxies +); diff --git a/DevBase.Net/Batch/RequeueDecision.cs b/DevBase.Net/Batch/RequeueDecision.cs new file mode 100644 index 0000000..679211f --- /dev/null +++ b/DevBase.Net/Batch/RequeueDecision.cs @@ -0,0 +1,19 @@ +using DevBase.Net.Core; + +namespace DevBase.Net.Batch; + +public readonly struct RequeueDecision +{ + public bool ShouldRequeue { get; } + public Request? ModifiedRequest { get; } + + private RequeueDecision(bool shouldRequeue, Request? modifiedRequest = null) + { + ShouldRequeue = shouldRequeue; + ModifiedRequest = modifiedRequest; + } + + public static RequeueDecision NoRequeue => new(false); + public static RequeueDecision Requeue() => new(true); + public static RequeueDecision RequeueWith(Request modifiedRequest) => new(true, modifiedRequest); +} diff --git a/DevBase.Net/Batch/Strategies/IProxyRotationStrategy.cs b/DevBase.Net/Batch/Strategies/IProxyRotationStrategy.cs new file mode 100644 index 0000000..b226413 --- /dev/null +++ b/DevBase.Net/Batch/Strategies/IProxyRotationStrategy.cs @@ -0,0 +1,8 @@ +using DevBase.Net.Proxy; + +namespace DevBase.Net.Batch.Strategies; + +public interface IProxyRotationStrategy +{ + TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex); +} diff --git a/DevBase.Net/Batch/Strategies/LeastFailuresStrategy.cs b/DevBase.Net/Batch/Strategies/LeastFailuresStrategy.cs new file mode 100644 index 0000000..6304562 --- /dev/null +++ b/DevBase.Net/Batch/Strategies/LeastFailuresStrategy.cs @@ -0,0 +1,17 @@ +using DevBase.Net.Proxy; + +namespace DevBase.Net.Batch.Strategies; + +public sealed class LeastFailuresStrategy : IProxyRotationStrategy +{ + public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) + { + List available = proxies.Where(p => p.IsAvailable()).ToList(); + if (available.Count == 0) + return null; + + TrackedProxyInfo selected = available.OrderBy(p => p.FailureCount).ThenBy(p => p.TotalTimeouts).First(); + currentIndex = proxies.IndexOf(selected); + return selected; + } +} diff --git a/DevBase.Net/Batch/Strategies/RandomStrategy.cs b/DevBase.Net/Batch/Strategies/RandomStrategy.cs new file mode 100644 index 0000000..61bba79 --- /dev/null +++ b/DevBase.Net/Batch/Strategies/RandomStrategy.cs @@ -0,0 +1,20 @@ +using DevBase.Net.Proxy; + +namespace DevBase.Net.Batch.Strategies; + +public sealed class RandomStrategy : IProxyRotationStrategy +{ + private static readonly Random Random = new(); + + public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) + { + List available = proxies.Where(p => p.IsAvailable()).ToList(); + if (available.Count == 0) + return null; + + int index = Random.Next(available.Count); + TrackedProxyInfo selected = available[index]; + currentIndex = proxies.IndexOf(selected); + return selected; + } +} diff --git a/DevBase.Net/Batch/Strategies/RoundRobinStrategy.cs b/DevBase.Net/Batch/Strategies/RoundRobinStrategy.cs new file mode 100644 index 0000000..dd66ee5 --- /dev/null +++ b/DevBase.Net/Batch/Strategies/RoundRobinStrategy.cs @@ -0,0 +1,26 @@ +using DevBase.Net.Proxy; + +namespace DevBase.Net.Batch.Strategies; + +public sealed class RoundRobinStrategy : IProxyRotationStrategy +{ + public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) + { + if (proxies.Count == 0) + return null; + + int attempts = 0; + while (attempts < proxies.Count) + { + currentIndex = (currentIndex + 1) % proxies.Count; + TrackedProxyInfo proxy = proxies[currentIndex]; + + if (proxy.IsAvailable()) + return proxy; + + attempts++; + } + + return null; + } +} diff --git a/DevBase.Net/Batch/Strategies/StickyStrategy.cs b/DevBase.Net/Batch/Strategies/StickyStrategy.cs new file mode 100644 index 0000000..ab73e3b --- /dev/null +++ b/DevBase.Net/Batch/Strategies/StickyStrategy.cs @@ -0,0 +1,30 @@ +using DevBase.Net.Proxy; + +namespace DevBase.Net.Batch.Strategies; + +public sealed class StickyStrategy : IProxyRotationStrategy +{ + public TrackedProxyInfo? SelectProxy(List proxies, ref int currentIndex) + { + if (proxies.Count == 0) + return null; + + if (currentIndex >= 0 && currentIndex < proxies.Count) + { + TrackedProxyInfo current = proxies[currentIndex]; + if (current.IsAvailable()) + return current; + } + + for (int i = 0; i < proxies.Count; i++) + { + if (proxies[i].IsAvailable()) + { + currentIndex = i; + return proxies[i]; + } + } + + return null; + } +} diff --git a/DevBase.Test/DevBaseRequests/BatchRequestsTest.cs b/DevBase.Test/DevBaseRequests/BatchRequestsTest.cs index 08ab196..cbb7ed1 100644 --- a/DevBase.Test/DevBaseRequests/BatchRequestsTest.cs +++ b/DevBase.Test/DevBaseRequests/BatchRequestsTest.cs @@ -1,3 +1,4 @@ +using DevBase.Net.Batch; using DevBase.Net.Core; namespace DevBase.Test.DevBaseRequests; diff --git a/DevBase.Test/DevBaseRequests/ProxiedBatchRequestsTest.cs b/DevBase.Test/DevBaseRequests/ProxiedBatchRequestsTest.cs index 09f1a27..1859a41 100644 --- a/DevBase.Test/DevBaseRequests/ProxiedBatchRequestsTest.cs +++ b/DevBase.Test/DevBaseRequests/ProxiedBatchRequestsTest.cs @@ -1,3 +1,6 @@ +using DevBase.Net.Batch; +using DevBase.Net.Batch.Proxied; +using DevBase.Net.Batch.Strategies; using DevBase.Net.Core; using DevBase.Net.Proxy; using DevBase.Net.Proxy.Enums;