Skip to content

Commit cfc46b6

Browse files
jkendall327jkendall327
committed
1 parent fff287d commit cfc46b6

7 files changed

Lines changed: 331 additions & 142 deletions

File tree

‎Octans.Core/Downloaders/DownloaderResolverOptions.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,4 @@ namespace Octans.Core.Downloaders;
33
public class DownloaderResolverOptions
44
{
55
public TimeSpan OperationTimeout { get; set; } = TimeSpan.FromSeconds(30);
6-
public long MaxResponseBytes { get; set; } = 5L * 1024 * 1024;
76
}

‎Octans.Core/Downloaders/DownloaderService.cs‎

Lines changed: 13 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
1-
using System.Text;
21
using Microsoft.Extensions.Logging;
32
using Microsoft.Extensions.Options;
43
using Octans.Core.Http;
54

65
namespace Octans.Core.Downloaders;
76

87
internal sealed class DownloaderService(
9-
IHttpClientFactory clientFactory,
108
IDownloaderFactory downloaderFactory,
11-
IDownloadRequestHeaderProvider requestHeaderProvider,
9+
IHttpDocumentFetcher documentFetcher,
1210
IOptions<DownloaderResolverOptions> options,
1311
ILogger<DownloaderService> logger)
1412
{
@@ -19,10 +17,6 @@ public async Task<IReadOnlyList<Uri>> ResolveAsync(Uri uri, CancellationToken ca
1917

2018
var downloaders = await downloaderFactory.GetDownloaders();
2119

22-
#pragma warning disable CA2000
23-
var client = clientFactory.CreateClient("DownloadClient");
24-
#pragma warning restore CA2000
25-
2620
string? raw = null;
2721

2822
foreach (var downloader in downloaders)
@@ -45,7 +39,7 @@ public async Task<IReadOnlyList<Uri>> ResolveAsync(Uri uri, CancellationToken ca
4539

4640
if (raw is null)
4741
{
48-
raw = await TryRunAsync(downloader, "fetch_html", () => GetStringAsync(client, uri, resolverToken));
42+
raw = await TryRunAsync(downloader, "fetch_html", () => documentFetcher.GetStringAsync(uri, resolverToken));
4943
if (raw is null)
5044
{
5145
return [];
@@ -68,7 +62,7 @@ public async Task<IReadOnlyList<Uri>> ResolveAsync(Uri uri, CancellationToken ca
6862
content = await TryRunAsync(
6963
downloader,
7064
"fetch_gallery_html",
71-
() => GetStringAsync(client, galleryUrl, resolverToken));
65+
() => documentFetcher.GetStringAsync(galleryUrl, resolverToken));
7266
if (content is null)
7367
{
7468
continue;
@@ -92,76 +86,6 @@ public async Task<IReadOnlyList<Uri>> ResolveAsync(Uri uri, CancellationToken ca
9286
return [];
9387
}
9488

95-
private async Task<string> GetStringAsync(HttpClient client, Uri uri, CancellationToken cancellationToken)
96-
{
97-
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
98-
requestHeaderProvider.ApplyHeaders(request);
99-
100-
using var response = await client.SendAsync(
101-
request,
102-
HttpCompletionOption.ResponseHeadersRead,
103-
cancellationToken);
104-
response.EnsureSuccessStatusCode();
105-
return await ReadBoundedStringAsync(response, uri, cancellationToken);
106-
}
107-
108-
private async Task<string> ReadBoundedStringAsync(
109-
HttpResponseMessage response,
110-
Uri uri,
111-
CancellationToken cancellationToken)
112-
{
113-
var maxResponseBytes = options.Value.MaxResponseBytes;
114-
var reportedBytes = response.Content.Headers.ContentLength;
115-
if (maxResponseBytes > 0 && reportedBytes > maxResponseBytes)
116-
{
117-
throw new DownloaderContractException(
118-
$"Downloader response from {uri.Host} reported {reportedBytes} bytes, exceeding the configured {maxResponseBytes} byte limit.");
119-
}
120-
121-
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
122-
using var buffer = new MemoryStream();
123-
var bytes = new byte[81920];
124-
long totalBytes = 0;
125-
126-
while (true)
127-
{
128-
var bytesRead = await stream.ReadAsync(bytes, cancellationToken);
129-
if (bytesRead <= 0)
130-
{
131-
break;
132-
}
133-
134-
if (maxResponseBytes > 0 && totalBytes > maxResponseBytes - bytesRead)
135-
{
136-
throw new DownloaderContractException(
137-
$"Downloader response from {uri.Host} exceeded the configured {maxResponseBytes} byte limit.");
138-
}
139-
140-
await buffer.WriteAsync(bytes.AsMemory(0, bytesRead), cancellationToken);
141-
totalBytes += bytesRead;
142-
}
143-
144-
return GetResponseEncoding(response).GetString(buffer.ToArray());
145-
}
146-
147-
private static Encoding GetResponseEncoding(HttpResponseMessage response)
148-
{
149-
var charset = response.Content.Headers.ContentType?.CharSet;
150-
if (string.IsNullOrWhiteSpace(charset))
151-
{
152-
return Encoding.UTF8;
153-
}
154-
155-
try
156-
{
157-
return Encoding.GetEncoding(charset);
158-
}
159-
catch (ArgumentException)
160-
{
161-
return Encoding.UTF8;
162-
}
163-
}
164-
16589
private CancellationTokenSource CreateOperationTimeout(CancellationToken cancellationToken)
16690
{
16791
var timeout = options.Value.OperationTimeout;
@@ -213,6 +137,16 @@ private static Uri CreateHttpUri(string value, string source)
213137
{
214138
return await action();
215139
}
140+
catch (HttpDocumentFetchException ex)
141+
{
142+
logger.LogWarning(
143+
ex,
144+
"Skipping downloader {DownloaderName} after failure during {DownloaderOperation}: {Message}",
145+
GetDownloaderName(downloader),
146+
operation,
147+
ex.Message);
148+
return default;
149+
}
216150
catch (DownloaderContractException ex)
217151
{
218152
logger.LogWarning(

‎Octans.Core/Http/DownloadServiceExtensions.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ public static IServiceCollection AddDownloadManager(
3737
services.TryAddSingleton<IDownloadDiskSpaceGuard, DownloadDiskSpaceGuard>();
3838
services.TryAddSingleton<IDownloadHostCircuitRegistry, DownloadHostCircuitRegistry>();
3939
services.TryAddSingleton<IDownloadRequestHeaderProvider, DownloadRequestHeaderProvider>();
40+
services.AddOptions<HttpDocumentFetcherOptions>();
41+
services.TryAddSingleton<IHttpDocumentFetcher, HttpDocumentFetcher>();
4042
services.TryAddSingleton<DownloadTelemetry>();
4143
services.TryAddSingleton<DownloadStagingPaths>();
4244
services.TryAddSingleton<IDownloadLifecycleService, DownloadLifecycleService>();

‎Octans.Core/Http/HttpDocumentFetcher.cs‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
using System.Net;
2+
using System.Text;
3+
using Microsoft.Extensions.Logging;
4+
using Microsoft.Extensions.Options;
5+
6+
namespace Octans.Core.Http;
7+
8+
public interface IHttpDocumentFetcher
9+
{
10+
Task<string> GetStringAsync(Uri uri, CancellationToken cancellationToken = default);
11+
}
12+
13+
public sealed class HttpDocumentFetcherOptions
14+
{
15+
public long MaxResponseBytes { get; set; } = 5L * 1024 * 1024;
16+
}
17+
18+
public sealed class HttpDocumentFetchException : Exception
19+
{
20+
public HttpDocumentFetchException()
21+
{
22+
}
23+
24+
public HttpDocumentFetchException(string message) : base(message)
25+
{
26+
}
27+
28+
public HttpDocumentFetchException(string message, Exception innerException) : base(message, innerException)
29+
{
30+
}
31+
32+
public static HttpDocumentFetchException ForStatus(Uri uri, HttpStatusCode statusCode)
33+
{
34+
return new($"Document request to {uri.Host} failed with HTTP {(int)statusCode} ({statusCode}).");
35+
}
36+
37+
public static HttpDocumentFetchException ForReportedSize(Uri uri, long reportedBytes, long maxBytes)
38+
{
39+
return new(
40+
$"Document response from {uri.Host} reported {reportedBytes} bytes, exceeding the configured {maxBytes} byte limit.");
41+
}
42+
43+
public static HttpDocumentFetchException ForReceivedSize(Uri uri, long maxBytes)
44+
{
45+
return new($"Document response from {uri.Host} exceeded the configured {maxBytes} byte limit.");
46+
}
47+
}
48+
49+
internal sealed class HttpDocumentFetcher(
50+
IHttpClientFactory clientFactory,
51+
IDownloadRequestHeaderProvider requestHeaderProvider,
52+
IOptions<HttpDocumentFetcherOptions> options,
53+
ILogger<HttpDocumentFetcher> logger) : IHttpDocumentFetcher
54+
{
55+
public async Task<string> GetStringAsync(Uri uri, CancellationToken cancellationToken = default)
56+
{
57+
try
58+
{
59+
using var client = clientFactory.CreateClient("DownloadClient");
60+
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
61+
requestHeaderProvider.ApplyHeaders(request);
62+
63+
using var response = await client.SendAsync(
64+
request,
65+
HttpCompletionOption.ResponseHeadersRead,
66+
cancellationToken);
67+
if (!response.IsSuccessStatusCode)
68+
{
69+
throw HttpDocumentFetchException.ForStatus(uri, response.StatusCode);
70+
}
71+
72+
return await ReadBoundedStringAsync(response, uri, cancellationToken);
73+
}
74+
catch (OperationCanceledException)
75+
{
76+
throw;
77+
}
78+
catch (HttpDocumentFetchException)
79+
{
80+
throw;
81+
}
82+
catch (Exception ex)
83+
{
84+
logger.LogDebug(ex, "Document request to {DocumentHost} failed.", uri.Host);
85+
throw new HttpDocumentFetchException($"Document request to {uri.Host} failed.", ex);
86+
}
87+
}
88+
89+
private async Task<string> ReadBoundedStringAsync(
90+
HttpResponseMessage response,
91+
Uri uri,
92+
CancellationToken cancellationToken)
93+
{
94+
var maxResponseBytes = options.Value.MaxResponseBytes;
95+
var reportedBytes = response.Content.Headers.ContentLength;
96+
if (maxResponseBytes > 0 && reportedBytes > maxResponseBytes)
97+
{
98+
throw HttpDocumentFetchException.ForReportedSize(uri, reportedBytes.Value, maxResponseBytes);
99+
}
100+
101+
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
102+
using var buffer = new MemoryStream();
103+
var bytes = new byte[81920];
104+
long totalBytes = 0;
105+
106+
while (true)
107+
{
108+
var bytesRead = await stream.ReadAsync(bytes, cancellationToken);
109+
if (bytesRead <= 0)
110+
{
111+
break;
112+
}
113+
114+
if (maxResponseBytes > 0 && totalBytes > maxResponseBytes - bytesRead)
115+
{
116+
throw HttpDocumentFetchException.ForReceivedSize(uri, maxResponseBytes);
117+
}
118+
119+
await buffer.WriteAsync(bytes.AsMemory(0, bytesRead), cancellationToken);
120+
totalBytes += bytesRead;
121+
}
122+
123+
return GetResponseEncoding(response).GetString(buffer.ToArray());
124+
}
125+
126+
private static Encoding GetResponseEncoding(HttpResponseMessage response)
127+
{
128+
var charset = response.Content.Headers.ContentType?.CharSet;
129+
if (string.IsNullOrWhiteSpace(charset))
130+
{
131+
return Encoding.UTF8;
132+
}
133+
134+
try
135+
{
136+
return Encoding.GetEncoding(charset);
137+
}
138+
catch (ArgumentException)
139+
{
140+
return Encoding.UTF8;
141+
}
142+
}
143+
}

‎Octans.Tests/Downloads/HttpDocumentFetcherTests.cs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
using System.Net;
2+
using System.Text;
3+
using FluentAssertions;
4+
using Microsoft.Extensions.Logging.Abstractions;
5+
using Microsoft.Extensions.Options;
6+
using NSubstitute;
7+
using Octans.Core.Http;
8+
using Octans.Core.Http.Models;
9+
10+
namespace Octans.Tests.Downloads;
11+
12+
public class HttpDocumentFetcherTests
13+
{
14+
[Fact]
15+
public async Task GetStringAsync_ShouldApplySharedHeadersAndReturnContent()
16+
{
17+
HttpRequestMessage? observedRequest = null;
18+
using var httpClient = new HttpClient(new StubHttpMessageHandler(request =>
19+
{
20+
observedRequest = request;
21+
return new HttpResponseMessage(HttpStatusCode.OK)
22+
{
23+
Content = new StringContent("hello", Encoding.UTF8, "text/html")
24+
};
25+
}));
26+
var downloadOptions = new DownloadManagerOptions();
27+
downloadOptions.RequestHeaders.DefaultUserAgent = "Octans-Test/1.0";
28+
downloadOptions.RequestHeaders.Headers["X-Octans-Test"] = "yes";
29+
var sut = CreateFetcher(httpClient, downloadOptions);
30+
31+
var result = await sut.GetStringAsync(new("https://example.com/document"));
32+
33+
result.Should().Be("hello");
34+
observedRequest.Should().NotBeNull();
35+
observedRequest!.Headers.GetValues("User-Agent").Should().Contain("Octans-Test/1.0");
36+
observedRequest.Headers.GetValues("X-Octans-Test").Should().Contain("yes");
37+
}
38+
39+
[Fact]
40+
public async Task GetStringAsync_ShouldRejectReportedResponseLargerThanConfiguredLimit()
41+
{
42+
using var httpClient = new HttpClient(new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
43+
{
44+
Content = new ByteArrayContent(Encoding.UTF8.GetBytes("too large"))
45+
}));
46+
var sut = CreateFetcher(httpClient, documentOptions: new() { MaxResponseBytes = 4 });
47+
48+
var act = async () => await sut.GetStringAsync(new("https://example.com/document"));
49+
50+
await act.Should()
51+
.ThrowAsync<HttpDocumentFetchException>()
52+
.WithMessage("*reported*exceeding*4 byte limit*");
53+
}
54+
55+
[Fact]
56+
public async Task GetStringAsync_ShouldRejectFailedStatusCode()
57+
{
58+
using var httpClient = new HttpClient(new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError)));
59+
var sut = CreateFetcher(httpClient);
60+
61+
var act = async () => await sut.GetStringAsync(new("https://example.com/document"));
62+
63+
await act.Should()
64+
.ThrowAsync<HttpDocumentFetchException>()
65+
.WithMessage("*HTTP 500*");
66+
}
67+
68+
private static HttpDocumentFetcher CreateFetcher(
69+
HttpClient httpClient,
70+
DownloadManagerOptions? downloadOptions = null,
71+
HttpDocumentFetcherOptions? documentOptions = null)
72+
{
73+
var clientFactory = Substitute.For<IHttpClientFactory>();
74+
clientFactory.CreateClient("DownloadClient").Returns(httpClient);
75+
var requestHeaderProvider = new DownloadRequestHeaderProvider(Options.Create(downloadOptions ?? new()));
76+
77+
return new(
78+
clientFactory,
79+
requestHeaderProvider,
80+
Options.Create(documentOptions ?? new()),
81+
NullLogger<HttpDocumentFetcher>.Instance);
82+
}
83+
}

0 commit comments

Comments
 (0)