Harden AOT request header defaults to prevent CDN 428 responses

src/Downloader.Test/UnitTests/RequestConfigurationTest.cs

@@ -0,0 +1,21 @@
+namespace Downloader.Test.UnitTests;
+
+public class RequestConfigurationTest(ITestOutputHelper output) : BaseTestClass(output)
+{
+    [Fact]
+    public void DefaultUserAgentIsValid()
+    {
+        // arrange
+        RequestConfiguration requestConfiguration = new();
+
+        // act
+        string userAgent = requestConfiguration.UserAgent;
+
+        // assert
+        Assert.False(string.IsNullOrWhiteSpace(userAgent));
+        Assert.StartsWith("Downloader/", userAgent);
+        Assert.False(userAgent.EndsWith("/", StringComparison.Ordinal));
+        Assert.NotEqual("Downloader/0.0.0", userAgent);
+        Assert.NotEqual("Downloader/0.0.0.0", userAgent);
+    }
+}

src/Downloader.Test/UnitTests/SocketClientTest.cs

@@ -340,6 +340,89 @@ public void GetTotalSizeFromContentRangeWhenNoHeaderTest()
         TestGetTotalSizeFromContentRange(-1, null);
     }
 
+    [Fact]
+    public void HttpClientUsesFallbackUserAgentWhenConfigurationUserAgentIsNullOrEmpty()
+    {
+        foreach (string userAgent in new string[] { null, string.Empty, "Downloader/" })
+        {
+            // arrange
+            RequestConfiguration requestConfiguration = new() {
+                UserAgent = userAgent
+            };
+            DownloadConfiguration downloadConfiguration = new() {
+                RequestConfiguration = requestConfiguration
+            };
+
+            // act
+            using SocketClient socketClient = new(downloadConfiguration);
+            string actualUserAgent = GetClient(socketClient).DefaultRequestHeaders.UserAgent.ToString();
+
+            // assert
+            Assert.False(string.IsNullOrWhiteSpace(actualUserAgent));
+            Assert.StartsWith("Downloader/", actualUserAgent);
+            Assert.False(actualUserAgent.EndsWith("/", StringComparison.Ordinal));
+        }
+    }
+
+    [Fact]
+    public void HttpClientKeepsCustomUserAgent()
+    {
+        // arrange
+        const string expectedUserAgent = "CustomDownloader/1.2.3";
+        RequestConfiguration requestConfiguration = new() {
+            UserAgent = expectedUserAgent
+        };
+        DownloadConfiguration downloadConfiguration = new() {
+            RequestConfiguration = requestConfiguration
+        };
+
+        // act
+        using SocketClient socketClient = new(downloadConfiguration);
+        string actualUserAgent = GetClient(socketClient).DefaultRequestHeaders.UserAgent.ToString();
+
+        // assert
+        Assert.Equal(expectedUserAgent, actualUserAgent);
+    }
+
+    [Fact]
+    public void HttpClientSetsDefaultAcceptHeaderWhenAcceptIsNotConfigured()
+    {
+        // arrange
+        RequestConfiguration requestConfiguration = new() {
+            Accept = null
+        };
+        DownloadConfiguration downloadConfiguration = new() {
+            RequestConfiguration = requestConfiguration
+        };
+
+        // act
+        using SocketClient socketClient = new(downloadConfiguration);
+        string actualAccept = string.Join(",", GetClient(socketClient).DefaultRequestHeaders.Accept.Select(x => x.MediaType));
+
+        // assert
+        Assert.Equal("*/*", actualAccept);
+    }
+
+    [Fact]
+    public void HttpClientKeepsCustomAcceptHeader()
+    {
+        // arrange
+        const string expectedAccept = "application/json";
+        RequestConfiguration requestConfiguration = new() {
+            Accept = expectedAccept
+        };
+        DownloadConfiguration downloadConfiguration = new() {
+            RequestConfiguration = requestConfiguration
+        };
+
+        // act
+        using SocketClient socketClient = new(downloadConfiguration);
+        string actualAccept = string.Join(",", GetClient(socketClient).DefaultRequestHeaders.Accept.Select(x => x.MediaType));
+
+        // assert
+        Assert.Equal(expectedAccept, actualAccept);
+    }
+
     private void TestGetTotalSizeFromContentRange(long expectedLength, string contentRange)
     {
         // arrange
@@ -353,4 +436,11 @@ private void TestGetTotalSizeFromContentRange(long expectedLength, string conten
         // assert
         Assert.Equal(expectedLength, actualLength);
     }
-}
+
+    private static HttpClient GetClient(SocketClient socketClient)
+    {
+        var propertyInfo = typeof(SocketClient).GetProperty("Client",
+            System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic);
+        return (HttpClient)propertyInfo.GetValue(socketClient);
+    }
+}

src/Downloader/RequestConfiguration.cs

@@ -13,6 +13,8 @@ namespace Downloader;
 /// </summary>
 public class RequestConfiguration
 {
+    private static readonly Version ZeroVersion = new(0, 0, 0, 0);
+
     /// <summary>
     /// Initializes a new instance of the <see cref="RequestConfiguration"/> class with default settings.
     /// </summary>
@@ -29,7 +31,53 @@ public RequestConfiguration()
         Pipelined = true;
         ProtocolVersion = HttpVersion.Version11;
         ConnectTimeout = 30 * 1000; // 30 seconds
-        UserAgent = $"{nameof(Downloader)}/{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3)}";
+        UserAgent = BuildDefaultUserAgent();
+    }
+
+    private static string BuildDefaultUserAgent()
+    {
+        const string fallbackVersion = "5.0";
+        string version = ResolveProductVersion();
+        return $"{nameof(Downloader)}/{version ?? fallbackVersion}";
+    }
+
+    private static string ResolveProductVersion()
+    {
+        Assembly assembly = typeof(RequestConfiguration).Assembly;
+
+        string version = NormalizeVersion(assembly.GetName().Version?.ToString(3));
+        if (!string.IsNullOrWhiteSpace(version))
+            return version;
+
+        version = NormalizeVersion(assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion);
+        if (!string.IsNullOrWhiteSpace(version))
+            return version;
+
+        version = NormalizeVersion(assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version);
+        if (!string.IsNullOrWhiteSpace(version))
+            return version;
+
+        return null;
+    }
+
+    private static string NormalizeVersion(string versionText)
+    {
+        if (string.IsNullOrWhiteSpace(versionText))
+            return null;
+
+        string candidate = versionText.Trim();
+        int metadataSeparatorIndex = candidate.IndexOf('+');
+        if (metadataSeparatorIndex >= 0)
+            candidate = candidate[..metadataSeparatorIndex];
+
+        int preReleaseSeparatorIndex = candidate.IndexOf('-');
+        if (preReleaseSeparatorIndex >= 0)
+            candidate = candidate[..preReleaseSeparatorIndex];
+
+        if (!Version.TryParse(candidate, out Version parsed) || parsed == ZeroVersion)
+            return null;
+
+        return parsed.Build >= 0 ? parsed.ToString(3) : parsed.ToString(2);
     }
 
     /// <summary>
@@ -270,4 +318,4 @@ public RequestConfiguration()
     /// The default value is "<seealso cref="Downloader"/>/{<seealso cref="Version"/>}".
     /// </summary>
     public string UserAgent { get; set; }
-}
+}

src/Downloader/SocketClient.cs

@@ -18,6 +18,9 @@
 public partial class SocketClient : IDisposable
 {
     private const string FilenameStartPointKey = "filename=";
+    private const string FallbackUserAgent = "Downloader/5.0";
+    private const string InvalidUserAgentWithZeroVersion3 = "Downloader/0.0.0";
+    private const string InvalidUserAgentWithZeroVersion4 = "Downloader/0.0.0.0";
 
     [GeneratedRegex(@"bytes\s*((?<from>\d*)\s*-\s*(?<to>\d*)|\*)\s*\/\s*(?<size>\d+|\*)", RegexOptions.Compiled)]
     private static partial Regex RangePatternRegex();
@@ -124,8 +127,8 @@
         client.DefaultRequestHeaders.Clear();
 
         // Add standard headers
-        AddHeaderIfNotEmpty(client.DefaultRequestHeaders, "Accept", requestConfig.Accept);
-        AddHeaderIfNotEmpty(client.DefaultRequestHeaders, "User-Agent", requestConfig.UserAgent);
+        AddHeaderIfNotEmpty(client.DefaultRequestHeaders, "Accept", ResolveAcceptHeader(requestConfig.Accept));
+        AddHeaderIfNotEmpty(client.DefaultRequestHeaders, "User-Agent", ResolveUserAgent(requestConfig.UserAgent));
         client.DefaultRequestHeaders.Add("Connection", requestConfig.KeepAlive ? "keep-alive" : "close");
         client.DefaultRequestHeaders.CacheControl ??= new CacheControlHeaderValue { NoCache = true };
 
@@ -165,64 +168,85 @@
             headers.Add(key, value);
     }
 
+    private static string ResolveAcceptHeader(string accept)
+    {
+        return string.IsNullOrWhiteSpace(accept) ? "*/*" : accept;
+    }
+
+    private static string ResolveUserAgent(string userAgent)
+    {
+        if (string.IsNullOrWhiteSpace(userAgent))
+            return FallbackUserAgent;
+
+        string resolvedUserAgent = userAgent.Trim();
+        if (resolvedUserAgent.EndsWith('/') ||
+            resolvedUserAgent.Equals(InvalidUserAgentWithZeroVersion3, StringComparison.OrdinalIgnoreCase) ||
+            resolvedUserAgent.Equals(InvalidUserAgentWithZeroVersion4, StringComparison.OrdinalIgnoreCase))
+        {
+            return FallbackUserAgent;
+        }
+
+        return resolvedUserAgent;
+    }
+
     /// <summary>
     /// Fetches the response headers asynchronously.
     /// </summary>
     /// <param name="addRange">Indicates whether to add a range header to the request.</param>
     /// <param name="request">The request of client</param>
     /// <param name="cancelToken">Cancel request token</param>
     private async Task FetchResponseHeaders(Request request, bool addRange, CancellationToken cancelToken = default)
     {
         try
         {
             if (!ResponseHeaders.IsEmpty)
                 return;
 
             var requestMsg = request.GetRequest();
             if (addRange)
                 requestMsg.Headers.Range = new RangeHeaderValue(0, 0);
 
             using var response = await SendRequestAsync(requestMsg, cancelToken).ConfigureAwait(false);
             if (!EnsureResponseAddressIsSameWithOrigin(request, response))
             {
                 await FetchResponseHeaders(request, true, cancelToken).ConfigureAwait(false);
             }
         }
         catch (HttpRequestException exp)
         {
             // issue #220: Some servers don't like the Range header and respond with errors like
             // 403 (Forbidden), 404 (Not Found), or 503 (Service Unavailable)
             // even though the file is perfectly downloadable with a normal request (no Range header).
             if (addRange && (exp.IsRequestedRangeNotSatisfiable() || !exp.IsRedirectError()))
             {
                 await FetchResponseHeaders(request, false, cancelToken);
             }
             else if (request.Configuration.AllowAutoRedirect &&
                      exp.IsRedirectError() &&
                      ResponseHeaders.TryGetValue(HttpHeaderNames.Location, out string redirectedUrl) &&
                      !string.IsNullOrWhiteSpace(redirectedUrl) &&
                      !request.Address.ToString().Equals(redirectedUrl, StringComparison.OrdinalIgnoreCase))
             {
                 // issue #223: normalize server-supplied redirect targets
                 // before new Uri(). Preserve this wrapper — the Location
                 // header is attacker-influenceable and may contain illegal
                 // path characters that would otherwise break Uri parsing on
                 // Linux or enable control-char injection.
                 request.Address = new Uri(UrlHelper.EnsurePathEncoded(redirectedUrl));
                 await FetchResponseHeaders(request, addRange, cancelToken).ConfigureAwait(false);
             }
             else
             {
                 // await Console.Error.WriteLineAsync(exp.Message);
                 throw;
             }
         }
     }
 
     /// <summary>
     /// Ensures that the response address is the same as the original address.
     /// </summary>
     /// <param name="request">The request of client</param>
     /// <param name="response">The web response to check.</param>
     /// <returns>True if the response address is the same as the original address; otherwise, false.</returns>
     private bool EnsureResponseAddressIsSameWithOrigin(Request request, HttpResponseMessage response)
@@ -460,4 +484,4 @@
                 Client?.Dispose();
         }
     }
-}
+}