Fix: URL-encoded special characters in $filter and $orderby break OData parsing

src/Core/Models/RestRequestContexts/RestRequestContext.cs

@@ -77,6 +77,12 @@ protected RestRequestContext(string entityName, DatabaseObject dbo)
     /// </summary>
     public NameValueCollection ParsedQueryString { get; set; } = new();
 
+    /// <summary>
+    /// Raw query string from the HTTP request (URL-encoded).
+    /// Used to preserve encoding for special characters in query parameters.
+    /// </summary>
+    public string RawQueryString { get; set; } = string.Empty;
+
     /// <summary>
     /// String holds information needed for pagination.
     /// Based on request this property may or may not be populated.

src/Core/Parsers/RequestParser.cs

@@ -113,14 +113,32 @@ public static void ParseQueryString(RestRequestContext context, ISqlMetadataProv
                         context.FieldsToBeReturned = context.ParsedQueryString[key]!.Split(",").ToList();
                         break;
                     case FILTER_URL:
-                        // save the AST that represents the filter for the query
-                        // ?$filter=<filter clause using microsoft api guidelines>
-                        string filterQueryString = $"?{FILTER_URL}={context.ParsedQueryString[key]}";
-                        context.FilterClauseInUrl = sqlMetadataProvider.GetODataParser().GetFilterClause(filterQueryString, $"{context.EntityName}.{context.DatabaseObject.FullName}");
+                        // Use raw (URL-encoded) filter value to preserve special characters like &
+                        string? rawFilterValue = ExtractRawQueryParameter(context.RawQueryString, FILTER_URL);
+                        // If key exists in ParsedQueryString but not in RawQueryString, something is wrong
+                        if (rawFilterValue is null)
+                        {
+                            throw new DataApiBuilderException(
+                                message: $"Unable to extract {FILTER_URL} parameter from query string.",
+                                statusCode: HttpStatusCode.BadRequest,
+                                subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
+                        }
+
+                        context.FilterClauseInUrl = sqlMetadataProvider.GetODataParser().GetFilterClause($"?{FILTER_URL}={rawFilterValue}", $"{context.EntityName}.{context.DatabaseObject.FullName}");
                         break;
                     case SORT_URL:
-                        string sortQueryString = $"?{SORT_URL}={context.ParsedQueryString[key]}";
-                        (context.OrderByClauseInUrl, context.OrderByClauseOfBackingColumns) = GenerateOrderByLists(context, sqlMetadataProvider, sortQueryString);
+                        // Use raw (URL-encoded) orderby value to preserve special characters
+                        string? rawSortValue = ExtractRawQueryParameter(context.RawQueryString, SORT_URL);
+                        // If key exists in ParsedQueryString but not in RawQueryString, something is wrong
+                        if (rawSortValue is null)
+                        {
+                            throw new DataApiBuilderException(
+                                message: $"Unable to extract {SORT_URL} parameter from query string.",
+                                statusCode: HttpStatusCode.BadRequest,
+                                subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
+                        }
+
+                        (context.OrderByClauseInUrl, context.OrderByClauseOfBackingColumns) = GenerateOrderByLists(context, sqlMetadataProvider, $"?{SORT_URL}={rawSortValue}");
                         break;
                     case AFTER_URL:
                         context.After = context.ParsedQueryString[key];
@@ -283,5 +301,38 @@ private static bool IsNull(string value)
         {
             return string.IsNullOrWhiteSpace(value) || string.Equals(value, "null", StringComparison.OrdinalIgnoreCase);
         }
+
+        /// <summary>
+        /// Extracts the raw (URL-encoded) value of a query parameter from a query string.
+        /// Preserves special characters like & in filter values (e.g., %26 stays as %26).
+        /// 
+        /// IMPORTANT: This method assumes the input queryString is a raw, URL-encoded query string
+        /// where special characters in parameter values are encoded (e.g., & is %26, space is %20).
+        /// It splits on unencoded '&' characters which are parameter separators in the URL standard.
+        /// If the queryString has already been decoded, this method will not work correctly.
+        /// </summary>
+        /// <param name="queryString">Raw URL-encoded query string (e.g., "?$filter=title%20eq%20%27A%26B%27")</param>
+        /// <param name="parameterName">The parameter name to extract (e.g., "$filter")</param>
+        /// <returns>The raw encoded value of the parameter, or null if not found</returns>
+        internal static string? ExtractRawQueryParameter(string queryString, string parameterName)
+        {
+            if (string.IsNullOrWhiteSpace(queryString))
+            {
+                return null;
+            }
+
+            // Split on '&' which are parameter separators in properly URL-encoded query strings.
+            // Any '&' characters within parameter values will be encoded as %26.
+            foreach (string param in queryString.TrimStart('?').Split('&'))
+            {
+                int idx = param.IndexOf('=');
+                if (idx >= 0 && param.Substring(0, idx).Equals(parameterName, StringComparison.OrdinalIgnoreCase))
+                {
+                    return idx < param.Length - 1 ? param.Substring(idx + 1) : string.Empty;
+                }
+            }
+
+            return null;
+        }
     }
 }

src/Core/Services/RestService.cs

@@ -174,6 +174,7 @@ RequestValidator requestValidator
 
                 if (!string.IsNullOrWhiteSpace(queryString))
                 {
+                    context.RawQueryString = queryString;
                     context.ParsedQueryString = HttpUtility.ParseQueryString(queryString);
                     RequestParser.ParseQueryString(context, sqlMetadataProvider);
                 }
@@ -277,6 +278,7 @@ private void PopulateStoredProcedureContext(
                     // So, $filter will be treated as any other parameter (inevitably will raise a Bad Request)
                     if (!string.IsNullOrWhiteSpace(queryString))
                     {
+                        context.RawQueryString = queryString;
                         context.ParsedQueryString = HttpUtility.ParseQueryString(queryString);
                     }
 

src/Service.Tests/SqlTests/RestApiTests/Find/DwSqlFindApiTests.cs

@@ -221,6 +221,12 @@ public class DwSqlFindApiTests : FindApiTestBase
                 $"WHERE (NOT (id < 3) OR id < 4) OR NOT (title = 'Awesome book') " +
                 $"FOR JSON PATH, INCLUDE_NULL_VALUES"
             },
+            {
+                "FindTestWithFilterContainingSpecialCharacters",
+                $"SELECT * FROM { _integrationTableName } " +
+                $"WHERE title = 'SOME%CONN' " +
+                $"FOR JSON PATH, INCLUDE_NULL_VALUES"
+            },
             {
                 "FindTestWithPrimaryKeyContainingForeignKey",
                 $"SELECT [id], [content] FROM reviews " +

src/Service.Tests/SqlTests/RestApiTests/Find/FindApiTestBase.cs

@@ -693,6 +693,23 @@ await SetupAndRunRestApiTest(
             );
         }
 
+        /// <summary>
+        /// Tests the REST Api for Find operation with a filter containing special characters
+        /// that need to be URL-encoded. Uses existing book with '%' character (SOME%CONN).
+        /// This validates that the fix for the double-decoding issue is working correctly.
+        /// </summary>
+        [TestMethod]
+        public async Task FindTestWithFilterContainingSpecialCharacters()
+        {
+            // Testing with SOME%CONN - the %25 is URL-encoded %
+            await SetupAndRunRestApiTest(
+                primaryKeyRoute: string.Empty,
+                queryString: "?$filter=title%20eq%20%27SOME%25CONN%27",
+                entityNameOrPath: _integrationEntityName,
+                sqlQuery: GetQuery(nameof(FindTestWithFilterContainingSpecialCharacters))
+            );
+        }
+
         /// <summary>
         /// Tests the REST Api for Find operation where we compare one field
         /// to the bool returned from another comparison.

src/Service.Tests/SqlTests/RestApiTests/Find/MsSqlFindApiTests.cs

@@ -228,6 +228,12 @@ public class MsSqlFindApiTests : FindApiTestBase
                 $"WHERE (NOT (id < 3) OR id < 4) OR NOT (title = 'Awesome book') " +
                 $"FOR JSON PATH, INCLUDE_NULL_VALUES"
             },
+            {
+                "FindTestWithFilterContainingSpecialCharacters",
+                $"SELECT * FROM { _integrationTableName } " +
+                $"WHERE title = 'SOME%CONN' " +
+                $"FOR JSON PATH, INCLUDE_NULL_VALUES"
+            },
             {
                 "FindTestWithPrimaryKeyContainingForeignKey",
                 $"SELECT [id], [content] FROM reviews " +

src/Service.Tests/SqlTests/RestApiTests/Find/MySqlFindApiTests.cs

@@ -397,6 +397,18 @@ ORDER BY id asc
                     ) AS subq
                 "
             },
+            {
+                "FindTestWithFilterContainingSpecialCharacters",
+                @"
+                    SELECT JSON_ARRAYAGG(JSON_OBJECT('id', id, 'title', title, 'publisher_id', publisher_id)) AS data
+                    FROM (
+                        SELECT *
+                        FROM " + _integrationTableName + @"
+                        WHERE title = 'SOME%CONN'
+                        ORDER BY id asc
+                    ) AS subq
+                "
+            },
             {
                 "FindTestWithFilterQueryStringBoolResultFilter",
                 @"

src/Service.Tests/SqlTests/RestApiTests/Find/PostgreSqlFindApiTests.cs

@@ -411,6 +411,17 @@ SELECT json_agg(to_jsonb(subq)) AS data
                       ORDER BY id asc
                   ) AS subq"
             },
+            {
+                "FindTestWithFilterContainingSpecialCharacters",
+                @"
+                  SELECT json_agg(to_jsonb(subq)) AS data
+                  FROM (
+                      SELECT *
+                      FROM " + _integrationTableName + @"
+                      WHERE title = 'SOME%CONN'
+                      ORDER BY id asc
+                  ) AS subq"
+            },
             {
                 "FindTestWithPrimaryKeyContainingForeignKey",
                 @"

src/Service.Tests/UnitTests/RequestParserUnitTests.cs

@@ -0,0 +1,80 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Azure.DataApiBuilder.Core.Parsers;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Azure.DataApiBuilder.Service.Tests.UnitTests
+{
+    /// <summary>
+    /// Test class for RequestParser utility methods.
+    /// Specifically tests the ExtractRawQueryParameter method which preserves
+    /// URL encoding for special characters in query parameters.
+    /// </summary>
+    [TestClass]
+    public class RequestParserUnitTests
+    {
+        /// <summary>
+        /// Tests that ExtractRawQueryParameter correctly extracts URL-encoded
+        /// parameter values, preserving special characters like ampersand (&).
+        /// </summary>
+        [DataTestMethod]
+        [DataRow("?$filter=region%20eq%20%27filter%20%26%20test%27", "$filter", "region%20eq%20%27filter%20%26%20test%27", DisplayName = "Extract filter with encoded ampersand (&)")]
+        [DataRow("?$filter=title%20eq%20%27A%20%26%20B%27&$select=id", "$filter", "title%20eq%20%27A%20%26%20B%27", DisplayName = "Extract filter with ampersand and other params")]
+        [DataRow("?$select=id&$filter=name%20eq%20%27test%27", "$filter", "name%20eq%20%27test%27", DisplayName = "Extract filter when not first parameter")]
+        [DataRow("?$orderby=name%20asc", "$orderby", "name%20asc", DisplayName = "Extract orderby parameter")]
+        [DataRow("?param1=value1&param2=value%26with%26ampersands", "param2", "value%26with%26ampersands", DisplayName = "Extract parameter with multiple ampersands")]
+        [DataRow("$filter=title%20eq%20%27test%27", "$filter", "title%20eq%20%27test%27", DisplayName = "Extract without leading question mark")]
+        [DataRow("?$filter=", "$filter", "", DisplayName = "Extract empty filter value")]
+        [DataRow("?$filter=name%20eq%20%27test%3D123%27", "$filter", "name%20eq%20%27test%3D123%27", DisplayName = "Extract filter with encoded equals sign (=)")]
+        [DataRow("?$filter=url%20eq%20%27http%3A%2F%2Fexample.com%3Fkey%3Dvalue%27", "$filter", "url%20eq%20%27http%3A%2F%2Fexample.com%3Fkey%3Dvalue%27", DisplayName = "Extract filter with encoded URL (: / ?)")]
+        [DataRow("?$filter=text%20eq%20%27A%2BB%27", "$filter", "text%20eq%20%27A%2BB%27", DisplayName = "Extract filter with encoded plus sign (+)")]
+        [DataRow("?$filter=value%20eq%20%2750%25%27", "$filter", "value%20eq%20%2750%25%27", DisplayName = "Extract filter with encoded percent sign (%)")]
+        [DataRow("?$filter=tag%20eq%20%27%23hashtag%27", "$filter", "tag%20eq%20%27%23hashtag%27", DisplayName = "Extract filter with encoded hash (#)")]
+        [DataRow("?$filter=expr%20eq%20%27a%3Cb%3Ed%27", "$filter", "expr%20eq%20%27a%3Cb%3Ed%27", DisplayName = "Extract filter with encoded less-than and greater-than (< >)")]
+        public void ExtractRawQueryParameter_PreservesEncoding(string queryString, string parameterName, string expectedValue)
+        {
+            // Call the internal method directly (no reflection needed)
+            string? result = RequestParser.ExtractRawQueryParameter(queryString, parameterName);
+
+            Assert.AreEqual(expectedValue, result,
+                $"Expected '{expectedValue}' but got '{result}' for parameter '{parameterName}' in query '{queryString}'");
+        }
+
+        /// <summary>
+        /// Tests that ExtractRawQueryParameter returns null when parameter is not found.
+        /// </summary>
+        [DataTestMethod]
+        [DataRow("?$filter=test", "$orderby", DisplayName = "Parameter not in query string")]
+        [DataRow("", "$filter", DisplayName = "Empty query string")]
+        [DataRow(null, "$filter", DisplayName = "Null query string")]
+        [DataRow("?otherParam=value", "$filter", DisplayName = "Different parameter")]
+        public void ExtractRawQueryParameter_ReturnsNull_WhenParameterNotFound(string? queryString, string parameterName)
+        {
+            // Call the internal method directly (no reflection needed)
+            string? result = RequestParser.ExtractRawQueryParameter(queryString, parameterName);
+
+            Assert.IsNull(result,
+                $"Expected null but got '{result}' for parameter '{parameterName}' in query '{queryString}'");
+        }
+
+        /// <summary>
+        /// Tests that ExtractRawQueryParameter handles edge cases correctly:
+        /// - Duplicate parameters (returns first occurrence)
+        /// - Case-insensitive parameter name matching
+        /// - Malformed query strings with unencoded ampersands
+        /// </summary>
+        [DataTestMethod]
+        [DataRow("?$filter=value&$filter=anothervalue", "$filter", "value", DisplayName = "Multiple same parameters - returns first")]
+        [DataRow("?$FILTER=value", "$filter", "value", DisplayName = "Case insensitive parameter matching")]
+        [DataRow("?param=value1&value2", "param", "value1", DisplayName = "Value with unencoded ampersand after parameter")]
+        public void ExtractRawQueryParameter_HandlesEdgeCases(string queryString, string parameterName, string expectedValue)
+        {
+            // Call the internal method directly (no reflection needed)
+            string? result = RequestParser.ExtractRawQueryParameter(queryString, parameterName);
+
+            Assert.AreEqual(expectedValue, result,
+                $"Expected '{expectedValue}' but got '{result}' for parameter '{parameterName}' in query '{queryString}'");
+        }
+    }
+}