From 9fa2a991e1c87cc7196dc2e946084fa7c82db150 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Jan 2026 16:24:38 +0000 Subject: [PATCH 1/4] Initial plan From 40bd128d000098fdc981154283e7a60560f28132 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Jan 2026 16:30:38 +0000 Subject: [PATCH 2/4] Add comprehensive datetime query filter documentation and tests Co-authored-by: philnach <19275540+philnach@users.noreply.github.com> --- .../AzureTableAPIDataSourceExtensionTests.cs | 112 ++++++++++++++++++ Extensions/AzureTableAPI/README.md | 87 ++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 Extensions/AzureTableAPI/Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests/AzureTableAPIDataSourceExtensionTests.cs diff --git a/Extensions/AzureTableAPI/Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests/AzureTableAPIDataSourceExtensionTests.cs b/Extensions/AzureTableAPI/Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests/AzureTableAPIDataSourceExtensionTests.cs new file mode 100644 index 0000000..8f878ff --- /dev/null +++ b/Extensions/AzureTableAPI/Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests/AzureTableAPIDataSourceExtensionTests.cs @@ -0,0 +1,112 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Cosmos.DataTransfer.AzureTableAPIExtension.Settings; +using Microsoft.Extensions.Configuration; + +namespace Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests +{ + [TestClass] + public class AzureTableAPIDataSourceExtensionTests + { + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_CanBeNull() + { + var settings = new AzureTableAPIDataSourceSettings(); + + Assert.IsNull(settings.QueryFilter, "QueryFilter should be null by default"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_CanBeSet() + { + var settings = new AzureTableAPIDataSourceSettings() + { + QueryFilter = "PartitionKey eq 'test'" + }; + + Assert.AreEqual("PartitionKey eq 'test'", settings.QueryFilter, "QueryFilter should be settable"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationBasic() + { + // Test basic filter deserialization + var json = """{"QueryFilter": "PartitionKey eq 'WI'"}"""; + var config = new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) + .Build(); + var settings = config.Get(); + + Assert.AreEqual("PartitionKey eq 'WI'", settings?.QueryFilter, "QueryFilter should be deserialized from JSON"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetime() + { + // Test datetime filter with JSON-escaped single quotes + var json = """{"QueryFilter": "Timestamp eq datetime\u00272023-01-12T16:53:31.1714422Z\u0027"}"""; + var config = new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) + .Build(); + var settings = config.Get(); + + Assert.AreEqual("Timestamp eq datetime'2023-01-12T16:53:31.1714422Z'", settings?.QueryFilter, + "QueryFilter with JSON-escaped datetime should be correctly deserialized"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeGreaterThan() + { + // Test datetime filter with 'ge' (greater than or equal) operator + var json = """{"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"}"""; + var config = new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) + .Build(); + var settings = config.Get(); + + Assert.AreEqual("Timestamp ge datetime'2023-05-15T03:30:32.663Z'", settings?.QueryFilter, + "QueryFilter with 'ge' datetime operator should be correctly deserialized"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeLessThan() + { + // Test datetime filter with 'lt' (less than) operator + var json = """{"QueryFilter": "Timestamp lt datetime\u00272024-12-08T06:06:00.976Z\u0027"}"""; + var config = new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) + .Build(); + var settings = config.Get(); + + Assert.AreEqual("Timestamp lt datetime'2024-12-08T06:06:00.976Z'", settings?.QueryFilter, + "QueryFilter with 'lt' datetime operator should be correctly deserialized"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeRange() + { + // Test datetime filter with range (combining 'ge' and 'lt') + var json = """{"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"}"""; + var config = new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) + .Build(); + var settings = config.Get(); + + Assert.AreEqual("Timestamp ge datetime'2023-01-01T00:00:00Z' and Timestamp lt datetime'2024-01-01T00:00:00Z'", settings?.QueryFilter, + "QueryFilter with datetime range should be correctly deserialized"); + } + + [TestMethod] + public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationCombinedFilters() + { + // Test combining partition key filter with datetime filter + var json = """{"QueryFilter": "PartitionKey eq \u0027users\u0027 and Timestamp ge datetime\u00272023-05-15T00:00:00Z\u0027"}"""; + var config = new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) + .Build(); + var settings = config.Get(); + + Assert.AreEqual("PartitionKey eq 'users' and Timestamp ge datetime'2023-05-15T00:00:00Z'", settings?.QueryFilter, + "QueryFilter combining partition key and datetime should be correctly deserialized"); + } + } +} diff --git a/Extensions/AzureTableAPI/README.md b/Extensions/AzureTableAPI/README.md index 58b423e..89d1b06 100644 --- a/Extensions/AzureTableAPI/README.md +++ b/Extensions/AzureTableAPI/README.md @@ -28,6 +28,55 @@ The following setting is supported for the Source: - `QueryFilter` - This enables you to specify an OData filter to be applied to the data being retrieved by the AzureTableAPI Source. This is used in cases where only a subset of data from the source Table is needed in the migration. Example usage to query a subset of entities from the source table: `PartitionKey eq 'foo'`. +#### Query Filter Examples + +The `QueryFilter` setting supports OData filter syntax for querying Azure Table API entities. Below are examples of common filter patterns: + +**Basic Filters:** +```json +"QueryFilter": "PartitionKey eq 'WI'" +``` + +**DateTime Filters:** + +When filtering by `Timestamp` or other datetime properties, you must use the `datetime` prefix with ISO 8601 format timestamps. In JSON configuration files, single quotes around the datetime value must be JSON-escaped as `\u0027`: + +```json +"QueryFilter": "Timestamp eq datetime\u00272023-01-12T16:53:31.1714422Z\u0027" +``` + +```json +"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027" +``` + +```json +"QueryFilter": "Timestamp lt datetime\u00272024-12-08T06:06:00.976Z\u0027" +``` + +**DateTime Range Filters:** + +To filter entities within a date range, combine multiple conditions with `and`: + +```json +"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027" +``` + +**Combined Filters:** + +You can combine partition key filters with datetime filters for more efficient queries: + +```json +"QueryFilter": "PartitionKey eq \u0027users\u0027 and Timestamp ge datetime\u00272023-05-15T00:00:00Z\u0027" +``` + +> **Important Notes:** +> - DateTime values must be in ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.fffZ` +> - The `datetime` prefix is required before the timestamp value +> - Single quotes around datetime values must be JSON-escaped as `\u0027` in JSON configuration files +> - The `Z` suffix indicates UTC time +> - For better query performance, include `PartitionKey` in your filter when possible +> - Supported datetime operators: `eq` (equal), `ne` (not equal), `gt` (greater than), `ge` (greater than or equal), `lt` (less than), `le` (less than or equal) + ### Additional Sink Settings The AzureTableAPI Sink extension has additional settings that can be configured for writing Table entities. @@ -106,3 +155,41 @@ The following are a couple example `settings.json` files for configuring the Azu "MaxConcurrentEntityWrites": 5 } ``` + +### Example DateTime Filter Configurations + +The following examples demonstrate how to use datetime filters in the `QueryFilter` setting: + +**Example 1: Filter entities modified after a specific date** + +```json +{ + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;EndpointSuffix=core.windows.net", + "Table": "SourceTable1", + "PartitionKeyFieldName": "PartitionKey", + "RowKeyFieldName": "RowKey", + "QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027" +} +``` + +**Example 2: Filter entities within a date range** + +```json +{ + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;EndpointSuffix=core.windows.net", + "Table": "SourceTable1", + "QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027" +} +``` + +**Example 3: Combine partition key with datetime filter** + +```json +{ + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;EndpointSuffix=core.windows.net", + "Table": "SourceTable1", + "PartitionKeyFieldName": "State", + "RowKeyFieldName": "id", + "QueryFilter": "PartitionKey eq \u0027CA\u0027 and Timestamp ge datetime\u00272023-06-01T00:00:00Z\u0027" +} +``` From fe47a49cd15b470000af2aa85637a9d769c3f394 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Jan 2026 16:31:40 +0000 Subject: [PATCH 3/4] Add AzureTableAPI datetime filter example to ExampleConfigs Co-authored-by: philnach <19275540+philnach@users.noreply.github.com> --- ExampleConfigs.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ExampleConfigs.md b/ExampleConfigs.md index d67810c..fbc01b2 100644 --- a/ExampleConfigs.md +++ b/ExampleConfigs.md @@ -132,6 +132,28 @@ } ``` +## AzureTableAPI to JSON (with DateTime Filter) + +```json +{ + "Source": "AzureTableAPI", + "Sink": "JSON", + "SourceSettings": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;EndpointSuffix=core.windows.net", + "Table": "SourceTable1", + "PartitionKeyFieldName": "PartitionKey", + "RowKeyFieldName": "RowKey", + "QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027" + }, + "SinkSettings": { + "FilePath": "D:\\output\\filtered-data.json", + "Indented": true + } +} +``` + +> **Note**: When using DateTime filters in the `QueryFilter` property, single quotes around the datetime value must be JSON-escaped as `\u0027`. The datetime must be in ISO 8601 format with the `datetime` prefix. + ## Cosmos-NoSQL to SqlServer ```json From 225fab4508de180fd6aeb40331b1abbf3d9e7430 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Jan 2026 17:32:38 +0000 Subject: [PATCH 4/4] Add troubleshooting table analyzing all query filters from issue Co-authored-by: philnach <19275540+philnach@users.noreply.github.com> --- Extensions/AzureTableAPI/README.md | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Extensions/AzureTableAPI/README.md b/Extensions/AzureTableAPI/README.md index 89d1b06..9e292d2 100644 --- a/Extensions/AzureTableAPI/README.md +++ b/Extensions/AzureTableAPI/README.md @@ -77,6 +77,40 @@ You can combine partition key filters with datetime filters for more efficient q > - For better query performance, include `PartitionKey` in your filter when possible > - Supported datetime operators: `eq` (equal), `ne` (not equal), `gt` (greater than), `ge` (greater than or equal), `lt` (less than), `le` (less than or equal) +#### Troubleshooting Common DateTime Filter Issues + +The following table analyzes common mistakes when specifying datetime filters. Each row shows a query that was attempted and identifies the specific issues: + +| Query Filter | Missing `datetime` Prefix | Wrong Date Format | Incorrect Encoding | Result | +|--------------|--------------------------|-------------------|-------------------|---------| +| `"QueryFilter": "Timestamp ge datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* | +| `"QueryFilter": "Timestamp le datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* | +| `"QueryFilter": "Timestamp eq datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* | +| `"QueryFilter": "Timestamp gt datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* | +| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax | +| `"QueryFilter": "Timestamp le datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax | +| `"QueryFilter": "Timestamp gt datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax | +| `"QueryFilter": "Timestamp eq datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax | +| `"QueryFilter": "Timestamp ge datetime'\u00272023-05-17T03:06:07.691Z\u0027'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Extra quote at end | ❌ Invalid syntax | +| `"QueryFilter": "Timestamp eq \u00272023-05-17T03:06:07.691Z\u0027"` | ❌ Missing | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ❌ No data (invalid) | +| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:10:39.058Z\u002B00:00'"` | ✅ Correct | ❌ Invalid timezone format | ❌ Space before quote, mixed encoding | ❌ Transfer fails | +| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:10:39.058Z\u002B00'"` | ✅ Correct | ❌ Invalid timezone format | ❌ Space before quote, mixed encoding | ❌ Transfer fails | +| `"QueryFilter": "Timestamp ge datetime 2023-05-17T03:10:39.058Z\u002B00"` | ✅ Correct | ❌ Invalid timezone format | ❌ No quotes around datetime | ❌ Transfer fails | +| `"QueryFilter": "Timestamp ge datetime'u00272023-05-17T03:06:07.691Zu0027'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Wrong escape sequence (missing `\`) | ❌ Transfer fails | +| `"QueryFilter": "Timestamp eq '\u00272023-05-17T03:06:07.691Z\u0027'"` | ❌ Missing | ✅ Correct (ISO 8601) | ❌ Extra quote at end | ❌ Transfer fails | + +\* **Note**: The first four queries are syntactically correct. If they returned no data, it may be because: +- No entities exist with timestamps matching the filter criteria +- The specific timestamp value doesn't match any entity timestamps (especially with `eq` operator) +- For exact matches with `eq`, consider using `ge` (greater than or equal) or `le` (less than or equal) operators instead, as table timestamps include high-precision fractional seconds + +**Key Takeaways:** +1. Always use `datetime` prefix before the timestamp value +2. Always use ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.fffZ` +3. Always JSON-escape single quotes as `\u0027` (not literal `'` characters) +4. No spaces between `datetime` and the opening quote +5. Timezone should be `Z` for UTC, not `+00:00` or other formats + ### Additional Sink Settings The AzureTableAPI Sink extension has additional settings that can be configured for writing Table entities.