Skip to content

Commit 095de3d

Browse files
markjbrownmarkjbrown
authored
Merge branch 'main' into copilot/add-sql-server-write-mode
2 parents 7403a1e + f28c644 commit 095de3d

File tree

5 files changed

+395
-1
lines changed

5 files changed

+395
-1
lines changed

‎ExampleConfigs.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,28 @@
132132
}
133133
```
134134

135+
## AzureTableAPI to JSON (with DateTime Filter)
136+
137+
```json
138+
{
139+
"Source": "AzureTableAPI",
140+
"Sink": "JSON",
141+
"SourceSettings": {
142+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
143+
"Table": "SourceTable1",
144+
"PartitionKeyFieldName": "PartitionKey",
145+
"RowKeyFieldName": "RowKey",
146+
"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"
147+
},
148+
"SinkSettings": {
149+
"FilePath": "D:\\output\\filtered-data.json",
150+
"Indented": true
151+
}
152+
}
153+
```
154+
155+
> **Note**: When using DateTime filters in the `QueryFilter` property, single quotes around the datetime value must be JSON-escaped as `\u0027`. The datetime must be in ISO 8601 format with the `datetime` prefix.
156+
135157
## Cosmos-NoSQL to SqlServer
136158

137159
```json

‎Extensions/AzureTableAPI/Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests/AzureTableAPIDataSourceExtensionTests.cs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
using Microsoft.VisualStudio.TestTools.UnitTesting;
2+
using Cosmos.DataTransfer.AzureTableAPIExtension.Settings;
3+
using Microsoft.Extensions.Configuration;
4+
5+
namespace Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests
6+
{
7+
[TestClass]
8+
public class AzureTableAPIDataSourceExtensionTests
9+
{
10+
[TestMethod]
11+
public void AzureTableAPIDataSourceSettings_QueryFilter_CanBeNull()
12+
{
13+
var settings = new AzureTableAPIDataSourceSettings();
14+
15+
Assert.IsNull(settings.QueryFilter, "QueryFilter should be null by default");
16+
}
17+
18+
[TestMethod]
19+
public void AzureTableAPIDataSourceSettings_QueryFilter_CanBeSet()
20+
{
21+
var settings = new AzureTableAPIDataSourceSettings()
22+
{
23+
QueryFilter = "PartitionKey eq 'test'"
24+
};
25+
26+
Assert.AreEqual("PartitionKey eq 'test'", settings.QueryFilter, "QueryFilter should be settable");
27+
}
28+
29+
[TestMethod]
30+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationBasic()
31+
{
32+
// Test basic filter deserialization
33+
var json = """{"QueryFilter": "PartitionKey eq 'WI'"}""";
34+
var config = new ConfigurationBuilder()
35+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
36+
.Build();
37+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
38+
39+
Assert.AreEqual("PartitionKey eq 'WI'", settings?.QueryFilter, "QueryFilter should be deserialized from JSON");
40+
}
41+
42+
[TestMethod]
43+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetime()
44+
{
45+
// Test datetime filter with JSON-escaped single quotes
46+
var json = """{"QueryFilter": "Timestamp eq datetime\u00272023-01-12T16:53:31.1714422Z\u0027"}""";
47+
var config = new ConfigurationBuilder()
48+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
49+
.Build();
50+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
51+
52+
Assert.AreEqual("Timestamp eq datetime'2023-01-12T16:53:31.1714422Z'", settings?.QueryFilter,
53+
"QueryFilter with JSON-escaped datetime should be correctly deserialized");
54+
}
55+
56+
[TestMethod]
57+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeGreaterThan()
58+
{
59+
// Test datetime filter with 'ge' (greater than or equal) operator
60+
var json = """{"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"}""";
61+
var config = new ConfigurationBuilder()
62+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
63+
.Build();
64+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
65+
66+
Assert.AreEqual("Timestamp ge datetime'2023-05-15T03:30:32.663Z'", settings?.QueryFilter,
67+
"QueryFilter with 'ge' datetime operator should be correctly deserialized");
68+
}
69+
70+
[TestMethod]
71+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeLessThan()
72+
{
73+
// Test datetime filter with 'lt' (less than) operator
74+
var json = """{"QueryFilter": "Timestamp lt datetime\u00272024-12-08T06:06:00.976Z\u0027"}""";
75+
var config = new ConfigurationBuilder()
76+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
77+
.Build();
78+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
79+
80+
Assert.AreEqual("Timestamp lt datetime'2024-12-08T06:06:00.976Z'", settings?.QueryFilter,
81+
"QueryFilter with 'lt' datetime operator should be correctly deserialized");
82+
}
83+
84+
[TestMethod]
85+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeRange()
86+
{
87+
// Test datetime filter with range (combining 'ge' and 'lt')
88+
var json = """{"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"}""";
89+
var config = new ConfigurationBuilder()
90+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
91+
.Build();
92+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
93+
94+
Assert.AreEqual("Timestamp ge datetime'2023-01-01T00:00:00Z' and Timestamp lt datetime'2024-01-01T00:00:00Z'", settings?.QueryFilter,
95+
"QueryFilter with datetime range should be correctly deserialized");
96+
}
97+
98+
[TestMethod]
99+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationCombinedFilters()
100+
{
101+
// Test combining partition key filter with datetime filter
102+
var json = """{"QueryFilter": "PartitionKey eq \u0027users\u0027 and Timestamp ge datetime\u00272023-05-15T00:00:00Z\u0027"}""";
103+
var config = new ConfigurationBuilder()
104+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
105+
.Build();
106+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
107+
108+
Assert.AreEqual("PartitionKey eq 'users' and Timestamp ge datetime'2023-05-15T00:00:00Z'", settings?.QueryFilter,
109+
"QueryFilter combining partition key and datetime should be correctly deserialized");
110+
}
111+
}
112+
}

‎Extensions/AzureTableAPI/README.md‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,89 @@ The following setting is supported for the Source:
2828

2929
- `QueryFilter` - This enables you to specify an OData filter to be applied to the data being retrieved by the AzureTableAPI Source. This is used in cases where only a subset of data from the source Table is needed in the migration. Example usage to query a subset of entities from the source table: `PartitionKey eq 'foo'`.
3030

31+
#### Query Filter Examples
32+
33+
The `QueryFilter` setting supports OData filter syntax for querying Azure Table API entities. Below are examples of common filter patterns:
34+
35+
**Basic Filters:**
36+
```json
37+
"QueryFilter": "PartitionKey eq 'WI'"
38+
```
39+
40+
**DateTime Filters:**
41+
42+
When filtering by `Timestamp` or other datetime properties, you must use the `datetime` prefix with ISO 8601 format timestamps. In JSON configuration files, single quotes around the datetime value must be JSON-escaped as `\u0027`:
43+
44+
```json
45+
"QueryFilter": "Timestamp eq datetime\u00272023-01-12T16:53:31.1714422Z\u0027"
46+
```
47+
48+
```json
49+
"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"
50+
```
51+
52+
```json
53+
"QueryFilter": "Timestamp lt datetime\u00272024-12-08T06:06:00.976Z\u0027"
54+
```
55+
56+
**DateTime Range Filters:**
57+
58+
To filter entities within a date range, combine multiple conditions with `and`:
59+
60+
```json
61+
"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"
62+
```
63+
64+
**Combined Filters:**
65+
66+
You can combine partition key filters with datetime filters for more efficient queries:
67+
68+
```json
69+
"QueryFilter": "PartitionKey eq \u0027users\u0027 and Timestamp ge datetime\u00272023-05-15T00:00:00Z\u0027"
70+
```
71+
72+
> **Important Notes:**
73+
> - DateTime values must be in ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.fffZ`
74+
> - The `datetime` prefix is required before the timestamp value
75+
> - Single quotes around datetime values must be JSON-escaped as `\u0027` in JSON configuration files
76+
> - The `Z` suffix indicates UTC time
77+
> - For better query performance, include `PartitionKey` in your filter when possible
78+
> - Supported datetime operators: `eq` (equal), `ne` (not equal), `gt` (greater than), `ge` (greater than or equal), `lt` (less than), `le` (less than or equal)
79+
80+
#### Troubleshooting Common DateTime Filter Issues
81+
82+
The following table analyzes common mistakes when specifying datetime filters. Each row shows a query that was attempted and identifies the specific issues:
83+
84+
| Query Filter | Missing `datetime` Prefix | Wrong Date Format | Incorrect Encoding | Result |
85+
|--------------|--------------------------|-------------------|-------------------|---------|
86+
| `"QueryFilter": "Timestamp ge datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
87+
| `"QueryFilter": "Timestamp le datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
88+
| `"QueryFilter": "Timestamp eq datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
89+
| `"QueryFilter": "Timestamp gt datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
90+
| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
91+
| `"QueryFilter": "Timestamp le datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
92+
| `"QueryFilter": "Timestamp gt datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
93+
| `"QueryFilter": "Timestamp eq datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
94+
| `"QueryFilter": "Timestamp ge datetime'\u00272023-05-17T03:06:07.691Z\u0027'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Extra quote at end | ❌ Invalid syntax |
95+
| `"QueryFilter": "Timestamp eq \u00272023-05-17T03:06:07.691Z\u0027"` | ❌ Missing | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ❌ No data (invalid) |
96+
| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:10:39.058Z\u002B00:00'"` | ✅ Correct | ❌ Invalid timezone format | ❌ Space before quote, mixed encoding | ❌ Transfer fails |
97+
| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:10:39.058Z\u002B00'"` | ✅ Correct | ❌ Invalid timezone format | ❌ Space before quote, mixed encoding | ❌ Transfer fails |
98+
| `"QueryFilter": "Timestamp ge datetime 2023-05-17T03:10:39.058Z\u002B00"` | ✅ Correct | ❌ Invalid timezone format | ❌ No quotes around datetime | ❌ Transfer fails |
99+
| `"QueryFilter": "Timestamp ge datetime'u00272023-05-17T03:06:07.691Zu0027'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Wrong escape sequence (missing `\`) | ❌ Transfer fails |
100+
| `"QueryFilter": "Timestamp eq '\u00272023-05-17T03:06:07.691Z\u0027'"` | ❌ Missing | ✅ Correct (ISO 8601) | ❌ Extra quote at end | ❌ Transfer fails |
101+
102+
\* **Note**: The first four queries are syntactically correct. If they returned no data, it may be because:
103+
- No entities exist with timestamps matching the filter criteria
104+
- The specific timestamp value doesn't match any entity timestamps (especially with `eq` operator)
105+
- For exact matches with `eq`, consider using `ge` (greater than or equal) or `le` (less than or equal) operators instead, as table timestamps include high-precision fractional seconds
106+
107+
**Key Takeaways:**
108+
1. Always use `datetime` prefix before the timestamp value
109+
2. Always use ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.fffZ`
110+
3. Always JSON-escape single quotes as `\u0027` (not literal `'` characters)
111+
4. No spaces between `datetime` and the opening quote
112+
5. Timezone should be `Z` for UTC, not `+00:00` or other formats
113+
31114
### Additional Sink Settings
32115

33116
The AzureTableAPI Sink extension has additional settings that can be configured for writing Table entities.
@@ -106,3 +189,41 @@ The following are a couple example `settings.json` files for configuring the Azu
106189
"MaxConcurrentEntityWrites": 5
107190
}
108191
```
192+
193+
### Example DateTime Filter Configurations
194+
195+
The following examples demonstrate how to use datetime filters in the `QueryFilter` setting:
196+
197+
**Example 1: Filter entities modified after a specific date**
198+
199+
```json
200+
{
201+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
202+
"Table": "SourceTable1",
203+
"PartitionKeyFieldName": "PartitionKey",
204+
"RowKeyFieldName": "RowKey",
205+
"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"
206+
}
207+
```
208+
209+
**Example 2: Filter entities within a date range**
210+
211+
```json
212+
{
213+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
214+
"Table": "SourceTable1",
215+
"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"
216+
}
217+
```
218+
219+
**Example 3: Combine partition key with datetime filter**
220+
221+
```json
222+
{
223+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
224+
"Table": "SourceTable1",
225+
"PartitionKeyFieldName": "State",
226+
"RowKeyFieldName": "id",
227+
"QueryFilter": "PartitionKey eq \u0027CA\u0027 and Timestamp ge datetime\u00272023-06-01T00:00:00Z\u0027"
228+
}
229+
```

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosDataSinkExtensionTests.cs‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,5 +366,120 @@ public static bool HasProperty(object obj, string name)
366366
var values = (IDictionary<string, object>)obj;
367367
return values.ContainsKey(name);
368368
}
369+
370+
private static string? InvokeGetPropertyValue(ExpandoObject item, string propertyName)
371+
{
372+
var method = typeof(CosmosDataSinkExtension).GetMethod("GetPropertyValue",
373+
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static);
374+
return method?.Invoke(null, new object[] { item, propertyName }) as string;
375+
}
376+
377+
[TestMethod]
378+
public void GetPropertyValue_WithSimpleProperty_ReturnsValue()
379+
{
380+
// Arrange
381+
var expando = new ExpandoObject();
382+
var dict = (IDictionary<string, object?>)expando;
383+
dict["id"] = "test-id-123";
384+
dict["name"] = "test-name";
385+
386+
// Act
387+
var result = InvokeGetPropertyValue(expando, "id");
388+
389+
// Assert
390+
Assert.IsNotNull(result);
391+
Assert.AreEqual("test-id-123", result);
392+
}
393+
394+
[TestMethod]
395+
public void GetPropertyValue_WithNestedProperty_ReturnsValue()
396+
{
397+
// Arrange - Create nested structure matching the issue example
398+
var expando = new ExpandoObject();
399+
var dict = (IDictionary<string, object?>)expando;
400+
dict["id"] = "test-id";
401+
402+
var nestedExpando = new ExpandoObject();
403+
var nestedDict = (IDictionary<string, object?>)nestedExpando;
404+
nestedDict["partitionkeyvalue2"] = "guid-value-123";
405+
nestedDict["somevalue4"] = "other-guid";
406+
nestedDict["UserName"] = "testuser";
407+
408+
dict["partitionkeyvalue1"] = nestedExpando;
409+
410+
// Act
411+
var result = InvokeGetPropertyValue(expando, "partitionkeyvalue1/partitionkeyvalue2");
412+
413+
// Assert
414+
Assert.IsNotNull(result);
415+
Assert.AreEqual("guid-value-123", result);
416+
}
417+
418+
[TestMethod]
419+
public void GetPropertyValue_WithDeeplyNestedProperty_ReturnsValue()
420+
{
421+
// Arrange - Create deeply nested structure
422+
var expando = new ExpandoObject();
423+
var dict = (IDictionary<string, object?>)expando;
424+
dict["id"] = "test-id";
425+
426+
var level1 = new ExpandoObject();
427+
var level1Dict = (IDictionary<string, object?>)level1;
428+
429+
var level2 = new ExpandoObject();
430+
var level2Dict = (IDictionary<string, object?>)level2;
431+
432+
var level3 = new ExpandoObject();
433+
var level3Dict = (IDictionary<string, object?>)level3;
434+
level3Dict["finalValue"] = "deeply-nested-value";
435+
436+
level2Dict["level3"] = level3;
437+
level1Dict["level2"] = level2;
438+
dict["level1"] = level1;
439+
440+
// Act
441+
var result = InvokeGetPropertyValue(expando, "level1/level2/level3/finalValue");
442+
443+
// Assert
444+
Assert.IsNotNull(result);
445+
Assert.AreEqual("deeply-nested-value", result);
446+
}
447+
448+
[TestMethod]
449+
public void GetPropertyValue_WithMissingNestedProperty_ReturnsNull()
450+
{
451+
// Arrange
452+
var expando = new ExpandoObject();
453+
var dict = (IDictionary<string, object?>)expando;
454+
dict["id"] = "test-id";
455+
456+
var nestedExpando = new ExpandoObject();
457+
var nestedDict = (IDictionary<string, object?>)nestedExpando;
458+
nestedDict["existingKey"] = "value";
459+
460+
dict["parent"] = nestedExpando;
461+
462+
// Act
463+
var result = InvokeGetPropertyValue(expando, "parent/nonExistentKey");
464+
465+
// Assert
466+
Assert.IsNull(result);
467+
}
468+
469+
[TestMethod]
470+
public void GetPropertyValue_WithNullIntermediateValue_ReturnsNull()
471+
{
472+
// Arrange
473+
var expando = new ExpandoObject();
474+
var dict = (IDictionary<string, object?>)expando;
475+
dict["id"] = "test-id";
476+
dict["parent"] = null;
477+
478+
// Act
479+
var result = InvokeGetPropertyValue(expando, "parent/child");
480+
481+
// Assert
482+
Assert.IsNull(result);
483+
}
369484
}
370485
}

0 commit comments

Comments
 (0)