Skip to content

Commit 39bc283

Browse files
philnachphilnach
authored
Merge branch 'main' into copilot/prevent-container-wipe-issue
2 parents 2217eab + b5622d6 commit 39bc283

File tree

24 files changed

+1625
-36
lines changed

24 files changed

+1625
-36
lines changed

‎ExampleConfigs.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,28 @@
132132
}
133133
```
134134

135+
## AzureTableAPI to JSON (with DateTime Filter)
136+
137+
```json
138+
{
139+
"Source": "AzureTableAPI",
140+
"Sink": "JSON",
141+
"SourceSettings": {
142+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
143+
"Table": "SourceTable1",
144+
"PartitionKeyFieldName": "PartitionKey",
145+
"RowKeyFieldName": "RowKey",
146+
"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"
147+
},
148+
"SinkSettings": {
149+
"FilePath": "D:\\output\\filtered-data.json",
150+
"Indented": true
151+
}
152+
}
153+
```
154+
155+
> **Note**: When using DateTime filters in the `QueryFilter` property, single quotes around the datetime value must be JSON-escaped as `\u0027`. The datetime must be in ISO 8601 format with the `datetime` prefix.
156+
135157
## Cosmos-NoSQL to SqlServer
136158

137159
```json

‎Extensions/AzureTableAPI/Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests/AzureTableAPIDataSourceExtensionTests.cs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
using Microsoft.VisualStudio.TestTools.UnitTesting;
2+
using Cosmos.DataTransfer.AzureTableAPIExtension.Settings;
3+
using Microsoft.Extensions.Configuration;
4+
5+
namespace Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests
6+
{
7+
[TestClass]
8+
public class AzureTableAPIDataSourceExtensionTests
9+
{
10+
[TestMethod]
11+
public void AzureTableAPIDataSourceSettings_QueryFilter_CanBeNull()
12+
{
13+
var settings = new AzureTableAPIDataSourceSettings();
14+
15+
Assert.IsNull(settings.QueryFilter, "QueryFilter should be null by default");
16+
}
17+
18+
[TestMethod]
19+
public void AzureTableAPIDataSourceSettings_QueryFilter_CanBeSet()
20+
{
21+
var settings = new AzureTableAPIDataSourceSettings()
22+
{
23+
QueryFilter = "PartitionKey eq 'test'"
24+
};
25+
26+
Assert.AreEqual("PartitionKey eq 'test'", settings.QueryFilter, "QueryFilter should be settable");
27+
}
28+
29+
[TestMethod]
30+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationBasic()
31+
{
32+
// Test basic filter deserialization
33+
var json = """{"QueryFilter": "PartitionKey eq 'WI'"}""";
34+
var config = new ConfigurationBuilder()
35+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
36+
.Build();
37+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
38+
39+
Assert.AreEqual("PartitionKey eq 'WI'", settings?.QueryFilter, "QueryFilter should be deserialized from JSON");
40+
}
41+
42+
[TestMethod]
43+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetime()
44+
{
45+
// Test datetime filter with JSON-escaped single quotes
46+
var json = """{"QueryFilter": "Timestamp eq datetime\u00272023-01-12T16:53:31.1714422Z\u0027"}""";
47+
var config = new ConfigurationBuilder()
48+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
49+
.Build();
50+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
51+
52+
Assert.AreEqual("Timestamp eq datetime'2023-01-12T16:53:31.1714422Z'", settings?.QueryFilter,
53+
"QueryFilter with JSON-escaped datetime should be correctly deserialized");
54+
}
55+
56+
[TestMethod]
57+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeGreaterThan()
58+
{
59+
// Test datetime filter with 'ge' (greater than or equal) operator
60+
var json = """{"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"}""";
61+
var config = new ConfigurationBuilder()
62+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
63+
.Build();
64+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
65+
66+
Assert.AreEqual("Timestamp ge datetime'2023-05-15T03:30:32.663Z'", settings?.QueryFilter,
67+
"QueryFilter with 'ge' datetime operator should be correctly deserialized");
68+
}
69+
70+
[TestMethod]
71+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeLessThan()
72+
{
73+
// Test datetime filter with 'lt' (less than) operator
74+
var json = """{"QueryFilter": "Timestamp lt datetime\u00272024-12-08T06:06:00.976Z\u0027"}""";
75+
var config = new ConfigurationBuilder()
76+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
77+
.Build();
78+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
79+
80+
Assert.AreEqual("Timestamp lt datetime'2024-12-08T06:06:00.976Z'", settings?.QueryFilter,
81+
"QueryFilter with 'lt' datetime operator should be correctly deserialized");
82+
}
83+
84+
[TestMethod]
85+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationWithDatetimeRange()
86+
{
87+
// Test datetime filter with range (combining 'ge' and 'lt')
88+
var json = """{"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"}""";
89+
var config = new ConfigurationBuilder()
90+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
91+
.Build();
92+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
93+
94+
Assert.AreEqual("Timestamp ge datetime'2023-01-01T00:00:00Z' and Timestamp lt datetime'2024-01-01T00:00:00Z'", settings?.QueryFilter,
95+
"QueryFilter with datetime range should be correctly deserialized");
96+
}
97+
98+
[TestMethod]
99+
public void AzureTableAPIDataSourceSettings_QueryFilter_JsonDeserializationCombinedFilters()
100+
{
101+
// Test combining partition key filter with datetime filter
102+
var json = """{"QueryFilter": "PartitionKey eq \u0027users\u0027 and Timestamp ge datetime\u00272023-05-15T00:00:00Z\u0027"}""";
103+
var config = new ConfigurationBuilder()
104+
.AddJsonStream(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
105+
.Build();
106+
var settings = config.Get<AzureTableAPIDataSourceSettings>();
107+
108+
Assert.AreEqual("PartitionKey eq 'users' and Timestamp ge datetime'2023-05-15T00:00:00Z'", settings?.QueryFilter,
109+
"QueryFilter combining partition key and datetime should be correctly deserialized");
110+
}
111+
}
112+
}

‎Extensions/AzureTableAPI/README.md‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,89 @@ The following setting is supported for the Source:
2828

2929
- `QueryFilter` - This enables you to specify an OData filter to be applied to the data being retrieved by the AzureTableAPI Source. This is used in cases where only a subset of data from the source Table is needed in the migration. Example usage to query a subset of entities from the source table: `PartitionKey eq 'foo'`.
3030

31+
#### Query Filter Examples
32+
33+
The `QueryFilter` setting supports OData filter syntax for querying Azure Table API entities. Below are examples of common filter patterns:
34+
35+
**Basic Filters:**
36+
```json
37+
"QueryFilter": "PartitionKey eq 'WI'"
38+
```
39+
40+
**DateTime Filters:**
41+
42+
When filtering by `Timestamp` or other datetime properties, you must use the `datetime` prefix with ISO 8601 format timestamps. In JSON configuration files, single quotes around the datetime value must be JSON-escaped as `\u0027`:
43+
44+
```json
45+
"QueryFilter": "Timestamp eq datetime\u00272023-01-12T16:53:31.1714422Z\u0027"
46+
```
47+
48+
```json
49+
"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"
50+
```
51+
52+
```json
53+
"QueryFilter": "Timestamp lt datetime\u00272024-12-08T06:06:00.976Z\u0027"
54+
```
55+
56+
**DateTime Range Filters:**
57+
58+
To filter entities within a date range, combine multiple conditions with `and`:
59+
60+
```json
61+
"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"
62+
```
63+
64+
**Combined Filters:**
65+
66+
You can combine partition key filters with datetime filters for more efficient queries:
67+
68+
```json
69+
"QueryFilter": "PartitionKey eq \u0027users\u0027 and Timestamp ge datetime\u00272023-05-15T00:00:00Z\u0027"
70+
```
71+
72+
> **Important Notes:**
73+
> - DateTime values must be in ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.fffZ`
74+
> - The `datetime` prefix is required before the timestamp value
75+
> - Single quotes around datetime values must be JSON-escaped as `\u0027` in JSON configuration files
76+
> - The `Z` suffix indicates UTC time
77+
> - For better query performance, include `PartitionKey` in your filter when possible
78+
> - Supported datetime operators: `eq` (equal), `ne` (not equal), `gt` (greater than), `ge` (greater than or equal), `lt` (less than), `le` (less than or equal)
79+
80+
#### Troubleshooting Common DateTime Filter Issues
81+
82+
The following table analyzes common mistakes when specifying datetime filters. Each row shows a query that was attempted and identifies the specific issues:
83+
84+
| Query Filter | Missing `datetime` Prefix | Wrong Date Format | Incorrect Encoding | Result |
85+
|--------------|--------------------------|-------------------|-------------------|---------|
86+
| `"QueryFilter": "Timestamp ge datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
87+
| `"QueryFilter": "Timestamp le datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
88+
| `"QueryFilter": "Timestamp eq datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
89+
| `"QueryFilter": "Timestamp gt datetime\u00272023-05-17T03:06:07.691Z\u0027"` | ✅ Correct | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ✅ Should work* |
90+
| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
91+
| `"QueryFilter": "Timestamp le datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
92+
| `"QueryFilter": "Timestamp gt datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
93+
| `"QueryFilter": "Timestamp eq datetime '2023-05-17T03:06:07.691Z'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Space before quote, not JSON-escaped | ❌ Invalid syntax |
94+
| `"QueryFilter": "Timestamp ge datetime'\u00272023-05-17T03:06:07.691Z\u0027'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Extra quote at end | ❌ Invalid syntax |
95+
| `"QueryFilter": "Timestamp eq \u00272023-05-17T03:06:07.691Z\u0027"` | ❌ Missing | ✅ Correct (ISO 8601) | ✅ Correct (`\u0027`) | ❌ No data (invalid) |
96+
| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:10:39.058Z\u002B00:00'"` | ✅ Correct | ❌ Invalid timezone format | ❌ Space before quote, mixed encoding | ❌ Transfer fails |
97+
| `"QueryFilter": "Timestamp ge datetime '2023-05-17T03:10:39.058Z\u002B00'"` | ✅ Correct | ❌ Invalid timezone format | ❌ Space before quote, mixed encoding | ❌ Transfer fails |
98+
| `"QueryFilter": "Timestamp ge datetime 2023-05-17T03:10:39.058Z\u002B00"` | ✅ Correct | ❌ Invalid timezone format | ❌ No quotes around datetime | ❌ Transfer fails |
99+
| `"QueryFilter": "Timestamp ge datetime'u00272023-05-17T03:06:07.691Zu0027'"` | ✅ Correct | ✅ Correct (ISO 8601) | ❌ Wrong escape sequence (missing `\`) | ❌ Transfer fails |
100+
| `"QueryFilter": "Timestamp eq '\u00272023-05-17T03:06:07.691Z\u0027'"` | ❌ Missing | ✅ Correct (ISO 8601) | ❌ Extra quote at end | ❌ Transfer fails |
101+
102+
\* **Note**: The first four queries are syntactically correct. If they returned no data, it may be because:
103+
- No entities exist with timestamps matching the filter criteria
104+
- The specific timestamp value doesn't match any entity timestamps (especially with `eq` operator)
105+
- For exact matches with `eq`, consider using `ge` (greater than or equal) or `le` (less than or equal) operators instead, as table timestamps include high-precision fractional seconds
106+
107+
**Key Takeaways:**
108+
1. Always use `datetime` prefix before the timestamp value
109+
2. Always use ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.fffZ`
110+
3. Always JSON-escape single quotes as `\u0027` (not literal `'` characters)
111+
4. No spaces between `datetime` and the opening quote
112+
5. Timezone should be `Z` for UTC, not `+00:00` or other formats
113+
31114
### Additional Sink Settings
32115

33116
The AzureTableAPI Sink extension has additional settings that can be configured for writing Table entities.
@@ -106,3 +189,41 @@ The following are a couple example `settings.json` files for configuring the Azu
106189
"MaxConcurrentEntityWrites": 5
107190
}
108191
```
192+
193+
### Example DateTime Filter Configurations
194+
195+
The following examples demonstrate how to use datetime filters in the `QueryFilter` setting:
196+
197+
**Example 1: Filter entities modified after a specific date**
198+
199+
```json
200+
{
201+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
202+
"Table": "SourceTable1",
203+
"PartitionKeyFieldName": "PartitionKey",
204+
"RowKeyFieldName": "RowKey",
205+
"QueryFilter": "Timestamp ge datetime\u00272023-05-15T03:30:32.663Z\u0027"
206+
}
207+
```
208+
209+
**Example 2: Filter entities within a date range**
210+
211+
```json
212+
{
213+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
214+
"Table": "SourceTable1",
215+
"QueryFilter": "Timestamp ge datetime\u00272023-01-01T00:00:00Z\u0027 and Timestamp lt datetime\u00272024-01-01T00:00:00Z\u0027"
216+
}
217+
```
218+
219+
**Example 3: Combine partition key with datetime filter**
220+
221+
```json
222+
{
223+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=<storage-account-name>;AccountKey=<key>;EndpointSuffix=core.windows.net",
224+
"Table": "SourceTable1",
225+
"PartitionKeyFieldName": "State",
226+
"RowKeyFieldName": "id",
227+
"QueryFilter": "PartitionKey eq \u0027CA\u0027 and Timestamp ge datetime\u00272023-06-01T00:00:00Z\u0027"
228+
}
229+
```

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
using System.ComponentModel.DataAnnotations;
2+
using Microsoft.Extensions.Logging;
3+
using Moq;
4+
5+
namespace Cosmos.DataTransfer.CosmosExtension.UnitTests
6+
{
7+
[TestClass]
8+
public class CertificateConfigurationTests
9+
{
10+
[TestMethod]
11+
public void CosmosSettingsBase_WithDisableSslValidation_ShouldValidateSuccessfully()
12+
{
13+
var settings = new TestableCosmosSettings
14+
{
15+
ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...",
16+
Database = "testDb",
17+
Container = "testContainer",
18+
DisableSslValidation = true
19+
};
20+
21+
var validationResults = settings.Validate(new ValidationContext(settings)).ToList();
22+
23+
Assert.AreEqual(0, validationResults.Count, "Should have no validation errors");
24+
}
25+
26+
[TestMethod]
27+
public void CosmosSettingsBase_WithDisableSslValidationFalse_ShouldUseDefaultValidation()
28+
{
29+
var settings = new TestableCosmosSettings
30+
{
31+
ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDj...",
32+
Database = "testDb",
33+
Container = "testContainer",
34+
DisableSslValidation = false
35+
};
36+
37+
var validationResults = settings.Validate(new ValidationContext(settings)).ToList();
38+
39+
Assert.AreEqual(0, validationResults.Count, "Settings should validate with DisableSslValidation=false");
40+
}
41+
42+
[TestMethod]
43+
public void CreateClient_WithDisableSslValidation_LogsWarningAndSetsCallback()
44+
{
45+
var loggerMock = new Mock<ILogger>();
46+
var settings = new TestableCosmosSettings
47+
{
48+
ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
49+
Database = "testDb",
50+
Container = "testContainer",
51+
DisableSslValidation = true
52+
};
53+
54+
var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object);
55+
56+
// Verify warning was logged when DisableSslValidation is true
57+
loggerMock.Verify(
58+
x => x.Log(
59+
LogLevel.Warning,
60+
It.IsAny<EventId>(),
61+
It.Is<It.IsAnyType>((v, t) => v.ToString()!.Contains("SSL certificate validation is DISABLED")),
62+
It.IsAny<Exception?>(),
63+
It.IsAny<Func<It.IsAnyType, Exception?, string>>()),
64+
Times.Once);
65+
66+
// Verify client was created successfully
67+
Assert.IsNotNull(client, "CosmosClient should be created");
68+
}
69+
70+
[TestMethod]
71+
public void CreateClient_WithoutDisableSslValidation_DoesNotLogWarning()
72+
{
73+
var loggerMock = new Mock<ILogger>();
74+
var settings = new TestableCosmosSettings
75+
{
76+
ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
77+
Database = "testDb",
78+
Container = "testContainer",
79+
DisableSslValidation = false
80+
};
81+
82+
var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object);
83+
84+
// Verify warning was NOT logged when DisableSslValidation is false
85+
loggerMock.Verify(
86+
x => x.Log(
87+
LogLevel.Warning,
88+
It.IsAny<EventId>(),
89+
It.Is<It.IsAnyType>((v, t) => v.ToString()!.Contains("SSL certificate validation is DISABLED")),
90+
It.IsAny<Exception?>(),
91+
It.IsAny<Func<It.IsAnyType, Exception?, string>>()),
92+
Times.Never);
93+
94+
// Verify client was created successfully
95+
Assert.IsNotNull(client, "CosmosClient should be created");
96+
}
97+
98+
private class TestableCosmosSettings : CosmosSettingsBase
99+
{
100+
}
101+
}
102+
}

0 commit comments

Comments
 (0)