From 48617d618d4da846dff36151f315645ec23c0cca Mon Sep 17 00:00:00 2001 From: Suman Karki Date: Wed, 21 May 2025 14:36:12 -0700 Subject: [PATCH 1/2] supporting datemath for between operator --- vm/datemath.go | 108 +++++++++++++++++++++++++++++++++++++++++-- vm/datemath_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+), 3 deletions(-) diff --git a/vm/datemath.go b/vm/datemath.go index 909a15a..4e52649 100644 --- a/vm/datemath.go +++ b/vm/datemath.go @@ -42,10 +42,9 @@ func CalcBoundaryFns(n expr.Node) BoundaryFns { return findDateMathFn(n) } -// NewDateConverter takes a node expression -func NewDateConverter(ctx expr.EvalIncludeContext, n expr.Node) (*DateConverter, error) { +func NewDateConverterWithAnchorTime(ctx expr.EvalIncludeContext, n expr.Node, at time.Time) (*DateConverter, error) { dc := &DateConverter{ - at: time.Now(), + at: at, } fns := findDateMathFn(n) dc.bt, dc.err = FindBoundary(dc.at, ctx, fns) @@ -54,6 +53,11 @@ func NewDateConverter(ctx expr.EvalIncludeContext, n expr.Node) (*DateConverter, } return dc, dc.err } + +// NewDateConverter takes a node expression +func NewDateConverter(ctx expr.EvalIncludeContext, n expr.Node) (*DateConverter, error) { + return NewDateConverterWithAnchorTime(ctx, n, time.Now()) +} func compareBoundaries(currBoundary, newBoundary time.Time) time.Time { // Should we check for is zero on the newBoundary? if currBoundary.IsZero() || newBoundary.Before(currBoundary) { @@ -183,6 +187,19 @@ func findDateMathFn(node expr.Node) BoundaryFns { case *expr.UnaryNode: return findDateMathFn(n.Arg) case *expr.TriNode: + // Only handle BETWEEN operator with specific node types + if n.Operator.T == lex.TokenBetween && len(n.Args) == 3 { + // Check if first arg is IdentityNode and other two are StringNodes + _, isFirstIdentity := n.Args[0].(*expr.IdentityNode) + _, isSecondString := n.Args[1].(*expr.StringNode) + _, isThirdString := n.Args[2].(*expr.StringNode) + + if isFirstIdentity && isSecondString && isThirdString { + fns = append(fns, findBoundaryForBetween(n)) + return fns + } + } + for _, arg := range n.Args { fns = append(fns, findDateMathFn(arg)...) } @@ -204,3 +221,88 @@ func findDateMathFn(node expr.Node) BoundaryFns { } return fns } + +// Ct = Comparison time, left hand side of expression +// Lb = Relative time result of Lower Bound Anchor Time offset by datemath "now-3d" +// Ub = Relative time result of Upper Bound Anchor Time offset by datemath "now+3d" +// Bt = Boundary time = calculated time at which expression will change boolean expression value +// example: FILTER Ct BETWEEN Lb AND Ub +// WHERE "now" = 01/22/2025 00:00:00 +// +// "Lb" = 01/19/2025 00:00:00 +// "Ub" = 01/25/2025 00:00:00 +// +// NOTE: When evaluating expressions like this, an entity "enters" the expression by passing the upper bound and sliding into the window +// and "exits" the expression by passing the lower bound and sliding out of the window. Although the "Upper Bound" is after the "Lower Bound", +// the order of entry and exit is reversed. This example should show how the entity appears to move backwards as the window moves forward. +// Example timeline: Ct = 01/22/2025 00:00:00 +// Day 0: now = 01/22/2025 00:00:00 === Lb--Ct++Ub+++++++++ +// Day 1: now = 01/23/2025 00:00:00 === -Lb-Ct+++Ub++++++++ +// Day 2: now = 01/24/2025 00:00:00 === --LbCt++++Ub+++++++ +// Day 3: now = 01/25/2025 00:00:00 === ---CLtb+++++Ub+++++ Ct == Lb +// Day 4: now = 01/26/2025 00:00:00 === ---CtLb++++++Ub++++ +// Day 5: now = 01/27/2025 00:00:00 === ---Ct-Lb++++++Ub+++ +// Day 6: now = 01/28/2025 00:00:00 === ---Ct--Lb++++++Ub++ +// Day 7: now = 01/29/2025 00:00:00 === ---Ct---Lb++++++Ub+ +// Day 8: now = 01/30/2025 00:00:00 === ---Ct----Lb++++++Ub +// +// Notice how it appears as if the entity is moving backwards through the window as the window slides forward in relation to "now". +// Ct = 01/01/2025 00:00:00 +// In this case the expression itself currently evaluates to false, and is already PRIOR to the lower bound. +// As such it will always evaluate to false, so we can skip queueing for reevaluation. +// // Ct = 01/30/2025 00:00:00 +// In this case the expression itself currently evaluates to false, and is AFTER of the prior to the upper bound. +// The next evaluation time should be on the day the upper bound is reached. Queue for reevaluation at (Ct - 3d) [since the Ub is now+3d]. +// // Ct = 01/22/2025 00:00:00 +// In this case the expression itself currently evaluates to true, and is BETWEEN the lower and upper bounds. +// The next evaluation time should be on the day the lower bound is reached. Queue for reevaluation at (Ct + 3d) [since the Lb is now-3d]. +// Notice that we are queuing for reevaluation based on the inverse of the corresponding bound's offset. This is made a bit easier if +// the expression is a static date, rather than a relative one. As we can use that directly as the re-evaluation time. +func findBoundaryForBetween(n *expr.TriNode) func(d *DateConverter, ctx expr.EvalIncludeContext) { + return func(d *DateConverter, ctx expr.EvalIncludeContext) { + arg1, arg2, arg3 := n.Args[0], n.Args[1], n.Args[2] + + lhv, ok := Eval(ctx, arg1) + if !ok { + return + } + ct, ok := value.ValueToTime(lhv) + if !ok { + // may be not a time field, so ignore doing any update + return + } + + val2 := strings.ToLower(arg2.(*expr.StringNode).Text) + date1, err := datemath.EvalAnchor(d.at, val2) + if err != nil { + // may be not a valid date expression, so ignore doing any update + return + } + + val3 := strings.ToLower(arg3.(*expr.StringNode).Text) + date2, err := datemath.EvalAnchor(d.at, val3) + if err != nil { + // may be not a valid date expression, so ignore doing any update + return + } + + // assign lower and upper bounds + lower, upper := date1, date2 + if date1.After(date2) { + lower, upper = date2, date1 + } + + if ct.Before(lower) { + // out of window's lower bound, so will always be false + return + } + + if ct.After(upper) || ct.Equal(upper) { + // in the future or right in the border, so will enter the window later sometime in the future, do re-evaluate + d.bt = compareBoundaries(d.bt, d.at.Add(ct.Sub(upper))) + return + } + // currently in the window, so will exit the window in the future, do re-evaluate + d.bt = compareBoundaries(d.bt, d.at.Add(ct.Sub(lower))) + } +} diff --git a/vm/datemath_test.go b/vm/datemath_test.go index 5599c8d..e75277c 100644 --- a/vm/datemath_test.go +++ b/vm/datemath_test.go @@ -253,3 +253,113 @@ func TestDateMath(t *testing.T) { _, err = vm.NewDateConverter(evalCtx, fs.Filter) assert.Equal(t, nil, err) } + +func TestDateBoundaryForBetween(t *testing.T) { + today := time.Now() + + type testCase struct { + testName string + filter string + match bool + expectedBoundaryTime time.Time + lastEvtTs time.Time + } + + tests := []testCase{ + { + testName: "within_window_1_day_before_lower_bound", + filter: `FILTER last_event BETWEEN "now-2d" AND "now+3d"`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 1), // Will exit after 1 day since lower bound is after 2 days + lastEvtTs: today.AddDate(0, 0, -1), + }, + { + testName: "within_window_including_other_filters", + filter: `FILTER AND( last_event BETWEEN "now-2d" AND "now+3d", exists subscription_expires )`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 1), // Will exit after 1 day since lower bound is after 2 days + lastEvtTs: today.AddDate(0, 0, -1), + }, + { + testName: "exact_current_time", + filter: `FILTER last_event BETWEEN "now-2d" AND "now+3d"`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 2), // Will exit after two days + lastEvtTs: today, + }, + { + testName: "just_inside_lower", + filter: `FILTER last_event BETWEEN "now-2d" AND "now+3d"`, + match: true, + expectedBoundaryTime: today.Add(time.Minute), // will be after a minute + lastEvtTs: today.AddDate(0, 0, -2).Add(time.Minute), // 2 days ago + 1 minute + }, + { + testName: "just_inside_upper", + filter: `FILTER last_event BETWEEN "now-2d" AND "now+3d"`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 5).Add(-time.Minute), // as window is of 5 days and this one just entered + lastEvtTs: today.AddDate(0, 0, 3).Add(-time.Minute), // will exit after 2 days later + }, + { + testName: "exact_boundary_lower", + filter: `FILTER last_event BETWEEN "now-2d" AND "now+3d"`, + match: false, // going to be false as it thinks it is out of window + expectedBoundaryTime: today, // already in the lowerbound, so should exit now + lastEvtTs: today.AddDate(0, 0, -2), // Exactly 2 days ago + }, + { + testName: "exact_boundary_upper", + filter: `FILTER last_event BETWEEN "now-2d" AND "now+3d"`, + match: false, // not entered yet + expectedBoundaryTime: today, // should enter right now + lastEvtTs: today.AddDate(0, 0, 3), // Exactly 3 days in future + }, + { + testName: "multiple_date_math", + filter: `FILTER AND(last_event BETWEEN "now-2d" AND "now+3d", subscription_expires > "now+1d")`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 2), // Will exit after 2 days last_event is in window and subscription_expires date is 6 days later + lastEvtTs: today, // today + }, + { + testName: "not_condition", + filter: `FILTER NOT(last_event BETWEEN "now-2d" AND "now+3d")`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 1), // Will enter after a day as it will be inside of window + lastEvtTs: today.AddDate(0, 0, 4), // 4 days in the future (right of window) + }, + { + testName: "multiple_between", + filter: `FILTER AND( last_event BETWEEN "now-2d" AND "now+3d", subscription_expires BETWEEN "now+1d" AND "now+7d")`, + match: true, + expectedBoundaryTime: today.AddDate(0, 0, 1), // Will exit after 1 day due to last_event sliding out of window after 1 day + lastEvtTs: today.AddDate(0, 0, -1), // 1 day ago + }, + } + + for _, tc := range tests { + t.Run(tc.testName, func(t *testing.T) { + evalCtx := datasource.NewContextMapTs(map[string]interface{}{ + "last_event": tc.lastEvtTs, + "subscription_expires": today.Add(time.Hour * 24 * 6), + "lastevent": map[string]time.Time{"signedup": today}, + "first.event": map[string]time.Time{"has.period": today}, + }, true, today) + + includeCtx := &includectx{ContextReader: evalCtx} + + fs := rel.MustParseFilter(tc.filter) + + dc, err := vm.NewDateConverterWithAnchorTime(includeCtx, fs.Filter, today) + require.NoError(t, err) + require.True(t, dc.HasDateMath) + + matched, evalOk := vm.Matches(includeCtx, fs) + assert.True(t, evalOk) + assert.Equal(t, tc.match, matched) + + assert.Equal(t, tc.expectedBoundaryTime.Unix(), dc.Boundary().Unix()) + }) + } +} From f02c936d5dc9a19a78366d08f04a7f5bd997bf57 Mon Sep 17 00:00:00 2001 From: Suman Karki Date: Thu, 22 May 2025 15:29:50 -0700 Subject: [PATCH 2/2] added some comments --- vm/datemath.go | 96 +++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/vm/datemath.go b/vm/datemath.go index 4e52649..734c92d 100644 --- a/vm/datemath.go +++ b/vm/datemath.go @@ -190,12 +190,9 @@ func findDateMathFn(node expr.Node) BoundaryFns { // Only handle BETWEEN operator with specific node types if n.Operator.T == lex.TokenBetween && len(n.Args) == 3 { // Check if first arg is IdentityNode and other two are StringNodes - _, isFirstIdentity := n.Args[0].(*expr.IdentityNode) - _, isSecondString := n.Args[1].(*expr.StringNode) - _, isThirdString := n.Args[2].(*expr.StringNode) - - if isFirstIdentity && isSecondString && isThirdString { - fns = append(fns, findBoundaryForBetween(n)) + fn := findBoundaryForBetween(n) + if fn != nil { + fns = append(fns, fn) return fns } } @@ -222,45 +219,56 @@ func findDateMathFn(node expr.Node) BoundaryFns { return fns } -// Ct = Comparison time, left hand side of expression -// Lb = Relative time result of Lower Bound Anchor Time offset by datemath "now-3d" -// Ub = Relative time result of Upper Bound Anchor Time offset by datemath "now+3d" -// Bt = Boundary time = calculated time at which expression will change boolean expression value -// example: FILTER Ct BETWEEN Lb AND Ub -// WHERE "now" = 01/22/2025 00:00:00 +// findBoundaryForBetween calculates the next time boundary for a BETWEEN expression +// with date math boundaries. It handles expressions like: +// +// time_column BETWEEN "now-3d" AND "now+3d" +// +// The function returns a boundary function that: +// 1. Evaluates the comparison time (Ct) against the window boundaries +// 2. Determines when the expression's boolean value will change +// 3. Returns the appropriate re-evaluation time // -// "Lb" = 01/19/2025 00:00:00 -// "Ub" = 01/25/2025 00:00:00 +// Example: // -// NOTE: When evaluating expressions like this, an entity "enters" the expression by passing the upper bound and sliding into the window -// and "exits" the expression by passing the lower bound and sliding out of the window. Although the "Upper Bound" is after the "Lower Bound", -// the order of entry and exit is reversed. This example should show how the entity appears to move backwards as the window moves forward. -// Example timeline: Ct = 01/22/2025 00:00:00 -// Day 0: now = 01/22/2025 00:00:00 === Lb--Ct++Ub+++++++++ -// Day 1: now = 01/23/2025 00:00:00 === -Lb-Ct+++Ub++++++++ -// Day 2: now = 01/24/2025 00:00:00 === --LbCt++++Ub+++++++ -// Day 3: now = 01/25/2025 00:00:00 === ---CLtb+++++Ub+++++ Ct == Lb -// Day 4: now = 01/26/2025 00:00:00 === ---CtLb++++++Ub++++ -// Day 5: now = 01/27/2025 00:00:00 === ---Ct-Lb++++++Ub+++ -// Day 6: now = 01/28/2025 00:00:00 === ---Ct--Lb++++++Ub++ -// Day 7: now = 01/29/2025 00:00:00 === ---Ct---Lb++++++Ub+ -// Day 8: now = 01/30/2025 00:00:00 === ---Ct----Lb++++++Ub +// Input: time_column BETWEEN "now-3d" AND "now+3d" +// When: now = 2025-01-22 +// Window: 2025-01-19 to 2025-01-25 // -// Notice how it appears as if the entity is moving backwards through the window as the window slides forward in relation to "now". -// Ct = 01/01/2025 00:00:00 -// In this case the expression itself currently evaluates to false, and is already PRIOR to the lower bound. -// As such it will always evaluate to false, so we can skip queueing for reevaluation. -// // Ct = 01/30/2025 00:00:00 -// In this case the expression itself currently evaluates to false, and is AFTER of the prior to the upper bound. -// The next evaluation time should be on the day the upper bound is reached. Queue for reevaluation at (Ct - 3d) [since the Ub is now+3d]. -// // Ct = 01/22/2025 00:00:00 -// In this case the expression itself currently evaluates to true, and is BETWEEN the lower and upper bounds. -// The next evaluation time should be on the day the lower bound is reached. Queue for reevaluation at (Ct + 3d) [since the Lb is now-3d]. -// Notice that we are queuing for reevaluation based on the inverse of the corresponding bound's offset. This is made a bit easier if -// the expression is a static date, rather than a relative one. As we can use that directly as the re-evaluation time. +// If Ct = 2025-01-01 (left side of window): +// - Expression is false +// - Will always be false as window is moving forward +// - Returns zero time (no re-evaluation needed) +// +// If Ct = 2025-01-30 (right side of window): +// - Expression is false +// - Will become true when window catches up (enter event) +// - Returns re-evaluation time when this will enter the window +// +// If Ct = 2025-01-22 (inside window): +// - Expression is true +// - Will become false when Ct passes lower bound (exit event) +// - Returns re-evaluation time when this will be exit the window func findBoundaryForBetween(n *expr.TriNode) func(d *DateConverter, ctx expr.EvalIncludeContext) { + + // Check if first arg is IdentityNode and other two are StringNodes + _, isFirstIdentity := n.Args[0].(*expr.IdentityNode) + _, isSecondString := n.Args[1].(*expr.StringNode) + _, isThirdString := n.Args[2].(*expr.StringNode) + + if !isFirstIdentity || !isSecondString || !isThirdString { + return nil + } + arg1, arg2, arg3 := n.Args[0], n.Args[1], n.Args[2] + + // datemath only if both date args are relative to an anchor time like "now-1d" + val2 := strings.ToLower(arg2.(*expr.StringNode).Text) + val3 := strings.ToLower(arg3.(*expr.StringNode).Text) + if !nowRegex.MatchString(val2) || !nowRegex.MatchString(val3) { + return nil + } + return func(d *DateConverter, ctx expr.EvalIncludeContext) { - arg1, arg2, arg3 := n.Args[0], n.Args[1], n.Args[2] lhv, ok := Eval(ctx, arg1) if !ok { @@ -268,21 +276,19 @@ func findBoundaryForBetween(n *expr.TriNode) func(d *DateConverter, ctx expr.Eva } ct, ok := value.ValueToTime(lhv) if !ok { - // may be not a time field, so ignore doing any update + d.err = fmt.Errorf("could not convert %T: %v to time.Time", lhv, lhv) return } - val2 := strings.ToLower(arg2.(*expr.StringNode).Text) date1, err := datemath.EvalAnchor(d.at, val2) if err != nil { - // may be not a valid date expression, so ignore doing any update + d.err = err return } - val3 := strings.ToLower(arg3.(*expr.StringNode).Text) date2, err := datemath.EvalAnchor(d.at, val3) if err != nil { - // may be not a valid date expression, so ignore doing any update + d.err = err return }