From 9624b5439ae3470e390f5de899ba8c7038bf3c33 Mon Sep 17 00:00:00 2001 From: Matt Gode Date: Tue, 27 Mar 2018 21:25:22 -0700 Subject: [PATCH 1/4] Add AdaptiveRangeLimit constant for adaptive range limits --- pager.go | 2 +- range.go | 4 ++++ range_test.go | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pager.go b/pager.go index 8b99d237..8476238c 100644 --- a/pager.go +++ b/pager.go @@ -32,7 +32,7 @@ type pager struct { } func addLimitTokenString(w io.Writer, limit int, token string) { - if limit > 0 { + if limit == -1 || limit > 0 { fmt.Fprintf(w, " limit %d", limit) } if token != "" { diff --git a/range.go b/range.go index bb5bc708..1a21022e 100644 --- a/range.go +++ b/range.go @@ -30,6 +30,10 @@ import ( "github.com/pkg/errors" ) +const ( + AdaptiveRangeLimit = -1 +) + // RangeOp is used to specify constraints to Range calls type RangeOp struct { pager diff --git a/range_test.go b/range_test.go index 4eb6bfd1..cf1449c8 100644 --- a/range_test.go +++ b/range_test.go @@ -69,6 +69,12 @@ var rangeTestCases = []struct { stringer: " limit 10", converted: " limit 10", }, + { + descript: "empty with adaptive limit", + rop: NewRangeOp(&AllTypes{}).Limit(AdaptiveRangeLimit), + stringer: " limit -1", + converted: " limit -1", + }, { descript: "empty with token", rop: NewRangeOp(&AllTypes{}).Offset("toketoketoke"), From c7da1dedae407c826e23e8059f22d0b47231ad28 Mon Sep 17 00:00:00 2001 From: Matt Gode Date: Tue, 27 Mar 2018 21:46:35 -0700 Subject: [PATCH 2/4] Add a comment to the adaptive range limit comment --- pager.go | 2 +- range.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pager.go b/pager.go index 8476238c..73bf730a 100644 --- a/pager.go +++ b/pager.go @@ -32,7 +32,7 @@ type pager struct { } func addLimitTokenString(w io.Writer, limit int, token string) { - if limit == -1 || limit > 0 { + if limit == AdaptiveRangeLimit || limit > 0 { fmt.Fprintf(w, " limit %d", limit) } if token != "" { diff --git a/range.go b/range.go index 1a21022e..a89a8215 100644 --- a/range.go +++ b/range.go @@ -31,6 +31,10 @@ import ( ) const ( + // AdaptiveRangeLimit is a sentinel value that is used to indicate an intent + // to range over data in a partition as fast as possible. The server will + // determine an appropriate limit to use to range over the partition as fast + // as possible while ensuring the server remains healthy. AdaptiveRangeLimit = -1 ) From f4863acf785f1fcde4946dc0e00f3bd8c87433f8 Mon Sep 17 00:00:00 2001 From: Matt Gode Date: Wed, 4 Apr 2018 21:01:13 -0700 Subject: [PATCH 3/4] Handle adaptive range limits in random and memory connectors --- connectors/memory/memory.go | 7 +++++++ connectors/memory/memory_test.go | 7 +++++++ connectors/random/random.go | 4 ++++ connectors/random/random_test.go | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/connectors/memory/memory.go b/connectors/memory/memory.go index ca833031..d82a2604 100644 --- a/connectors/memory/memory.go +++ b/connectors/memory/memory.go @@ -65,6 +65,8 @@ type partitionRange struct { end int } +const defaultRangeLimit = 200 + // remove deletes the values referenced by the partitionRange. Since this function modifies // the data stored in the in-memory connector, a write lock must be held when calling // this function. @@ -492,6 +494,11 @@ func (c *Connector) Range(_ context.Context, ei *dosa.EntityInfo, columnConditio partitionRange.start += offset } } + + if limit == dosa.AdaptiveRangeLimit { + limit = defaultRangeLimit + } + slice := partitionRange.values() token = "" if len(slice) > limit { diff --git a/connectors/memory/memory_test.go b/connectors/memory/memory_test.go index 9dd97703..70109089 100644 --- a/connectors/memory/memory_test.go +++ b/connectors/memory/memory_test.go @@ -665,6 +665,13 @@ func TestConnector_Range(t *testing.T) { assert.NoError(t, err) assert.Empty(t, data) assert.Empty(t, token) + + // Test with adaptive limits + data, _, _ = sut.Range(context.TODO(), clusteredEi, map[string][]*dosa.Condition{ + "f1": {{Op: dosa.Eq, Value: dosa.FieldValue("data")}}, + "c1": {{Op: dosa.Eq, Value: dosa.FieldValue(int64(1))}}, + }, dosa.All(), "", 200) + assert.Len(t, data, idcount) } func TestConnector_TUUIDs(t *testing.T) { diff --git a/connectors/random/random.go b/connectors/random/random.go index d538778e..21588acb 100644 --- a/connectors/random/random.go +++ b/connectors/random/random.go @@ -33,6 +33,7 @@ import ( const ( maxBlobSize = 32 maxStringSize = 64 + defaultRangeLimit = 200 ) // Connector is a connector implementation for testing @@ -149,6 +150,9 @@ func (c *Connector) MultiRemove(ctx context.Context, ei *dosa.EntityInfo, multiV // Range returns a random set of data, and a random continuation token func (c *Connector) Range(ctx context.Context, ei *dosa.EntityInfo, columnConditions map[string][]*dosa.Condition, minimumFields []string, token string, limit int) ([]map[string]dosa.FieldValue, string, error) { + if limit == dosa.AdaptiveRangeLimit { + limit = defaultRangeLimit + } vals := make([]map[string]dosa.FieldValue, limit) for inx := range vals { vals[inx] = Data(ei, minimumFields) diff --git a/connectors/random/random_test.go b/connectors/random/random_test.go index 04ef04e9..1fe538d2 100644 --- a/connectors/random/random_test.go +++ b/connectors/random/random_test.go @@ -121,6 +121,12 @@ func TestRandom_Range(t *testing.T) { assert.NoError(t, err) } +func TestRandom_RangeAdaptiveLimits(t *testing.T) { + vals, _, err := sut.Range(ctx, testInfo, testConditions, minimumFields, "", dosa.AdaptiveRangeLimit) + assert.Len(t, vals, 200) + assert.NoError(t, err) +} + func TestRandom_Scan(t *testing.T) { vals, _, err := sut.Scan(ctx, testInfo, minimumFields, "", 32) assert.NotNil(t, vals) From 493eaa14700cc88ba5b1deb6fd9ea8ff7df7628a Mon Sep 17 00:00:00 2001 From: Matt Gode Date: Wed, 4 Apr 2018 21:01:43 -0700 Subject: [PATCH 4/4] Fix formatting issues --- connectors/random/random.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/connectors/random/random.go b/connectors/random/random.go index 21588acb..f9a058e0 100644 --- a/connectors/random/random.go +++ b/connectors/random/random.go @@ -31,8 +31,8 @@ import ( ) const ( - maxBlobSize = 32 - maxStringSize = 64 + maxBlobSize = 32 + maxStringSize = 64 defaultRangeLimit = 200 )