diff --git a/database/spanner/spanner.go b/database/spanner/spanner.go index b733302d5..6ca1931e2 100644 --- a/database/spanner/spanner.go +++ b/database/spanner/spanner.go @@ -13,15 +13,14 @@ import ( "cloud.google.com/go/spanner" sdb "cloud.google.com/go/spanner/admin/database/apiv1" - "cloud.google.com/go/spanner/spansql" - - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database" - adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" + "github.com/cloudspannerecosystem/memefish" "github.com/hashicorp/go-multierror" uatomic "go.uber.org/atomic" "google.golang.org/api/iterator" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" ) func init() { @@ -51,7 +50,7 @@ var ( type Config struct { MigrationsTable string DatabaseName string - // Whether to parse the migration DDL with spansql before + // Whether to parse the migration DDL with memefish before // running them towards Spanner. // Parsing outputs clean DDL statements such as reformatted // and void of comments. @@ -343,13 +342,13 @@ func (s *Spanner) ensureVersionTable() (err error) { func cleanStatements(migration []byte) ([]string, error) { // The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC // (see https://issuetracker.google.com/issues/159730604) we use - // spansql to parse the DDL and output valid stamements without comments - ddl, err := spansql.ParseDDL("", string(migration)) + // memefish to parse the SQL statements and output valid stamements without comments + astStmts, err := memefish.ParseStatements("", string(migration)) if err != nil { return nil, err } - stmts := make([]string, 0, len(ddl.List)) - for _, stmt := range ddl.List { + stmts := make([]string, 0, len(astStmts)) + for _, stmt := range astStmts { stmts = append(stmts, stmt.SQL()) } return stmts, nil diff --git a/database/spanner/spanner_test.go b/database/spanner/spanner_test.go index d6ab4db32..d3c2cb271 100644 --- a/database/spanner/spanner_test.go +++ b/database/spanner/spanner_test.go @@ -5,14 +5,13 @@ import ( "os" "testing" - "github.com/golang-migrate/migrate/v4" - - dt "github.com/golang-migrate/migrate/v4/database/testing" - _ "github.com/golang-migrate/migrate/v4/source/file" - "cloud.google.com/go/spanner/spannertest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" ) // withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator @@ -75,75 +74,75 @@ func TestCleanStatements(t *testing.T) { { name: "single statement, single line, no semicolon, no comment", multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id)", - expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"}, }, { name: "single statement, multi line, no semicolon, no comment", multiStatement: `CREATE TABLE table_name ( - id STRING(255) NOT NULL, + id STRING(255) NOT NULL ) PRIMARY KEY (id)`, - expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"}, }, { name: "single statement, single line, with semicolon, no comment", multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id);", - expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"}, }, { name: "single statement, multi line, with semicolon, no comment", multiStatement: `CREATE TABLE table_name ( - id STRING(255) NOT NULL, + id STRING(255) NOT NULL ) PRIMARY KEY (id);`, - expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"}, }, { name: "multi statement, with trailing semicolon. no comment", // From https://github.com/mattes/migrate/pull/281 multiStatement: `CREATE TABLE table_name ( - id STRING(255) NOT NULL, - ) PRIMARY KEY(id); + id STRING(255) NOT NULL + ) PRIMARY KEY (id); CREATE INDEX table_name_id_idx ON table_name (id);`, expected: []string{`CREATE TABLE table_name ( - id STRING(255) NOT NULL, -) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + id STRING(255) NOT NULL +) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, }, { name: "multi statement, no trailing semicolon, no comment", // From https://github.com/mattes/migrate/pull/281 multiStatement: `CREATE TABLE table_name ( - id STRING(255) NOT NULL, - ) PRIMARY KEY(id); + id STRING(255) NOT NULL + ) PRIMARY KEY (id); CREATE INDEX table_name_id_idx ON table_name (id)`, expected: []string{`CREATE TABLE table_name ( - id STRING(255) NOT NULL, -) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + id STRING(255) NOT NULL +) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, }, { name: "multi statement, no trailing semicolon, standalone comment", // From https://github.com/mattes/migrate/pull/281 multiStatement: `CREATE TABLE table_name ( -- standalone comment - id STRING(255) NOT NULL, - ) PRIMARY KEY(id); + id STRING(255) NOT NULL + ) PRIMARY KEY (id); CREATE INDEX table_name_id_idx ON table_name (id)`, expected: []string{`CREATE TABLE table_name ( - id STRING(255) NOT NULL, -) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + id STRING(255) NOT NULL +) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, }, { name: "multi statement, no trailing semicolon, inline comment", // From https://github.com/mattes/migrate/pull/281 multiStatement: `CREATE TABLE table_name ( id STRING(255) NOT NULL, -- inline comment - ) PRIMARY KEY(id); + ) PRIMARY KEY (id); CREATE INDEX table_name_id_idx ON table_name (id)`, expected: []string{`CREATE TABLE table_name ( - id STRING(255) NOT NULL, -) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + id STRING(255) NOT NULL +) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, }, { name: "alter table with SET OPTIONS", @@ -155,9 +154,9 @@ func TestCleanStatements(t *testing.T) { name: "column with NUMERIC type", multiStatement: `CREATE TABLE table_name ( id STRING(255) NOT NULL, - sum NUMERIC, + sum NUMERIC ) PRIMARY KEY (id)`, - expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC,\n) PRIMARY KEY(id)"}, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC\n) PRIMARY KEY (id)"}, }, } diff --git a/go.mod b/go.mod index 3c20151f2..bf401b3eb 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/ClickHouse/clickhouse-go v1.4.3 github.com/aws/aws-sdk-go v1.49.6 github.com/cenkalti/backoff/v4 v4.1.2 + github.com/cloudspannerecosystem/memefish v0.6.1 github.com/cockroachdb/cockroach-go/v2 v2.1.1 github.com/dhui/dktest v0.4.5 github.com/docker/docker v27.2.0+incompatible @@ -147,7 +148,7 @@ require ( github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.11 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect - github.com/mattn/go-colorable v0.1.6 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect diff --git a/go.sum b/go.sum index e30a51a2a..9b5e555e2 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,8 @@ github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCU github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A= +github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -131,6 +133,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cloudspannerecosystem/memefish v0.6.1 h1:EJNZNq0E2vrYGBlu/xBs6jN7a5eq9ovF/wK+5Mo1iks= +github.com/cloudspannerecosystem/memefish v0.6.1/go.mod h1:mVw0xBxy0yOgm990BuR0+nqP8J+yBAAf7N/2uL69rBU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= @@ -272,8 +276,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -412,6 +416,8 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/k0kubun/pp/v3 v3.4.1 h1:1WdFZDRRqe8UsR61N/2RoOZ3ziTEqgTPVqKrHeb779Y= +github.com/k0kubun/pp/v3 v3.4.1/go.mod h1:+SiNiqKnBfw1Nkj82Lh5bIeKQOAkPy6Xw9CAZUZ8npI= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -452,8 +458,9 @@ github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=