Skip to content

Migrate spansql to memefish #1287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Migrate spansql to memefish #1287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions database/spanner/spanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,14 @@ import (

"cloud.google.com/go/spanner"
sdb "cloud.google.com/go/spanner/admin/database/apiv1"
"cloud.google.com/go/spanner/spansql"

"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database"

adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
"github.com/cloudspannerecosystem/memefish"
"github.com/hashicorp/go-multierror"
uatomic "go.uber.org/atomic"
"google.golang.org/api/iterator"

"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database"
)

func init() {
Expand Down Expand Up @@ -51,7 +50,7 @@ var (
type Config struct {
MigrationsTable string
DatabaseName string
// Whether to parse the migration DDL with spansql before
// Whether to parse the migration DDL with memefish before
// running them towards Spanner.
// Parsing outputs clean DDL statements such as reformatted
// and void of comments.
Expand Down Expand Up @@ -343,13 +342,13 @@ func (s *Spanner) ensureVersionTable() (err error) {
func cleanStatements(migration []byte) ([]string, error) {
// The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC
// (see https://issuetracker.google.com/issues/159730604) we use
// spansql to parse the DDL and output valid stamements without comments
ddl, err := spansql.ParseDDL("", string(migration))
// memefish to parse the SQL statements and output valid stamements without comments
astStmts, err := memefish.ParseStatements("", string(migration))
if err != nil {
return nil, err
}
stmts := make([]string, 0, len(ddl.List))
for _, stmt := range ddl.List {
stmts := make([]string, 0, len(astStmts))
for _, stmt := range astStmts {
stmts = append(stmts, stmt.SQL())
}
return stmts, nil
Expand Down
55 changes: 27 additions & 28 deletions database/spanner/spanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@ import (
"os"
"testing"

"github.com/golang-migrate/migrate/v4"

dt "github.com/golang-migrate/migrate/v4/database/testing"
_ "github.com/golang-migrate/migrate/v4/source/file"

"cloud.google.com/go/spanner/spannertest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/golang-migrate/migrate/v4"
dt "github.com/golang-migrate/migrate/v4/database/testing"
_ "github.com/golang-migrate/migrate/v4/source/file"
)

// withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator
Expand Down Expand Up @@ -75,75 +74,75 @@ func TestCleanStatements(t *testing.T) {
{
name: "single statement, single line, no semicolon, no comment",
multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id)",
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
},
{
name: "single statement, multi line, no semicolon, no comment",
multiStatement: `CREATE TABLE table_name (
id STRING(255) NOT NULL,
id STRING(255) NOT NULL
) PRIMARY KEY (id)`,
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
},
{
name: "single statement, single line, with semicolon, no comment",
multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id);",
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
},
{
name: "single statement, multi line, with semicolon, no comment",
multiStatement: `CREATE TABLE table_name (
id STRING(255) NOT NULL,
id STRING(255) NOT NULL
) PRIMARY KEY (id);`,
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
},
{
name: "multi statement, with trailing semicolon. no comment",
// From https://github.com/mattes/migrate/pull/281
multiStatement: `CREATE TABLE table_name (
id STRING(255) NOT NULL,
) PRIMARY KEY(id);
id STRING(255) NOT NULL
) PRIMARY KEY (id);

CREATE INDEX table_name_id_idx ON table_name (id);`,
expected: []string{`CREATE TABLE table_name (
id STRING(255) NOT NULL,
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
id STRING(255) NOT NULL
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
},
{
name: "multi statement, no trailing semicolon, no comment",
// From https://github.com/mattes/migrate/pull/281
multiStatement: `CREATE TABLE table_name (
id STRING(255) NOT NULL,
) PRIMARY KEY(id);
id STRING(255) NOT NULL
) PRIMARY KEY (id);

CREATE INDEX table_name_id_idx ON table_name (id)`,
expected: []string{`CREATE TABLE table_name (
id STRING(255) NOT NULL,
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
id STRING(255) NOT NULL
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
},
{
name: "multi statement, no trailing semicolon, standalone comment",
// From https://github.com/mattes/migrate/pull/281
multiStatement: `CREATE TABLE table_name (
-- standalone comment
id STRING(255) NOT NULL,
) PRIMARY KEY(id);
id STRING(255) NOT NULL
) PRIMARY KEY (id);

CREATE INDEX table_name_id_idx ON table_name (id)`,
expected: []string{`CREATE TABLE table_name (
id STRING(255) NOT NULL,
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
id STRING(255) NOT NULL
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
},
{
name: "multi statement, no trailing semicolon, inline comment",
// From https://github.com/mattes/migrate/pull/281
multiStatement: `CREATE TABLE table_name (
id STRING(255) NOT NULL, -- inline comment
) PRIMARY KEY(id);
) PRIMARY KEY (id);

CREATE INDEX table_name_id_idx ON table_name (id)`,
expected: []string{`CREATE TABLE table_name (
id STRING(255) NOT NULL,
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
id STRING(255) NOT NULL
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
},
{
name: "alter table with SET OPTIONS",
Expand All @@ -155,9 +154,9 @@ func TestCleanStatements(t *testing.T) {
name: "column with NUMERIC type",
multiStatement: `CREATE TABLE table_name (
id STRING(255) NOT NULL,
sum NUMERIC,
sum NUMERIC
) PRIMARY KEY (id)`,
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC,\n) PRIMARY KEY(id)"},
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC\n) PRIMARY KEY (id)"},
},
}

Expand Down
3 changes: 2 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/ClickHouse/clickhouse-go v1.4.3
github.com/aws/aws-sdk-go v1.49.6
github.com/cenkalti/backoff/v4 v4.1.2
github.com/cloudspannerecosystem/memefish v0.6.1
github.com/cockroachdb/cockroach-go/v2 v2.1.1
github.com/dhui/dktest v0.4.5
github.com/docker/docker v27.2.0+incompatible
Expand Down Expand Up @@ -147,7 +148,7 @@ require (
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/mattn/go-colorable v0.1.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
Expand Down
13 changes: 10 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCU
github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
Expand Down Expand Up @@ -131,6 +133,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cloudspannerecosystem/memefish v0.6.1 h1:EJNZNq0E2vrYGBlu/xBs6jN7a5eq9ovF/wK+5Mo1iks=
github.com/cloudspannerecosystem/memefish v0.6.1/go.mod h1:mVw0xBxy0yOgm990BuR0+nqP8J+yBAAf7N/2uL69rBU=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc=
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM=
Expand Down Expand Up @@ -272,8 +276,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
Expand Down Expand Up @@ -412,6 +416,8 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo=
github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
github.com/k0kubun/pp/v3 v3.4.1 h1:1WdFZDRRqe8UsR61N/2RoOZ3ziTEqgTPVqKrHeb779Y=
github.com/k0kubun/pp/v3 v3.4.1/go.mod h1:+SiNiqKnBfw1Nkj82Lh5bIeKQOAkPy6Xw9CAZUZ8npI=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
Expand Down Expand Up @@ -452,8 +458,9 @@ github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQ
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
Expand Down