Allow specifying the Spanner DB default timezone as part of schema migration (GoogleCloudPlatform#1229)

* Add target profile flag to specify the default timezone of the Spanner DB

* Allow specifying the default timezone via the web UI

* Include setting of database options in DDL generation

* Pass through DB options (currently only default timezone) when migrating schema

* Ensure default time zone DDL command is included in output files and add default time zone to integration tests

* Fix test issues from merge of master branch

Author: nmemond

accessors/spanner/spanner_accessor.go

@@ -306,9 +306,9 @@ func (sp *SpannerAccessorImpl) CreateDatabase(ctx context.Context, dbURI string,
 		req.DatabaseDialect = adminpb.DatabaseDialect_POSTGRESQL
 	} else {
 		if migrationType == constants.DATAFLOW_MIGRATION {
-			req.ExtraStatements = ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences)
+			req.ExtraStatements = ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences, conv.DatabaseOptions)
 		} else {
-			req.ExtraStatements = ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: false, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences)
+			req.ExtraStatements = ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: false, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences, conv.DatabaseOptions)
 		}
 
 	}
@@ -350,7 +350,7 @@ func (sp *SpannerAccessorImpl) UpdateDatabase(ctx context.Context, dbURI string,
 	// Spanner DDL doesn't accept them), and protects table and col names
 	// using backticks (to avoid any issues with Spanner reserved words).
 	// Foreign Keys are set to false since we create them post data migration.
-	schema := ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: false, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences)
+	schema := ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: false, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences, conv.DatabaseOptions)
 	if len(schema) == 0 {
 		return nil
 	}
@@ -378,6 +378,10 @@ func (sp *SpannerAccessorImpl) CreateOrUpdateDatabase(ctx context.Context, dbURI
 	if err != nil {
 		return err
 	}
+	if conv.DatabaseOptions.DefaultTimezone != "" && len(tablesExistingOnSpanner) > 0 {
+		logger.Log.Warn("Spanner database already contains tables, can not set default time zone of '" + conv.DatabaseOptions.DefaultTimezone + "'. See https://docs.cloud.google.com/spanner/docs/set-default-time-zone#limitations.")
+		conv.DatabaseOptions.DefaultTimezone = ""
+	}
 	if dbExists {
 		if conv.SpDialect != constants.DIALECT_POSTGRESQL && migrationType == constants.DATAFLOW_MIGRATION {
 			return fmt.Errorf("spanner migration tool does not support minimal downtime schema/schema-and-data migrations to an existing database")
@@ -477,7 +481,8 @@ func (sp *SpannerAccessorImpl) UpdateDDLForeignKeys(ctx context.Context, dbURI s
 	// Spanner DDL doesn't accept them), and protects table and col names
 	// using backticks (to avoid any issues with Spanner reserved words).
 	// Sequences will not be passed as they have already been created.
-	fkStmts := ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: false, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, make(map[string]ddl.Sequence))
+	// Database options will not be passed since they have also already been set.
+	fkStmts := ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: false, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, make(map[string]ddl.Sequence), ddl.DatabaseOptions{})
 	if len(fkStmts) == 0 {
 		return
 	}

accessors/spanner/spanner_accessor_test.go

@@ -598,6 +598,10 @@ func TestSpannerAccessorImpl_CreateDatabase(t *testing.T) {
 				},
 			},
 		}
+		conv.DatabaseOptions = ddl.DatabaseOptions{
+			DbName: "database-id",
+			DefaultTimezone: "America/New_York",
+		}
 		spA := SpannerAccessorImpl{AdminClient: &tc.acm}
 		err := spA.CreateDatabase(ctx, dbURI, conv, "", tc.migrationType)
 		assert.Equal(t, tc.expectError, err != nil, tc.name)
@@ -1010,6 +1014,10 @@ func TestSpannerAccessorImpl_CreateOrUpdateDatabase(t *testing.T) {
 			PrimaryKeys: []ddl.IndexKey{{ColId: "c1"}},
 			Id:          "t1",
 		}
+		conv.DatabaseOptions = ddl.DatabaseOptions{
+			DbName: "database-id",
+			DefaultTimezone: "America/New_York",
+		}
 		spA := SpannerAccessorImpl{AdminClient: &tc.acm}
 		err := spA.CreateOrUpdateDatabase(ctx, dbURI, "", conv, tc.migrationType, tc.tablesExistingOnSpanner)
 		assert.Equal(t, tc.expectError, err != nil, tc.name)

assessment/assessment_engine.go

@@ -139,7 +139,8 @@ func performQueryAssessment(ctx context.Context, collectors assessmentCollectors
 		ddl.GetDDL(
 			ddl.Config{Comments: true, ProtectIds: false, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: "mysql"},
 			conv.SpSchema,
-			conv.SpSequences),
+			conv.SpSequences,
+			conv.DatabaseOptions),
 		"\n")
 
 	for _, query := range queries {
@@ -216,7 +217,8 @@ func initializeCollectors(conv *internal.Conv, sourceProfile profiles.SourceProf
 			ddl.GetDDL(
 				ddl.Config{Comments: true, ProtectIds: false, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: "mysql"},
 				conv.SpSchema,
-				conv.SpSequences),
+				conv.SpSequences,
+				conv.DatabaseOptions),
 			"\n")
 
 		logger.Log.Debug("mysqlSchema", zap.String("schema", mysqlSchema))

cmd/import_test.go

@@ -844,7 +844,8 @@ func fetchDDLString(conv *internal.Conv) string {
 		ddl.GetDDL(
 			ddl.Config{Comments: false, ProtectIds: false, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: "mysql"},
 			conv.SpSchema,
-			conv.SpSequences), ";"), "\n", " ", -1)
+			conv.SpSequences,
+			conv.DatabaseOptions), ";"), "\n", " ", -1)
 }
 
 func TestGetDialectWithDefaults(t *testing.T) {

cmd/utils.go

@@ -36,6 +36,7 @@ import (
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/profiles"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/writer"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/webv2/helpers"
 	"google.golang.org/grpc/metadata"
@@ -162,6 +163,11 @@ func MigrateDatabase(ctx context.Context, migrationProjectId string, targetProfi
 	}
 	defer adminClient.Close()
 	defer client.Close()
+	// Before this point, the actual DB name to use isn't finalized...
+	conv.DatabaseOptions = ddl.DatabaseOptions{
+		DbName: targetProfile.Conn.Sp.Dbname,
+		DefaultTimezone: targetProfile.Conn.Sp.DefaultTimezone,
+	}
 	switch v := cmd.(type) {
 	case *SchemaCmd:
 		err = migrateSchema(ctx, targetProfile, sourceProfile, ioHelper, conv, dbURI, adminClient, client)

conversion/conversion.go

@@ -43,6 +43,7 @@ import (
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/profiles"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/sources/common"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/sources/csv"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/writer"
 )
 
@@ -76,15 +77,21 @@ type ConvImpl struct{}
 // SchemaConv performs the schema conversion
 // The SourceProfile param provides the connection details to use the go SQL library.
 func (ci *ConvImpl) SchemaConv(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, schemaFromSource SchemaFromSourceInterface) (*internal.Conv, error) {
+	var conv *internal.Conv
+	var err error
 	switch sourceProfile.Driver {
 	case constants.POSTGRES, constants.MYSQL, constants.DYNAMODB, constants.SQLSERVER, constants.ORACLE, constants.CASSANDRA:
-		return schemaFromSource.schemaFromDatabase(migrationProjectId, sourceProfile, targetProfile, &GetInfoImpl{}, &common.ProcessSchemaImpl{})
+		conv, err = schemaFromSource.schemaFromDatabase(migrationProjectId, sourceProfile, targetProfile, &GetInfoImpl{}, &common.ProcessSchemaImpl{})
 	case constants.PGDUMP, constants.MYSQLDUMP:
 		expressionVerificationAccessor, _ := expressions_api.NewExpressionVerificationAccessorImpl(context.Background(), targetProfile.Conn.Sp.Project, targetProfile.Conn.Sp.Instance)
-		return schemaFromSource.SchemaFromDump(targetProfile.Conn.Sp.Project, targetProfile.Conn.Sp.Instance, sourceProfile.Driver, targetProfile.Conn.Sp.Dialect, ioHelper, &ProcessDumpByDialectImpl{ExpressionVerificationAccessor: expressionVerificationAccessor})
+		conv, err = schemaFromSource.SchemaFromDump(targetProfile.Conn.Sp.Project, targetProfile.Conn.Sp.Instance, sourceProfile.Driver, targetProfile.Conn.Sp.Dialect, ioHelper, &ProcessDumpByDialectImpl{ExpressionVerificationAccessor: expressionVerificationAccessor})
 	default:
 		return nil, fmt.Errorf("schema conversion for driver %s not supported", sourceProfile.Driver)
 	}
+	conv.DatabaseOptions = ddl.DatabaseOptions{
+		DefaultTimezone: targetProfile.Conn.Sp.DefaultTimezone,
+	}
+	return conv, err
 }
 
 // DataConv performs the data conversion

conversion/store_files.go

@@ -42,7 +42,7 @@ func WriteSchemaFile(conv *internal.Conv, now time.Time, name string, out *os.Fi
 	// and doesn't add backticks around table and column names. This file is
 	// intended for explanatory and documentation purposes, and is not strictly
 	// legal Cloud Spanner DDL (Cloud Spanner doesn't currently support comments).
-	spDDL := ddl.GetDDL(ddl.Config{Comments: true, ProtectIds: false, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences)
+	spDDL := ddl.GetDDL(ddl.Config{Comments: true, ProtectIds: false, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences, conv.DatabaseOptions)
 	if len(spDDL) == 0 {
 		spDDL = []string{"\n-- Schema is empty -- no tables found\n"}
 	}
@@ -69,7 +69,7 @@ func WriteSchemaFile(conv *internal.Conv, now time.Time, name string, out *os.Fi
 
 	// We change 'Comments' to false and 'ProtectIds' to true below to write out a
 	// schema file that is a legal Cloud Spanner DDL.
-	spDDL = ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences)
+	spDDL = ddl.GetDDL(ddl.Config{Comments: false, ProtectIds: true, Tables: true, ForeignKeys: true, SpDialect: conv.SpDialect, Source: driver}, conv.SpSchema, conv.SpSequences, conv.DatabaseOptions)
 	if len(spDDL) == 0 {
 		spDDL = []string{"\n-- Schema is empty -- no tables found\n"}
 	}

docs/cli/flags.md

@@ -71,4 +71,10 @@ appropriate instance using gcloud.
 * **`dialect`**: Specifies the dialect of Spanner database. By default, Spanner
 databases are created with GoogleSQL dialect. You can override the same by
 setting `dialect=PostgreSQL` in the `-target-profile`. Learn more about support
-for PostgreSQL dialect in Cloud Spanner [here](https://cloud.google.com/spanner/docs/postgresql-interface).
+for PostgreSQL dialect in Cloud Spanner [here](https://cloud.google.com/spanner/docs/postgresql-interface).
+
+* **`defaultTimezone`**: Specifies the default timezone of the Spanner database. Must be a valid entry from the [IANA
+Time Zone Database](https://www.iana.org/time-zones). If not specified, the default timezone is not set when creating
+the Spanner database, and the database will therefore default to `America/Los_Angeles` (the default timezone for
+Spanner databases). Note, the default timezone can only be set on an empty Spanner database without any tables; a
+warning will be logged and this setting will be ignored if the database already includes tables.

internal/convert.go

@@ -58,6 +58,7 @@ type Conv struct {
 	SpProjectId        string                  // Spanner Project Id
 	SpInstanceId       string                  // Spanner Instance Id
 	Source             string                  // Source Database type being migrated
+	DatabaseOptions    ddl.DatabaseOptions
 }
 
 type InvalidCheckExp struct {
@@ -375,6 +376,7 @@ func MakeConv() *Conv {
 		Rules:        []Rule{},
 		SpSequences:  make(map[string]ddl.Sequence),
 		SrcSequences: make(map[string]ddl.Sequence),
+		DatabaseOptions: ddl.DatabaseOptions{},
 	}
 }
 

profiles/target_profile.go

@@ -46,6 +46,7 @@ type TargetProfileConnectionSpanner struct {
 	Instance string
 	Dbname   string
 	Dialect  string
+	DefaultTimezone string
 }
 
 type TargetProfileConnection struct {
@@ -146,6 +147,9 @@ func NewTargetProfile(s string) (TargetProfile, error) {
 	if dialect, ok := params["dialect"]; ok {
 		sp.Dialect = strings.ToLower(dialect)
 	}
+	if defaultTimezone, ok := params["defaultTimezone"]; ok {
+		sp.DefaultTimezone = defaultTimezone
+	}
 	if sp.Dialect == "" {
 		sp.Dialect = constants.DIALECT_GOOGLESQL
 	} else if sp.Dialect != constants.DIALECT_POSTGRESQL && sp.Dialect != constants.DIALECT_GOOGLESQL {
@@ -157,6 +161,13 @@ func NewTargetProfile(s string) (TargetProfile, error) {
 		return TargetProfile{}, fmt.Errorf("found empty string for instance. please specify instance (spanner instance) in the target-profile")
 	}
 
+	if sp.DefaultTimezone != "" {
+		_, err := time.LoadLocation(sp.DefaultTimezone)
+		if err != nil {
+			return TargetProfile{}, err
+		}
+	}
+
 	conn := TargetProfileConnection{Ty: TargetProfileConnectionTypeSpanner, Sp: sp}
 	return TargetProfile{Ty: TargetProfileTypeConnection, Conn: conn}, nil
 }