diff --git a/internal/cmd/commands/database/init.go b/internal/cmd/commands/database/init.go index 0627ef6972..c55179677f 100644 --- a/internal/cmd/commands/database/init.go +++ b/internal/cmd/commands/database/init.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/boundary/internal/cmd/base" "github.com/hashicorp/boundary/internal/cmd/config" "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/migrations" "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/internal/types/scope" "github.com/hashicorp/boundary/sdk/wrapper" @@ -37,6 +38,7 @@ type InitCommand struct { flagLogLevel string flagLogFormat string flagMigrationUrl string + flagAllowDevMigrations bool flagSkipInitialLoginRoleCreation bool flagSkipAuthMethodCreation bool flagSkipScopesCreation bool @@ -116,6 +118,12 @@ func (c *InitCommand) Flags() *base.FlagSets { f = set.NewFlagSet("Init Options") + f.BoolVar(&base.BoolVar{ + Name: "allow-development-migrations", + Target: &c.flagAllowDevMigrations, + Usage: "If set the init will continue even if the schema includes database update steps that may not be supported in the next official release. Boundary does not provide a rollback mechanism so a backup should be taken independently if needed.", + }) + f.BoolVar(&base.BoolVar{ Name: "skip-initial-login-role-creation", Target: &c.flagSkipInitialLoginRoleCreation, @@ -176,6 +184,20 @@ func (c *InitCommand) Run(args []string) (retCode int) { }() } + if migrations.DevMigration != c.flagAllowDevMigrations { + if migrations.DevMigration { + c.UI.Error(base.WrapAtLength("This version of the binary has " + + "dev database schema updates which may not be supported in the " + + "next official release. To proceed anyways please use the " + + "'-allow-development-migrations' flag.")) + return 2 + } else { + c.UI.Error(base.WrapAtLength("The '-allow-development-migrations' " + + "flag was set but this binary has no dev database schema updates.")) + return 3 + } + } + c.srv = base.NewServer(&base.Command{UI: c.UI}) if err := c.srv.SetupLogging(c.flagLogLevel, c.flagLogFormat, c.Config.LogLevel, c.Config.LogFormat); err != nil { @@ -455,6 +477,13 @@ func (c *InitCommand) ParseFlagsAndConfig(args []string) int { return 1 } + // Validation + switch { + case len(c.flagConfig) == 0: + c.UI.Error("Must specify a config file using -config") + return 1 + } + wrapperPath := c.flagConfig if c.flagConfigKms != "" { wrapperPath = c.flagConfigKms @@ -472,13 +501,6 @@ func (c *InitCommand) ParseFlagsAndConfig(args []string) int { } } - // Validation - switch { - case len(c.flagConfig) == 0: - c.UI.Error("Must specify a config file using -config") - return 1 - } - c.Config, err = config.LoadFile(c.flagConfig, wrapper) if err != nil { c.UI.Error("Error parsing config: " + err.Error()) diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 7f72101042..fd3bfaae87 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -82,7 +82,7 @@ func TestMigrate(t *testing.T) { name: "valid", args: args{ connectionUrl: url, - migrationsDirectory: "migrations/postgres", + migrationsDirectory: "migrations/postgres/0", }, wantErr: false, }, @@ -90,7 +90,7 @@ func TestMigrate(t *testing.T) { name: "bad-url", args: args{ connectionUrl: "", - migrationsDirectory: "migrations/postgres", + migrationsDirectory: "migrations/postgres/0", }, wantErr: true, }, diff --git a/internal/db/migrations/genmigrations/generate.go b/internal/db/migrations/genmigrations/generate.go index cdf67ce473..3724f969ca 100644 --- a/internal/db/migrations/genmigrations/generate.go +++ b/internal/db/migrations/genmigrations/generate.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "sort" + "strconv" "strings" "text/template" ) @@ -20,7 +21,7 @@ func generate(dialect string) { fmt.Printf("error opening dir with dialect %s: %v\n", dialect, err) os.Exit(1) } - names, err := dir.Readdirnames(0) + versions, err := dir.Readdirnames(0) if err != nil { fmt.Printf("error reading dir names with dialect %s: %v\n", dialect, err) os.Exit(1) @@ -28,34 +29,85 @@ func generate(dialect string) { outBuf := bytes.NewBuffer(nil) valuesBuf := bytes.NewBuffer(nil) - sort.Strings(names) + sort.Strings(versions) - for _, name := range names { - if !strings.HasSuffix(name, ".sql") { - continue + isDev := false + largestVer := 0 + for _, ver := range versions { + var verVal int + switch ver { + case "dev": + verVal = largestVer + 1 + default: + if verVal, err = strconv.Atoi(ver); err != nil { + fmt.Printf("error reading major schema version directory %q. Must be a number or 'dev'\n", ver) + os.Exit(1) + } + if verVal > largestVer { + largestVer = verVal + } } - contents, err := ioutil.ReadFile(fmt.Sprintf("%s/%s/%s", baseDir, dialect, name)) + + dir, err := os.Open(fmt.Sprintf("%s/%s/%s", baseDir, dialect, ver)) if err != nil { - fmt.Printf("error opening file %s with dialect %s: %v", name, dialect, err) + fmt.Printf("error opening dir with dialect %s: %v\n", dialect, err) os.Exit(1) } - if err := migrationsValueTemplate.Execute(valuesBuf, struct { - Name string - Contents string - }{ - Name: name, - Contents: string(contents), - }); err != nil { - fmt.Printf("error executing migrations value template for file %s: %s", name, err) + names, err := dir.Readdirnames(0) + if err != nil { + fmt.Printf("error reading dir names with dialect %s: %v\n", dialect, err) os.Exit(1) } + + if ver == "dev" && len(names) > 0 { + isDev = true + } + + sort.Strings(names) + for _, name := range names { + if !strings.HasSuffix(name, ".sql") { + continue + } + + contents, err := ioutil.ReadFile(fmt.Sprintf("%s/%s/%s/%s", baseDir, dialect, ver, name)) + if err != nil { + fmt.Printf("error opening file %s with dialect %s: %v", name, dialect, err) + os.Exit(1) + } + + vName := name + nameParts := strings.SplitN(name, "_", 2) + if len(nameParts) != 2 { + continue + } + + nameVer, err := strconv.Atoi(nameParts[0]) + if err != nil { + fmt.Printf("Unable to get file version from %q\n", name) + continue + } + vName = fmt.Sprintf("%02d_%s", (verVal*1000)+nameVer, nameParts[1]) + + if err := migrationsValueTemplate.Execute(valuesBuf, struct { + Name string + Contents string + }{ + Name: vName, + Contents: string(contents), + }); err != nil { + fmt.Printf("error executing migrations value template for file %s/%s: %s", ver, name, err) + os.Exit(1) + } + } } if err := migrationsTemplate.Execute(outBuf, struct { - Type string - Values string + Type string + Values string + DevMigration bool }{ - Type: dialect, - Values: valuesBuf.String(), + Type: dialect, + Values: valuesBuf.String(), + DevMigration: isDev, }); err != nil { fmt.Printf("error executing migrations value template for dialect %s: %s", dialect, err) os.Exit(1) @@ -76,6 +128,11 @@ import ( "bytes" ) +// DevMigration is true if the database schema that would be applied by +// InitStore would be from files in the /dev directory which indicates it would +// not be safe to run in a non dev environment. +var DevMigration = {{ .DevMigration }} + var {{ .Type }}Migrations = map[string]*fakeFile{ "migrations": { name: "migrations", diff --git a/internal/db/migrations/postgres.gen.go b/internal/db/migrations/postgres.gen.go index dea5eada71..c6f02c710e 100644 --- a/internal/db/migrations/postgres.gen.go +++ b/internal/db/migrations/postgres.gen.go @@ -1,6 +1,11 @@ // Code generated by "make migrations"; DO NOT EDIT. package migrations +// DevMigration is true if the database schema that would be applied by +// InitStore would be from files in the /dev directory which indicates it would +// not be safe to run in a non dev environment. +var DevMigration = false + var postgresMigrations = map[string]*fakeFile{ "migrations": { name: "migrations", diff --git a/internal/db/migrations/postgres/01_domain_types.down.sql b/internal/db/migrations/postgres/0/01_domain_types.down.sql similarity index 100% rename from internal/db/migrations/postgres/01_domain_types.down.sql rename to internal/db/migrations/postgres/0/01_domain_types.down.sql diff --git a/internal/db/migrations/postgres/01_domain_types.up.sql b/internal/db/migrations/postgres/0/01_domain_types.up.sql similarity index 100% rename from internal/db/migrations/postgres/01_domain_types.up.sql rename to internal/db/migrations/postgres/0/01_domain_types.up.sql diff --git a/internal/db/migrations/postgres/02_oplog.down.sql b/internal/db/migrations/postgres/0/02_oplog.down.sql similarity index 100% rename from internal/db/migrations/postgres/02_oplog.down.sql rename to internal/db/migrations/postgres/0/02_oplog.down.sql diff --git a/internal/db/migrations/postgres/02_oplog.up.sql b/internal/db/migrations/postgres/0/02_oplog.up.sql similarity index 100% rename from internal/db/migrations/postgres/02_oplog.up.sql rename to internal/db/migrations/postgres/0/02_oplog.up.sql diff --git a/internal/db/migrations/postgres/03_db.down.sql b/internal/db/migrations/postgres/0/03_db.down.sql similarity index 100% rename from internal/db/migrations/postgres/03_db.down.sql rename to internal/db/migrations/postgres/0/03_db.down.sql diff --git a/internal/db/migrations/postgres/03_db.up.sql b/internal/db/migrations/postgres/0/03_db.up.sql similarity index 100% rename from internal/db/migrations/postgres/03_db.up.sql rename to internal/db/migrations/postgres/0/03_db.up.sql diff --git a/internal/db/migrations/postgres/06_iam.down.sql b/internal/db/migrations/postgres/0/06_iam.down.sql similarity index 100% rename from internal/db/migrations/postgres/06_iam.down.sql rename to internal/db/migrations/postgres/0/06_iam.down.sql diff --git a/internal/db/migrations/postgres/06_iam.up.sql b/internal/db/migrations/postgres/0/06_iam.up.sql similarity index 100% rename from internal/db/migrations/postgres/06_iam.up.sql rename to internal/db/migrations/postgres/0/06_iam.up.sql diff --git a/internal/db/migrations/postgres/07_auth.down.sql b/internal/db/migrations/postgres/0/07_auth.down.sql similarity index 100% rename from internal/db/migrations/postgres/07_auth.down.sql rename to internal/db/migrations/postgres/0/07_auth.down.sql diff --git a/internal/db/migrations/postgres/07_auth.up.sql b/internal/db/migrations/postgres/0/07_auth.up.sql similarity index 100% rename from internal/db/migrations/postgres/07_auth.up.sql rename to internal/db/migrations/postgres/0/07_auth.up.sql diff --git a/internal/db/migrations/postgres/08_servers.down.sql b/internal/db/migrations/postgres/0/08_servers.down.sql similarity index 100% rename from internal/db/migrations/postgres/08_servers.down.sql rename to internal/db/migrations/postgres/0/08_servers.down.sql diff --git a/internal/db/migrations/postgres/08_servers.up.sql b/internal/db/migrations/postgres/0/08_servers.up.sql similarity index 100% rename from internal/db/migrations/postgres/08_servers.up.sql rename to internal/db/migrations/postgres/0/08_servers.up.sql diff --git a/internal/db/migrations/postgres/11_auth_token.down.sql b/internal/db/migrations/postgres/0/11_auth_token.down.sql similarity index 100% rename from internal/db/migrations/postgres/11_auth_token.down.sql rename to internal/db/migrations/postgres/0/11_auth_token.down.sql diff --git a/internal/db/migrations/postgres/11_auth_token.up.sql b/internal/db/migrations/postgres/0/11_auth_token.up.sql similarity index 100% rename from internal/db/migrations/postgres/11_auth_token.up.sql rename to internal/db/migrations/postgres/0/11_auth_token.up.sql diff --git a/internal/db/migrations/postgres/12_auth_password.down.sql b/internal/db/migrations/postgres/0/12_auth_password.down.sql similarity index 100% rename from internal/db/migrations/postgres/12_auth_password.down.sql rename to internal/db/migrations/postgres/0/12_auth_password.down.sql diff --git a/internal/db/migrations/postgres/12_auth_password.up.sql b/internal/db/migrations/postgres/0/12_auth_password.up.sql similarity index 100% rename from internal/db/migrations/postgres/12_auth_password.up.sql rename to internal/db/migrations/postgres/0/12_auth_password.up.sql diff --git a/internal/db/migrations/postgres/13_auth_password_argon.down.sql b/internal/db/migrations/postgres/0/13_auth_password_argon.down.sql similarity index 100% rename from internal/db/migrations/postgres/13_auth_password_argon.down.sql rename to internal/db/migrations/postgres/0/13_auth_password_argon.down.sql diff --git a/internal/db/migrations/postgres/13_auth_password_argon.up.sql b/internal/db/migrations/postgres/0/13_auth_password_argon.up.sql similarity index 100% rename from internal/db/migrations/postgres/13_auth_password_argon.up.sql rename to internal/db/migrations/postgres/0/13_auth_password_argon.up.sql diff --git a/internal/db/migrations/postgres/14_auth_password_views.down.sql b/internal/db/migrations/postgres/0/14_auth_password_views.down.sql similarity index 100% rename from internal/db/migrations/postgres/14_auth_password_views.down.sql rename to internal/db/migrations/postgres/0/14_auth_password_views.down.sql diff --git a/internal/db/migrations/postgres/14_auth_password_views.up.sql b/internal/db/migrations/postgres/0/14_auth_password_views.up.sql similarity index 100% rename from internal/db/migrations/postgres/14_auth_password_views.up.sql rename to internal/db/migrations/postgres/0/14_auth_password_views.up.sql diff --git a/internal/db/migrations/postgres/20_host.down.sql b/internal/db/migrations/postgres/0/20_host.down.sql similarity index 100% rename from internal/db/migrations/postgres/20_host.down.sql rename to internal/db/migrations/postgres/0/20_host.down.sql diff --git a/internal/db/migrations/postgres/20_host.up.sql b/internal/db/migrations/postgres/0/20_host.up.sql similarity index 100% rename from internal/db/migrations/postgres/20_host.up.sql rename to internal/db/migrations/postgres/0/20_host.up.sql diff --git a/internal/db/migrations/postgres/22_static_host.down.sql b/internal/db/migrations/postgres/0/22_static_host.down.sql similarity index 100% rename from internal/db/migrations/postgres/22_static_host.down.sql rename to internal/db/migrations/postgres/0/22_static_host.down.sql diff --git a/internal/db/migrations/postgres/22_static_host.up.sql b/internal/db/migrations/postgres/0/22_static_host.up.sql similarity index 100% rename from internal/db/migrations/postgres/22_static_host.up.sql rename to internal/db/migrations/postgres/0/22_static_host.up.sql diff --git a/internal/db/migrations/postgres/30_keys.down.sql b/internal/db/migrations/postgres/0/30_keys.down.sql similarity index 100% rename from internal/db/migrations/postgres/30_keys.down.sql rename to internal/db/migrations/postgres/0/30_keys.down.sql diff --git a/internal/db/migrations/postgres/30_keys.up.sql b/internal/db/migrations/postgres/0/30_keys.up.sql similarity index 100% rename from internal/db/migrations/postgres/30_keys.up.sql rename to internal/db/migrations/postgres/0/30_keys.up.sql diff --git a/internal/db/migrations/postgres/31_keys.down.sql b/internal/db/migrations/postgres/0/31_keys.down.sql similarity index 100% rename from internal/db/migrations/postgres/31_keys.down.sql rename to internal/db/migrations/postgres/0/31_keys.down.sql diff --git a/internal/db/migrations/postgres/31_keys.up.sql b/internal/db/migrations/postgres/0/31_keys.up.sql similarity index 100% rename from internal/db/migrations/postgres/31_keys.up.sql rename to internal/db/migrations/postgres/0/31_keys.up.sql diff --git a/internal/db/migrations/postgres/40_targets.down.sql b/internal/db/migrations/postgres/0/40_targets.down.sql similarity index 100% rename from internal/db/migrations/postgres/40_targets.down.sql rename to internal/db/migrations/postgres/0/40_targets.down.sql diff --git a/internal/db/migrations/postgres/40_targets.up.sql b/internal/db/migrations/postgres/0/40_targets.up.sql similarity index 100% rename from internal/db/migrations/postgres/40_targets.up.sql rename to internal/db/migrations/postgres/0/40_targets.up.sql diff --git a/internal/db/migrations/postgres/41_targets.down.sql b/internal/db/migrations/postgres/0/41_targets.down.sql similarity index 100% rename from internal/db/migrations/postgres/41_targets.down.sql rename to internal/db/migrations/postgres/0/41_targets.down.sql diff --git a/internal/db/migrations/postgres/41_targets.up.sql b/internal/db/migrations/postgres/0/41_targets.up.sql similarity index 100% rename from internal/db/migrations/postgres/41_targets.up.sql rename to internal/db/migrations/postgres/0/41_targets.up.sql diff --git a/internal/db/migrations/postgres/50_session.down.sql b/internal/db/migrations/postgres/0/50_session.down.sql similarity index 100% rename from internal/db/migrations/postgres/50_session.down.sql rename to internal/db/migrations/postgres/0/50_session.down.sql diff --git a/internal/db/migrations/postgres/50_session.up.sql b/internal/db/migrations/postgres/0/50_session.up.sql similarity index 100% rename from internal/db/migrations/postgres/50_session.up.sql rename to internal/db/migrations/postgres/0/50_session.up.sql diff --git a/internal/db/migrations/postgres/51_connection.down.sql b/internal/db/migrations/postgres/0/51_connection.down.sql similarity index 100% rename from internal/db/migrations/postgres/51_connection.down.sql rename to internal/db/migrations/postgres/0/51_connection.down.sql diff --git a/internal/db/migrations/postgres/51_connection.up.sql b/internal/db/migrations/postgres/0/51_connection.up.sql similarity index 100% rename from internal/db/migrations/postgres/51_connection.up.sql rename to internal/db/migrations/postgres/0/51_connection.up.sql diff --git a/internal/db/migrations/postgres/60_wh_domain_types.down.sql b/internal/db/migrations/postgres/0/60_wh_domain_types.down.sql similarity index 100% rename from internal/db/migrations/postgres/60_wh_domain_types.down.sql rename to internal/db/migrations/postgres/0/60_wh_domain_types.down.sql diff --git a/internal/db/migrations/postgres/60_wh_domain_types.up.sql b/internal/db/migrations/postgres/0/60_wh_domain_types.up.sql similarity index 100% rename from internal/db/migrations/postgres/60_wh_domain_types.up.sql rename to internal/db/migrations/postgres/0/60_wh_domain_types.up.sql diff --git a/internal/db/migrations/postgres/62_wh_datetime.down.sql b/internal/db/migrations/postgres/0/62_wh_datetime.down.sql similarity index 100% rename from internal/db/migrations/postgres/62_wh_datetime.down.sql rename to internal/db/migrations/postgres/0/62_wh_datetime.down.sql diff --git a/internal/db/migrations/postgres/62_wh_datetime.up.sql b/internal/db/migrations/postgres/0/62_wh_datetime.up.sql similarity index 100% rename from internal/db/migrations/postgres/62_wh_datetime.up.sql rename to internal/db/migrations/postgres/0/62_wh_datetime.up.sql diff --git a/internal/db/migrations/postgres/65_wh_session_dimensions.down.sql b/internal/db/migrations/postgres/0/65_wh_session_dimensions.down.sql similarity index 100% rename from internal/db/migrations/postgres/65_wh_session_dimensions.down.sql rename to internal/db/migrations/postgres/0/65_wh_session_dimensions.down.sql diff --git a/internal/db/migrations/postgres/65_wh_session_dimensions.up.sql b/internal/db/migrations/postgres/0/65_wh_session_dimensions.up.sql similarity index 100% rename from internal/db/migrations/postgres/65_wh_session_dimensions.up.sql rename to internal/db/migrations/postgres/0/65_wh_session_dimensions.up.sql diff --git a/internal/db/migrations/postgres/66_wh_session_dimensions.down.sql b/internal/db/migrations/postgres/0/66_wh_session_dimensions.down.sql similarity index 100% rename from internal/db/migrations/postgres/66_wh_session_dimensions.down.sql rename to internal/db/migrations/postgres/0/66_wh_session_dimensions.down.sql diff --git a/internal/db/migrations/postgres/66_wh_session_dimensions.up.sql b/internal/db/migrations/postgres/0/66_wh_session_dimensions.up.sql similarity index 100% rename from internal/db/migrations/postgres/66_wh_session_dimensions.up.sql rename to internal/db/migrations/postgres/0/66_wh_session_dimensions.up.sql diff --git a/internal/db/migrations/postgres/68_wh_session_facts.down.sql b/internal/db/migrations/postgres/0/68_wh_session_facts.down.sql similarity index 100% rename from internal/db/migrations/postgres/68_wh_session_facts.down.sql rename to internal/db/migrations/postgres/0/68_wh_session_facts.down.sql diff --git a/internal/db/migrations/postgres/68_wh_session_facts.up.sql b/internal/db/migrations/postgres/0/68_wh_session_facts.up.sql similarity index 100% rename from internal/db/migrations/postgres/68_wh_session_facts.up.sql rename to internal/db/migrations/postgres/0/68_wh_session_facts.up.sql diff --git a/internal/db/migrations/postgres/69_wh_session_facts.down.sql b/internal/db/migrations/postgres/0/69_wh_session_facts.down.sql similarity index 100% rename from internal/db/migrations/postgres/69_wh_session_facts.down.sql rename to internal/db/migrations/postgres/0/69_wh_session_facts.down.sql diff --git a/internal/db/migrations/postgres/69_wh_session_facts.up.sql b/internal/db/migrations/postgres/0/69_wh_session_facts.up.sql similarity index 100% rename from internal/db/migrations/postgres/69_wh_session_facts.up.sql rename to internal/db/migrations/postgres/0/69_wh_session_facts.up.sql diff --git a/internal/oplog/testing.go b/internal/oplog/testing.go index 5f6696668e..79b0c653c5 100644 --- a/internal/oplog/testing.go +++ b/internal/oplog/testing.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/golang-migrate/migrate/v4" + "github.com/hashicorp/boundary/internal/db/migrations" "github.com/hashicorp/boundary/internal/docker" "github.com/hashicorp/boundary/internal/oplog/oplog_test" wrapping "github.com/hashicorp/go-kms-wrapping" @@ -79,7 +80,9 @@ func testWrapper(t *testing.T) wrapping.Wrapper { func testInitStore(t *testing.T, cleanup func() error, url string) { t.Helper() // run migrations - m, err := migrate.New("file://../db/migrations/postgres", url) + source, err := migrations.NewMigrationSource("postgres") + require.NoError(t, err, "Error creating migration source") + m, err := migrate.NewWithSourceInstance("postgres", source, url) require.NoError(t, err, "Error creating migrations") if err := m.Up(); err != nil && err != migrate.ErrNoChange {