Generate postgres.gen.go using schema version directory structure (#808)

* Reorganize the db migration .sql scripts into individual schema version directories and generate the sql httpfs fakeFile accordingly.

* Created a /dev schema version directory to hold db schema updates that have not been finalized.

* Add an 'allow-development-migrations' flag to allow `boundary database init` to be executed even when there are schema updates in /dev.

internal/cmd/commands/database/init.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/db/migrations"
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/types/scope"
 	"github.com/hashicorp/boundary/sdk/wrapper"
@@ -37,6 +38,7 @@ type InitCommand struct {
 	flagLogLevel                     string
 	flagLogFormat                    string
 	flagMigrationUrl                 string
+	flagAllowDevMigrations           bool
 	flagSkipInitialLoginRoleCreation bool
 	flagSkipAuthMethodCreation       bool
 	flagSkipScopesCreation           bool
@@ -116,6 +118,12 @@ func (c *InitCommand) Flags() *base.FlagSets {
 
 	f = set.NewFlagSet("Init Options")
 
+	f.BoolVar(&base.BoolVar{
+		Name:   "allow-development-migrations",
+		Target: &c.flagAllowDevMigrations,
+		Usage:  "If set the init will continue even if the schema includes database update steps that may not be supported in the next official release.  Boundary does not provide a rollback mechanism so a backup should be taken independently if needed.",
+	})
+
 	f.BoolVar(&base.BoolVar{
 		Name:   "skip-initial-login-role-creation",
 		Target: &c.flagSkipInitialLoginRoleCreation,
@@ -176,6 +184,20 @@ func (c *InitCommand) Run(args []string) (retCode int) {
 		}()
 	}
 
+	if migrations.DevMigration != c.flagAllowDevMigrations {
+		if migrations.DevMigration {
+			c.UI.Error(base.WrapAtLength("This version of the binary has " +
+				"dev database schema updates which may not be supported in the " +
+				"next official release. To proceed anyways please use the " +
+				"'-allow-development-migrations' flag."))
+			return 2
+		} else {
+			c.UI.Error(base.WrapAtLength("The '-allow-development-migrations' " +
+				"flag was set but this binary has no dev database schema updates."))
+			return 3
+		}
+	}
+
 	c.srv = base.NewServer(&base.Command{UI: c.UI})
 
 	if err := c.srv.SetupLogging(c.flagLogLevel, c.flagLogFormat, c.Config.LogLevel, c.Config.LogFormat); err != nil {
@@ -455,6 +477,13 @@ func (c *InitCommand) ParseFlagsAndConfig(args []string) int {
 		return 1
 	}
 
+	// Validation
+	switch {
+	case len(c.flagConfig) == 0:
+		c.UI.Error("Must specify a config file using -config")
+		return 1
+	}
+
 	wrapperPath := c.flagConfig
 	if c.flagConfigKms != "" {
 		wrapperPath = c.flagConfigKms
@@ -472,13 +501,6 @@ func (c *InitCommand) ParseFlagsAndConfig(args []string) int {
 		}
 	}
 
-	// Validation
-	switch {
-	case len(c.flagConfig) == 0:
-		c.UI.Error("Must specify a config file using -config")
-		return 1
-	}
-
 	c.Config, err = config.LoadFile(c.flagConfig, wrapper)
 	if err != nil {
 		c.UI.Error("Error parsing config: " + err.Error())

internal/db/db_test.go

@@ -82,7 +82,7 @@ func TestMigrate(t *testing.T) {
 			name: "valid",
 			args: args{
 				connectionUrl:       url,
-				migrationsDirectory: "migrations/postgres",
+				migrationsDirectory: "migrations/postgres/0",
 			},
 			wantErr: false,
 		},
@@ -90,7 +90,7 @@ func TestMigrate(t *testing.T) {
 			name: "bad-url",
 			args: args{
 				connectionUrl:       "",
-				migrationsDirectory: "migrations/postgres",
+				migrationsDirectory: "migrations/postgres/0",
 			},
 			wantErr: true,
 		},

internal/db/migrations/genmigrations/generate.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"text/template"
 )
@@ -20,7 +21,7 @@ func generate(dialect string) {
 		fmt.Printf("error opening dir with dialect %s: %v\n", dialect, err)
 		os.Exit(1)
 	}
-	names, err := dir.Readdirnames(0)
+	versions, err := dir.Readdirnames(0)
 	if err != nil {
 		fmt.Printf("error reading dir names with dialect %s: %v\n", dialect, err)
 		os.Exit(1)
@@ -28,34 +29,85 @@ func generate(dialect string) {
 	outBuf := bytes.NewBuffer(nil)
 	valuesBuf := bytes.NewBuffer(nil)
 
-	sort.Strings(names)
+	sort.Strings(versions)
 
-	for _, name := range names {
-		if !strings.HasSuffix(name, ".sql") {
-			continue
+	isDev := false
+	largestVer := 0
+	for _, ver := range versions {
+		var verVal int
+		switch ver {
+		case "dev":
+			verVal = largestVer + 1
+		default:
+			if verVal, err = strconv.Atoi(ver); err != nil {
+				fmt.Printf("error reading major schema version directory %q.  Must be a number or 'dev'\n", ver)
+				os.Exit(1)
+			}
+			if verVal > largestVer {
+				largestVer = verVal
+			}
 		}
-		contents, err := ioutil.ReadFile(fmt.Sprintf("%s/%s/%s", baseDir, dialect, name))
+
+		dir, err := os.Open(fmt.Sprintf("%s/%s/%s", baseDir, dialect, ver))
 		if err != nil {
-			fmt.Printf("error opening file %s with dialect %s: %v", name, dialect, err)
+			fmt.Printf("error opening dir with dialect %s: %v\n", dialect, err)
 			os.Exit(1)
 		}
-		if err := migrationsValueTemplate.Execute(valuesBuf, struct {
-			Name     string
-			Contents string
-		}{
-			Name:     name,
-			Contents: string(contents),
-		}); err != nil {
-			fmt.Printf("error executing migrations value template for file %s: %s", name, err)
+		names, err := dir.Readdirnames(0)
+		if err != nil {
+			fmt.Printf("error reading dir names with dialect %s: %v\n", dialect, err)
 			os.Exit(1)
 		}
+
+		if ver == "dev" && len(names) > 0 {
+			isDev = true
+		}
+
+		sort.Strings(names)
+		for _, name := range names {
+			if !strings.HasSuffix(name, ".sql") {
+				continue
+			}
+
+			contents, err := ioutil.ReadFile(fmt.Sprintf("%s/%s/%s/%s", baseDir, dialect, ver, name))
+			if err != nil {
+				fmt.Printf("error opening file %s with dialect %s: %v", name, dialect, err)
+				os.Exit(1)
+			}
+
+			vName := name
+			nameParts := strings.SplitN(name, "_", 2)
+			if len(nameParts) != 2 {
+				continue
+			}
+
+			nameVer, err := strconv.Atoi(nameParts[0])
+			if err != nil {
+				fmt.Printf("Unable to get file version from %q\n", name)
+				continue
+			}
+			vName = fmt.Sprintf("%02d_%s", (verVal*1000)+nameVer, nameParts[1])
+
+			if err := migrationsValueTemplate.Execute(valuesBuf, struct {
+				Name     string
+				Contents string
+			}{
+				Name:     vName,
+				Contents: string(contents),
+			}); err != nil {
+				fmt.Printf("error executing migrations value template for file %s/%s: %s", ver, name, err)
+				os.Exit(1)
+			}
+		}
 	}
 	if err := migrationsTemplate.Execute(outBuf, struct {
-		Type   string
-		Values string
+		Type         string
+		Values       string
+		DevMigration bool
 	}{
-		Type:   dialect,
-		Values: valuesBuf.String(),
+		Type:         dialect,
+		Values:       valuesBuf.String(),
+		DevMigration: isDev,
 	}); err != nil {
 		fmt.Printf("error executing migrations value template for dialect %s: %s", dialect, err)
 		os.Exit(1)
@@ -76,6 +128,11 @@ import (
 	"bytes"
 )
 
+// DevMigration is true if the database schema that would be applied by
+// InitStore would be from files in the /dev directory which indicates it would
+// not be safe to run in a non dev environment.
+var DevMigration = {{ .DevMigration }}
+
 var {{ .Type }}Migrations = map[string]*fakeFile{
 	"migrations": {
 		name: "migrations",

internal/db/migrations/postgres.gen.go

@@ -1,6 +1,11 @@
 // Code generated by "make migrations"; DO NOT EDIT.
 package migrations
 
+// DevMigration is true if the database schema that would be applied by
+// InitStore would be from files in the /dev directory which indicates it would
+// not be safe to run in a non dev environment.
+var DevMigration = false
+
 var postgresMigrations = map[string]*fakeFile{
 	"migrations": {
 		name: "migrations",

internal/oplog/testing.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/golang-migrate/migrate/v4"
+	"github.com/hashicorp/boundary/internal/db/migrations"
 	"github.com/hashicorp/boundary/internal/docker"
 	"github.com/hashicorp/boundary/internal/oplog/oplog_test"
 	wrapping "github.com/hashicorp/go-kms-wrapping"
@@ -79,7 +80,9 @@ func testWrapper(t *testing.T) wrapping.Wrapper {
 func testInitStore(t *testing.T, cleanup func() error, url string) {
 	t.Helper()
 	// run migrations
-	m, err := migrate.New("file://../db/migrations/postgres", url)
+	source, err := migrations.NewMigrationSource("postgres")
+	require.NoError(t, err, "Error creating migration source")
+	m, err := migrate.NewWithSourceInstance("postgres", source, url)
 	require.NoError(t, err, "Error creating migrations")
 
 	if err := m.Up(); err != nil && err != migrate.ErrNoChange {