WIP - state store configuration change

internal/command/init_test.go

@@ -5763,6 +5763,80 @@ func TestInit_configErrorsImpactingStateStore(t *testing.T) {
 	}
 }
 
+func TestInit_stateStoreConfigFileChange(t *testing.T) {
+	// Create a temporary working directory and copy in test fixtures
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath("init-state-store"), td)
+	t.Chdir(td)
+
+	mockProvider := mockPluggableStateStorageProvider()
+	mockProviderAddress := addrs.NewDefaultProvider("test")
+	providerSource, close := newMockProviderSource(t, map[string][]string{
+		"hashicorp/test": {"1.2.3"},
+	})
+	defer close()
+
+	tOverrides := &testingOverrides{
+		Providers: map[addrs.Provider]providers.Factory{
+			mockProviderAddress: providers.FactoryFixed(mockProvider),
+		},
+	}
+
+	{
+		ui := new(cli.MockUi)
+		view, done := testView(t)
+		c := &InitCommand{
+			Meta: Meta{
+				testingOverrides:          tOverrides,
+				ProviderSource:            providerSource,
+				Ui:                        ui,
+				View:                      view,
+				AllowExperimentalFeatures: true,
+			},
+		}
+
+		args := []string{
+			"-enable-pluggable-state-storage-experiment",
+		}
+		if code := c.Run(args); code != 0 {
+			t.Fatalf("bad: \n%s", done(t).Stderr())
+		}
+
+		// Read our saved stateStore config and verify we have our settings
+		state := testDataStateRead(t, filepath.Join(DefaultDataDir, DefaultStateFilename))
+		if got, want := normalizeJSON(t, state.StateStore.ConfigRaw), `{"value":"foobar"}`; got != want {
+			t.Errorf("wrong config\ngot:  %s\nwant: %s", got, want)
+		}
+	}
+	{
+		ui := new(cli.MockUi)
+		view, done := testView(t)
+		c := &InitCommand{
+			Meta: Meta{
+				testingOverrides:          tOverrides,
+				ProviderSource:            providerSource,
+				Ui:                        ui,
+				View:                      view,
+				AllowExperimentalFeatures: true,
+			},
+		}
+		args := []string{
+			"-enable-pluggable-state-storage-experiment",
+			"-backend-config", "input.tfbackend.hcl",
+			"-migrate-state",
+		}
+		if code := c.Run(args); code != 0 {
+			t.Fatalf("bad: \n%s", done(t).Stderr())
+		}
+
+		// Read our saved state store config and verify we have our settings
+		state := testDataStateRead(t, filepath.Join(DefaultDataDir, DefaultStateFilename))
+		if got, want := normalizeJSON(t, state.StateStore.ConfigRaw), `{"value":"changed"}`; got != want {
+			t.Errorf("wrong config\ngot:  %s\nwant: %s", got, want)
+		}
+	}
+}
+
 // newMockProviderSource is a helper to succinctly construct a mock provider
 // source that contains a set of packages matching the given provider versions
 // that are available for installation (from temporary local files).

internal/command/meta_backend.go

@@ -1269,14 +1269,47 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di
 			return savedStateStore, diags
 		}
 
-		// Above caters only for unchanged config
+		// If our configuration (the result of both the literal configuration and given
-		// but this switch case will also handle changes,
+		// -backend-config options) is the same, then we're just initializing a previously
-		// which isn't implemented yet.
+		// configured state store. The literal configuration may differ, however, so while we
-		return nil, diags.Append(&hcl.Diagnostic{
+		// don't need to migrate, we update the state store cache hash value.
-			Severity: hcl.DiagError,
+		if !m.stateStoreConfigNeedsMigration(stateStoreConfig, s.StateStore, opts) {
-			Summary:  "Not implemented yet",
+			log.Printf("[TRACE] Meta.Backend: using already-initialized %q state store configuration", stateStoreConfig.Type)
-			Detail:   "Changing a state store configuration is not implemented yet",
+			savedStateStore, moreDiags := m.savedStateStore(sMgr)
-		})
+			diags = diags.Append(moreDiags)
+			if moreDiags.HasErrors() {
+				return nil, diags
+			}
+
+			// It's possible for a state store to be unchanged, and the config itself to
+			// have changed by moving a parameter from the config to `-backend-config`
+			// In this case, we update the Hash.
+			moreDiags = m.updateSavedStateStoreHash(cHash, sMgr)
+			if moreDiags.HasErrors() {
+				return nil, diags
+			}
+			// Verify that selected workspace exist. Otherwise prompt user to create one
+			if opts.Init && savedStateStore != nil {
+				if err := m.selectWorkspace(savedStateStore); err != nil {
+					diags = diags.Append(err)
+					return nil, diags
+				}
+			}
+
+			return savedStateStore, diags
+		}
+		log.Printf("[TRACE] Meta.Backend: state store configuration has changed (from type %q to type %q)", s.StateStore.Type, stateStoreConfig.Type)
+
+		if !opts.Init {
+			// user ran another cmd that is not init but they are required to initialize because of a potential relevant change to their state store configuration
+			initDiag := m.determineInitReason(s.StateStore.Type, stateStoreConfig.Type, cloud.ConfigChangeIrrelevant)
+			diags = diags.Append(initDiag)
+			return nil, diags
+		}
+
+		log.Printf("[WARN] state store config has changed since last init")
+
+		return m.stateStore_changed(stateStoreConfig, cHash, sMgr, opts)
 
 	default:
 		diags = diags.Append(fmt.Errorf(
@@ -1892,6 +1925,22 @@ func (m *Meta) updateSavedBackendHash(cHash int, sMgr *clistate.LocalState) tfdi
 	return diags
 }
 
+func (m *Meta) updateSavedStateStoreHash(cHash int, sMgr *clistate.LocalState) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+
+	s := sMgr.State()
+
+	if s.StateStore.Hash != uint64(cHash) {
+		s.StateStore.Hash = uint64(cHash)
+		if err := sMgr.WriteState(s); err != nil {
+			diags = diags.Append(errStateStoreWriteSavedDiag(err))
+		}
+		// No need to call PersistState as it's a no-op
+	}
+
+	return diags
+}
+
 // backend returns an operations backend that may use a backend, cloud, or state_store block for state storage.
 // Based on the supplied config, it prepares arguments to pass into (Meta).Backend, which returns the operations backend.
 //
@@ -2437,6 +2486,193 @@ func (m *Meta) stateStore_to_backend(ssSMgr *clistate.LocalState, dstBackendType
 	return dstBackend, diags
 }
 
+// stateStoreConfigNeedsMigration returns true if migration might be required to
+// move from the configured state store to the given cached state store config.
+//
+// This must be called with the synthetic *configs.StateStore that results from
+// merging in any command-line options for correct behavior.
+//
+// If either the given configuration or cached configuration are invalid then
+// this function will conservatively assume that migration is required,
+// expecting that the migration code will subsequently deal with the same
+// errors.
+func (m *Meta) stateStoreConfigNeedsMigration(cfg *configs.StateStore, cfgState *workdir.StateStoreConfigState, opts *BackendOpts) bool {
+	if cfgState == nil || cfgState.Empty() {
+		log.Print("[TRACE] stateStoreConfigNeedsMigration: no cached config, so migration is required")
+		return true
+	}
+	if cfg.Type != cfgState.Type {
+		log.Printf("[TRACE] stateStoreConfigNeedsMigration: type changed from %q to %q, so migration is required", cfgState.Type, cfg.Type)
+		return true
+	}
+
+	// TODO: change of provider FQN
+	// TODO: change of provider version
+	// TODO: change of provider configuration
+
+	// We need the state store schema to do our comparison here.
+	ssBackend, _, _, ssDiags := m.stateStoreInitFromConfig(cfg, opts.Locks)
+	if ssDiags.HasErrors() {
+		log.Printf("[ERROR] Unable to initialise state store: %s", ssDiags)
+		return true
+	}
+
+	schema := ssBackend.ConfigSchema()
+	decSpec := schema.NoneRequired().DecoderSpec()
+	givenVal, diags := hcldec.Decode(cfg.Config, decSpec, nil)
+	if diags.HasErrors() {
+		log.Printf("[TRACE] stateStoreConfigNeedsMigration: failed to decode given config; migration codepath must handle problem: %s", diags.Error())
+		return true // let the migration codepath deal with these errors
+	}
+
+	cachedVal, err := cfgState.Config(schema)
+	if err != nil {
+		log.Printf("[TRACE] stateStoreConfigNeedsMigration: failed to decode cached config; migration codepath must handle problem: %s", err)
+		return true // let the migration codepath deal with the error
+	}
+
+	// If we get all the way down here then it's the exact equality of the
+	// two decoded values that decides our outcome. It's safe to use RawEquals
+	// here (rather than Equals) because we know that unknown values can
+	// never appear in backend configurations.
+	if cachedVal.RawEquals(givenVal) {
+		log.Print("[TRACE] stateStoreConfigNeedsMigration: given configuration matches cached configuration, so no migration is required")
+		return false
+	}
+	log.Print("[TRACE] stateStoreConfigNeedsMigration: configuration values have changed, so migration is required")
+	return true
+}
+
+func (m *Meta) stateStore_changed(cfg *configs.StateStore, cfgHash int, sMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	vt := arguments.ViewJSON
+	// Set default viewtype if none was set as the StateLocker needs to know exactly
+	// what viewType we want to have.
+	if opts == nil || opts.ViewType != vt {
+		vt = arguments.ViewHuman
+	}
+
+	// TODO: print out the reason for migration
+	// if s.Backend.Type != c.Type {
+	// 	view.Output(views.BackendMigrateTypeChangeMessage, s.Backend.Type, c.Type)
+	// } else {
+	// 	view.Output(views.BackendReconfigureMessage)
+	// }
+
+	// Get the destination state store
+	dstB, storeConfigVal, providerConfigVal, moreDiags := m.stateStoreInitFromConfig(cfg, opts.Locks)
+	diags = diags.Append(moreDiags)
+	if moreDiags.HasErrors() {
+		return nil, diags
+	}
+
+	// Grab the source state store
+	srcB, srcBDiags := m.savedStateStore(sMgr)
+	diags = diags.Append(srcBDiags)
+	if srcBDiags.HasErrors() {
+		return nil, diags
+	}
+
+	// Get the old state
+	s := sMgr.State()
+
+	// Perform the migration
+	err := m.backendMigrateState(&backendMigrateOpts{
+		SourceType:      s.StateStore.Type,
+		DestinationType: cfg.Type,
+		Source:          srcB,
+		Destination:     dstB,
+		ViewType:        vt,
+	})
+	if err != nil {
+		diags = diags.Append(err)
+		return nil, diags
+	}
+
+	if m.stateLock {
+		view := views.NewStateLocker(vt, m.View)
+		stateLocker := clistate.NewLocker(m.stateLockTimeout, view)
+		if err := stateLocker.Lock(sMgr, "state store from plan"); err != nil {
+			diags = diags.Append(fmt.Errorf("Error locking state: %s", err))
+			return nil, diags
+		}
+		defer stateLocker.Unlock()
+	}
+
+	var pVersion *version.Version // This will remain nil for builtin providers or unmanaged providers.
+	if cfg.ProviderAddr.IsBuiltIn() {
+		diags = diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagWarning,
+			Summary:  "State storage is using a builtin provider",
+			Detail:   "Terraform is using a builtin provider for initializing state storage. Terraform will be less able to detect when state migrations are required in future init commands.",
+		})
+	} else {
+		isReattached, err := reattach.IsProviderReattached(cfg.ProviderAddr, os.Getenv("TF_REATTACH_PROVIDERS"))
+		if err != nil {
+			diags = diags.Append(fmt.Errorf("Unable to determine if state storage provider is reattached while initializing state store for the first time. This is a bug in Terraform and should be reported: %w", err))
+			return nil, diags
+		}
+		if isReattached {
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary:  "State storage provider is not managed by Terraform",
+				Detail:   "Terraform is using a provider supplied via TF_REATTACH_PROVIDERS for initializing state storage. Terraform will be less able to detect when state migrations are required in future init commands.",
+			})
+		} else {
+			// The provider is not built in and is being managed by Terraform
+			// This is the most common scenario, by far.
+			var vDiags tfdiags.Diagnostics
+			pVersion, vDiags = getStateStorageProviderVersion(cfg, opts.Locks)
+			diags = diags.Append(vDiags)
+			if vDiags.HasErrors() {
+				return nil, diags
+			}
+		}
+	}
+
+	// Update the state to the new configuration
+	s = sMgr.State()
+	if s == nil {
+		s = workdir.NewBackendStateFile()
+	}
+
+	s.StateStore = &workdir.StateStoreConfigState{
+		Type: cfg.Type,
+		Hash: uint64(cfgHash),
+		Provider: &workdir.ProviderConfigState{
+			Source:  &cfg.ProviderAddr,
+			Version: pVersion,
+		},
+	}
+	err = s.StateStore.SetConfig(storeConfigVal, dstB.ConfigSchema())
+	if err != nil {
+		diags = diags.Append(fmt.Errorf("Failed to set state store configuration: %w", err))
+		return nil, diags
+	}
+
+	// We need to briefly convert away from backend.Backend interface to use the method
+	// for accessing the provider schema. In this method we _always_ expect the concrete value
+	// to be backendPluggable.Pluggable.
+	plug := dstB.(*backendPluggable.Pluggable)
+	err = s.StateStore.Provider.SetConfig(providerConfigVal, plug.ProviderSchema())
+	if err != nil {
+		diags = diags.Append(fmt.Errorf("Failed to set state store provider configuration: %w", err))
+		return nil, diags
+	}
+
+	if err := sMgr.WriteState(s); err != nil {
+		diags = diags.Append(errBackendWriteSavedDiag(err))
+		return nil, diags
+	}
+	if err := sMgr.PersistState(); err != nil {
+		diags = diags.Append(errBackendWriteSavedDiag(err))
+		return nil, diags
+	}
+
+	return dstB, diags
+}
+
 // getStateStorageProviderVersion gets the current version of the state store provider that's in use. This is achieved
 // by inspecting the current locks.
 //

internal/command/meta_backend_errors.go

@@ -227,6 +227,23 @@ above, resolve it, and try again.`, innerError)
 	)
 }
 
+// errStateStoreWriteSavedDiag creates a diagnostic to present to users when
+// an init command experiences an error while writing to the backend state file.
+func errStateStoreWriteSavedDiag(innerError error) tfdiags.Diagnostic {
+	msg := fmt.Sprintf(`Error saving the state store configuration: %s
+
+Terraform saves the complete state store configuration in a local file for
+configuring the state store on future operations. This cannot be disabled. Errors
+are usually due to simple file permission errors. Please look at the error
+above, resolve it, and try again.`, innerError)
+
+	return tfdiags.Sourceless(
+		tfdiags.Error,
+		"State store initialization failed",
+		msg,
+	)
+}
+
 // errBackendNoExistingWorkspaces is returned by calling code when it expects a backend.Backend
 // to report one or more workspaces exist.
 //

internal/command/testdata/init-state-store/input.tfbackend.hcl

@@ -0,0 +1 @@
+value = "changed"