diff --git a/go.mod b/go.mod index 7505396c94..d68dfd7cbe 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.4.3 github.com/hashicorp/go-retryablehttp v0.7.0 + github.com/hashicorp/go-safetemp v1.0.0 github.com/hashicorp/go-tfe v0.21.0 github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.3.0 @@ -144,7 +145,6 @@ require ( github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-msgpack v0.5.4 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-slug v0.7.0 // indirect github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/hashicorp/jsonapi v0.0.0-20210826224640-ee7dae0fb22d // indirect diff --git a/internal/getmodules/getter.go b/internal/getmodules/getter.go index 95f334762e..ba4c8d89ce 100644 --- a/internal/getmodules/getter.go +++ b/internal/getmodules/getter.go @@ -73,7 +73,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{ var goGetterGetters = map[string]getter.Getter{ "file": new(getter.FileGetter), "gcs": new(getter.GCSGetter), - "git": new(getter.GitGetter), + "git": new(gitGetter), "hg": new(getter.HgGetter), "s3": new(getter.S3Getter), "http": getterHTTPGetter, diff --git a/internal/getmodules/git_getter.go b/internal/getmodules/git_getter.go new file mode 100644 index 0000000000..535112e7ec --- /dev/null +++ b/internal/getmodules/git_getter.go @@ -0,0 +1,382 @@ +package getmodules + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io/ioutil" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "syscall" + + getter "github.com/hashicorp/go-getter" + urlhelper "github.com/hashicorp/go-getter/helper/url" + safetemp "github.com/hashicorp/go-safetemp" + version "github.com/hashicorp/go-version" +) + +// getter is our base getter; it regroups +// fields all getters have in common. +type getterCommon struct { + client *getter.Client +} + +func (g *getterCommon) SetClient(c *getter.Client) { g.client = c } + +// Context tries to returns the Contex from the getter's +// client. otherwise context.Background() is returned. +func (g *getterCommon) Context() context.Context { + if g == nil || g.client == nil { + return context.Background() + } + return g.client.Ctx +} + +// gitGetter is a temporary fork of getter.GitGetter to allow us to tactically +// fix https://github.com/hashicorp/terraform/issues/30119 only within +// Terraform. +// +// This should be only a brief workaround to help us decouple work on the +// Terraform CLI v1.1.1 release so that we can get it done without having to +// coordinate with every other go-getter caller first. However, this fork +// should be healed promptly after v1.1.1 by upstreaming something like this +// fix into upstream go-getter, so that other go-getter callers can also +// benefit from it. +type gitGetter struct { + getterCommon +} + +var defaultBranchRegexp = regexp.MustCompile(`\s->\sorigin/(.*)`) +var lsRemoteSymRefRegexp = regexp.MustCompile(`ref: refs/heads/([^\s]+).*`) + +func (g *gitGetter) ClientMode(_ *url.URL) (getter.ClientMode, error) { + return getter.ClientModeDir, nil +} + +func (g *gitGetter) Get(dst string, u *url.URL) error { + ctx := g.Context() + if _, err := exec.LookPath("git"); err != nil { + return fmt.Errorf("git must be available and on the PATH") + } + + // The port number must be parseable as an integer. If not, the user + // was probably trying to use a scp-style address, in which case the + // ssh:// prefix must be removed to indicate that. + // + // This is not necessary in versions of Go which have patched + // CVE-2019-14809 (e.g. Go 1.12.8+) + if portStr := u.Port(); portStr != "" { + if _, err := strconv.ParseUint(portStr, 10, 16); err != nil { + return fmt.Errorf("invalid port number %q; if using the \"scp-like\" git address scheme where a colon introduces the path instead, remove the ssh:// portion and use just the git:: prefix", portStr) + } + } + + // Extract some query parameters we use + var ref, sshKey string + var depth int + q := u.Query() + if len(q) > 0 { + ref = q.Get("ref") + q.Del("ref") + + sshKey = q.Get("sshkey") + q.Del("sshkey") + + if n, err := strconv.Atoi(q.Get("depth")); err == nil { + depth = n + } + q.Del("depth") + + // Copy the URL + var newU url.URL = *u + u = &newU + u.RawQuery = q.Encode() + } + + var sshKeyFile string + if sshKey != "" { + // Check that the git version is sufficiently new. + if err := checkGitVersion("2.3"); err != nil { + return fmt.Errorf("Error using ssh key: %v", err) + } + + // We have an SSH key - decode it. + raw, err := base64.StdEncoding.DecodeString(sshKey) + if err != nil { + return err + } + + // Create a temp file for the key and ensure it is removed. + fh, err := ioutil.TempFile("", "go-getter") + if err != nil { + return err + } + sshKeyFile = fh.Name() + defer os.Remove(sshKeyFile) + + // Set the permissions prior to writing the key material. + if err := os.Chmod(sshKeyFile, 0600); err != nil { + return err + } + + // Write the raw key into the temp file. + _, err = fh.Write(raw) + fh.Close() + if err != nil { + return err + } + } + + // Clone or update the repository + _, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + err = g.update(ctx, dst, sshKeyFile, ref, depth) + } else { + err = g.clone(ctx, dst, sshKeyFile, u, ref, depth) + } + if err != nil { + return err + } + + // Next: check out the proper tag/branch if it is specified, and checkout + if ref != "" { + if err := g.checkout(dst, ref); err != nil { + return err + } + } + + // Lastly, download any/all submodules. + return g.fetchSubmodules(ctx, dst, sshKeyFile, depth) +} + +// GetFile for Git doesn't support updating at this time. It will download +// the file every time. +func (g *gitGetter) GetFile(dst string, u *url.URL) error { + td, tdcloser, err := safetemp.Dir("", "getter") + if err != nil { + return err + } + defer tdcloser.Close() + + // Get the filename, and strip the filename from the URL so we can + // just get the repository directly. + filename := filepath.Base(u.Path) + u.Path = filepath.Dir(u.Path) + + // Get the full repository + if err := g.Get(td, u); err != nil { + return err + } + + // Copy the single file + u, err = urlhelper.Parse(fmtFileURL(filepath.Join(td, filename))) + if err != nil { + return err + } + + fg := &getter.FileGetter{Copy: true} + return fg.GetFile(dst, u) +} + +func (g *gitGetter) checkout(dst string, ref string) error { + cmd := exec.Command("git", "checkout", ref) + cmd.Dir = dst + return getRunCommand(cmd) +} + +func (g *gitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, ref string, depth int) error { + args := []string{"clone"} + + if ref == "" { + ref = findRemoteDefaultBranch(u) + } + if depth > 0 { + args = append(args, "--depth", strconv.Itoa(depth)) + } + + args = append(args, "--branch", ref, u.String(), dst) + cmd := exec.CommandContext(ctx, "git", args...) + setupGitEnv(cmd, sshKeyFile) + return getRunCommand(cmd) +} + +func (g *gitGetter) update(ctx context.Context, dst, sshKeyFile, ref string, depth int) error { + // Determine if we're a branch. If we're NOT a branch, then we just + // switch to master prior to checking out + cmd := exec.CommandContext(ctx, "git", "show-ref", "-q", "--verify", "refs/heads/"+ref) + cmd.Dir = dst + + if getRunCommand(cmd) != nil { + // Not a branch, switch to default branch. This will also catch + // non-existent branches, in which case we want to switch to default + // and then checkout the proper branch later. + ref = findDefaultBranch(dst) + } + + // We have to be on a branch to pull + if err := g.checkout(dst, ref); err != nil { + return err + } + + if depth > 0 { + cmd = exec.Command("git", "pull", "--depth", strconv.Itoa(depth), "--ff-only") + } else { + cmd = exec.Command("git", "pull", "--ff-only") + } + + cmd.Dir = dst + setupGitEnv(cmd, sshKeyFile) + return getRunCommand(cmd) +} + +// fetchSubmodules downloads any configured submodules recursively. +func (g *gitGetter) fetchSubmodules(ctx context.Context, dst, sshKeyFile string, depth int) error { + args := []string{"submodule", "update", "--init", "--recursive"} + if depth > 0 { + args = append(args, "--depth", strconv.Itoa(depth)) + } + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = dst + setupGitEnv(cmd, sshKeyFile) + return getRunCommand(cmd) +} + +// findDefaultBranch checks the repo's origin remote for its default branch +// (generally "master"). "master" is returned if an origin default branch +// can't be determined. +func findDefaultBranch(dst string) string { + var stdoutbuf bytes.Buffer + cmd := exec.Command("git", "branch", "-r", "--points-at", "refs/remotes/origin/HEAD") + cmd.Dir = dst + cmd.Stdout = &stdoutbuf + err := cmd.Run() + matches := defaultBranchRegexp.FindStringSubmatch(stdoutbuf.String()) + if err != nil || matches == nil { + return "master" + } + return matches[len(matches)-1] +} + +// findRemoteDefaultBranch checks the remote repo's HEAD symref to return the remote repo's +// default branch. "master" is returned if no HEAD symref exists. +func findRemoteDefaultBranch(u *url.URL) string { + var stdoutbuf bytes.Buffer + cmd := exec.Command("git", "ls-remote", "--symref", u.String(), "HEAD") + cmd.Stdout = &stdoutbuf + err := cmd.Run() + matches := lsRemoteSymRefRegexp.FindStringSubmatch(stdoutbuf.String()) + if err != nil || matches == nil { + return "master" + } + return matches[len(matches)-1] +} + +// setupGitEnv sets up the environment for the given command. This is used to +// pass configuration data to git and ssh and enables advanced cloning methods. +func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) { + const gitSSHCommand = "GIT_SSH_COMMAND=" + var sshCmd []string + + // If we have an existing GIT_SSH_COMMAND, we need to append our options. + // We will also remove our old entry to make sure the behavior is the same + // with versions of Go < 1.9. + env := os.Environ() + for i, v := range env { + if strings.HasPrefix(v, gitSSHCommand) && len(v) > len(gitSSHCommand) { + sshCmd = []string{v} + + env[i], env[len(env)-1] = env[len(env)-1], env[i] + env = env[:len(env)-1] + break + } + } + + if len(sshCmd) == 0 { + sshCmd = []string{gitSSHCommand + "ssh"} + } + + if sshKeyFile != "" { + // We have an SSH key temp file configured, tell ssh about this. + if runtime.GOOS == "windows" { + sshKeyFile = strings.Replace(sshKeyFile, `\`, `/`, -1) + } + sshCmd = append(sshCmd, "-i", sshKeyFile) + } + + env = append(env, strings.Join(sshCmd, " ")) + cmd.Env = env +} + +// checkGitVersion is used to check the version of git installed on the system +// against a known minimum version. Returns an error if the installed version +// is older than the given minimum. +func checkGitVersion(min string) error { + want, err := version.NewVersion(min) + if err != nil { + return err + } + + out, err := exec.Command("git", "version").Output() + if err != nil { + return err + } + + fields := strings.Fields(string(out)) + if len(fields) < 3 { + return fmt.Errorf("Unexpected 'git version' output: %q", string(out)) + } + v := fields[2] + if runtime.GOOS == "windows" && strings.Contains(v, ".windows.") { + // on windows, git version will return for example: + // git version 2.20.1.windows.1 + // Which does not follow the semantic versionning specs + // https://semver.org. We remove that part in order for + // go-version to not error. + v = v[:strings.Index(v, ".windows.")] + } + + have, err := version.NewVersion(v) + if err != nil { + return err + } + + if have.LessThan(want) { + return fmt.Errorf("Required git version = %s, have %s", want, have) + } + + return nil +} + +// getRunCommand is a helper that will run a command and capture the output +// in the case an error happens. +func getRunCommand(cmd *exec.Cmd) error { + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + if err == nil { + return nil + } + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + return fmt.Errorf( + "%s exited with %d: %s", + cmd.Path, + status.ExitStatus(), + buf.String()) + } + } + + return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) +} diff --git a/internal/getmodules/git_getter_test.go b/internal/getmodules/git_getter_test.go new file mode 100644 index 0000000000..79b3e15925 --- /dev/null +++ b/internal/getmodules/git_getter_test.go @@ -0,0 +1,724 @@ +package getmodules + +import ( + "bytes" + "encoding/base64" + "io/ioutil" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + getter "github.com/hashicorp/go-getter" + urlhelper "github.com/hashicorp/go-getter/helper/url" +) + +var testHasGit bool + +func init() { + if _, err := exec.LookPath("git"); err == nil { + testHasGit = true + } +} + +func TestGitGetter_impl(t *testing.T) { + var _ getter.Getter = new(gitGetter) +} + +func TestGitGetter(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + repo := testGitRepo(t, "basic") + repo.commitFile("foo.txt", "hello") + + // With a dir that doesn't exist + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "foo.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_branch(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + repo := testGitRepo(t, "branch") + repo.git("checkout", "-b", "test-branch") + repo.commitFile("branch.txt", "branch") + + q := repo.url.Query() + q.Add("ref", "test-branch") + repo.url.RawQuery = q.Encode() + + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "branch.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Get again should work + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "branch.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_remoteWithoutMaster(t *testing.T) { + if !testHasGit { + t.Log("git not found, skipping") + t.Skip() + } + + g := new(gitGetter) + dst := tempDir(t) + + repo := testGitRepo(t, "branch") + repo.git("checkout", "-b", "test-branch") + repo.commitFile("branch.txt", "branch") + + q := repo.url.Query() + repo.url.RawQuery = q.Encode() + + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "branch.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Get again should work + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "branch.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_shallowClone(t *testing.T) { + if !testHasGit { + t.Log("git not found, skipping") + t.Skip() + } + + g := new(gitGetter) + dst := tempDir(t) + + repo := testGitRepo(t, "upstream") + repo.commitFile("upstream.txt", "0") + repo.commitFile("upstream.txt", "1") + + // Specifiy a clone depth of 1 + q := repo.url.Query() + q.Add("depth", "1") + repo.url.RawQuery = q.Encode() + + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Assert rev-list count is '1' + cmd := exec.Command("git", "rev-list", "HEAD", "--count") + cmd.Dir = dst + b, err := cmd.Output() + if err != nil { + t.Fatalf("err: %s", err) + } + + out := strings.TrimSpace(string(b)) + if out != "1" { + t.Fatalf("expected rev-list count to be '1' but got %v", out) + } +} + +func TestGitGetter_shallowCloneWithTag(t *testing.T) { + if !testHasGit { + t.Log("git not found, skipping") + t.Skip() + } + + g := new(gitGetter) + dst := tempDir(t) + + repo := testGitRepo(t, "upstream") + repo.commitFile("v1.0.txt", "0") + repo.git("tag", "v1.0") + repo.commitFile("v1.1.txt", "1") + + // Specifiy a clone depth of 1 with a tag + q := repo.url.Query() + q.Add("ref", "v1.0") + q.Add("depth", "1") + repo.url.RawQuery = q.Encode() + + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Assert rev-list count is '1' + cmd := exec.Command("git", "rev-list", "HEAD", "--count") + cmd.Dir = dst + b, err := cmd.Output() + if err != nil { + t.Fatalf("err: %s", err) + } + + out := strings.TrimSpace(string(b)) + if out != "1" { + t.Fatalf("expected rev-list count to be '1' but got %v", out) + } + + // Verify the v1.0 file exists + mainPath := filepath.Join(dst, "v1.0.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the v1.1 file does not exists + mainPath = filepath.Join(dst, "v1.1.txt") + if _, err := os.Stat(mainPath); err == nil { + t.Fatalf("expected v1.1 file to not exist") + } +} + +func TestGitGetter_branchUpdate(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + // First setup the state with a fresh branch + repo := testGitRepo(t, "branch-update") + repo.git("checkout", "-b", "test-branch") + repo.commitFile("branch.txt", "branch") + + // Get the "test-branch" branch + q := repo.url.Query() + q.Add("ref", "test-branch") + repo.url.RawQuery = q.Encode() + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "branch.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Commit an update to the branch + repo.commitFile("branch-update.txt", "branch-update") + + // Get again should work + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "branch-update.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_tag(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + repo := testGitRepo(t, "tag") + repo.commitFile("tag.txt", "tag") + repo.git("tag", "v1.0") + + q := repo.url.Query() + q.Add("ref", "v1.0") + repo.url.RawQuery = q.Encode() + + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "tag.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + + // Get again should work + if err := g.Get(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath = filepath.Join(dst, "tag.txt") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_GetFile(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempTestFile(t) + defer os.RemoveAll(filepath.Dir(dst)) + + repo := testGitRepo(t, "file") + repo.commitFile("file.txt", "hello") + + // Download the file + repo.url.Path = filepath.Join(repo.url.Path, "file.txt") + if err := g.GetFile(dst, repo.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + if _, err := os.Stat(dst); err != nil { + t.Fatalf("err: %s", err) + } + assertContents(t, dst, "hello") +} + +func TestGitGetter_gitVersion(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + if runtime.GOOS == "windows" { + t.Skip("skipping on windows since the test requires sh") + } + dir, err := ioutil.TempDir("", "go-getter") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + script := filepath.Join(dir, "git") + err = ioutil.WriteFile( + script, + []byte("#!/bin/sh\necho \"git version 2.0 (Some Metadata Here)\n\""), + 0700) + if err != nil { + t.Fatal(err) + } + + defer func(v string) { + os.Setenv("PATH", v) + }(os.Getenv("PATH")) + + os.Setenv("PATH", dir) + + // Asking for a higher version throws an error + if err := checkGitVersion("2.3"); err == nil { + t.Fatal("expect git version error") + } + + // Passes when version is satisfied + if err := checkGitVersion("1.9"); err != nil { + t.Fatal(err) + } +} + +func TestGitGetter_sshKey(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) + + // avoid getting locked by a github authenticity validation prompt + os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") + defer os.Setenv("GIT_SSH_COMMAND", "") + + u, err := urlhelper.Parse("ssh://git@github.com/hashicorp/test-private-repo" + + "?sshkey=" + encodedKey) + if err != nil { + t.Fatal(err) + } + + if err := g.Get(dst, u); err != nil { + t.Fatalf("err: %s", err) + } + + readmePath := filepath.Join(dst, "README.md") + if _, err := os.Stat(readmePath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_sshSCPStyle(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) + + // avoid getting locked by a github authenticity validation prompt + os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") + defer os.Setenv("GIT_SSH_COMMAND", "") + + // This test exercises the combination of the git detector and the + // git getter, to make sure that together they make scp-style URLs work. + client := &getter.Client{ + Src: "git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey, + Dst: dst, + Pwd: ".", + + Mode: getter.ClientModeDir, + + Detectors: []getter.Detector{ + new(getter.GitDetector), + }, + Getters: map[string]getter.Getter{ + "git": g, + }, + } + + if err := client.Get(); err != nil { + t.Fatalf("client.Get failed: %s", err) + } + + readmePath := filepath.Join(dst, "README.md") + if _, err := os.Stat(readmePath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_sshExplicitPort(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) + + // avoid getting locked by a github authenticity validation prompt + os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") + defer os.Setenv("GIT_SSH_COMMAND", "") + + // This test exercises the combination of the git detector and the + // git getter, to make sure that together they make scp-style URLs work. + client := &getter.Client{ + Src: "git::ssh://git@github.com:22/hashicorp/test-private-repo?sshkey=" + encodedKey, + Dst: dst, + Pwd: ".", + + Mode: getter.ClientModeDir, + + Detectors: []getter.Detector{ + new(getter.GitDetector), + }, + Getters: map[string]getter.Getter{ + "git": g, + }, + } + + if err := client.Get(); err != nil { + t.Fatalf("client.Get failed: %s", err) + } + + readmePath := filepath.Join(dst, "README.md") + if _, err := os.Stat(readmePath); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestGitGetter_sshSCPStyleInvalidScheme(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) + + // avoid getting locked by a github authenticity validation prompt + os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") + defer os.Setenv("GIT_SSH_COMMAND", "") + + // This test exercises the combination of the git detector and the + // git getter, to make sure that together they make scp-style URLs work. + client := &getter.Client{ + Src: "git::ssh://git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey, + Dst: dst, + Pwd: ".", + + Mode: getter.ClientModeDir, + + Detectors: []getter.Detector{ + new(getter.GitDetector), + }, + Getters: map[string]getter.Getter{ + "git": g, + }, + } + + err := client.Get() + if err == nil { + t.Fatalf("get succeeded; want error") + } + + got := err.Error() + want1, want2 := `invalid source string`, `invalid port number "hashicorp"` + if !(strings.Contains(got, want1) || strings.Contains(got, want2)) { + t.Fatalf("wrong error\ngot: %s\nwant: %q or %q", got, want1, want2) + } +} + +func TestGitGetter_submodule(t *testing.T) { + if !testHasGit { + t.Skip("git not found, skipping") + } + + g := new(gitGetter) + dst := tempDir(t) + + relpath := func(basepath, targpath string) string { + relpath, err := filepath.Rel(basepath, targpath) + if err != nil { + t.Fatal(err) + } + return strings.Replace(relpath, `\`, `/`, -1) + // on windows git still prefers relatives paths + // containing `/` for submodules + } + + // Set up the grandchild + gc := testGitRepo(t, "grandchild") + gc.commitFile("grandchild.txt", "grandchild") + + // Set up the child + c := testGitRepo(t, "child") + c.commitFile("child.txt", "child") + c.git("submodule", "add", "-f", relpath(c.dir, gc.dir)) + c.git("commit", "-m", "Add grandchild submodule") + + // Set up the parent + p := testGitRepo(t, "parent") + p.commitFile("parent.txt", "parent") + p.git("submodule", "add", "-f", relpath(p.dir, c.dir)) + p.git("commit", "-m", "Add child submodule") + + // Clone the root repository + if err := g.Get(dst, p.url); err != nil { + t.Fatalf("err: %s", err) + } + + // Check that the files exist + for _, path := range []string{ + filepath.Join(dst, "parent.txt"), + filepath.Join(dst, "child", "child.txt"), + filepath.Join(dst, "child", "grandchild", "grandchild.txt"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("err: %s", err) + } + } +} + +func TestGitGetter_setupGitEnv_sshKey(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows since the test requires sh") + } + + cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND") + setupGitEnv(cmd, "/tmp/foo.pem") + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + + actual := strings.TrimSpace(string(out)) + if actual != "ssh -i /tmp/foo.pem" { + t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual) + } +} + +func TestGitGetter_setupGitEnvWithExisting_sshKey(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("skipping on windows since the test requires sh") + return + } + + // start with an existing ssh command configuration + os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes") + defer os.Setenv("GIT_SSH_COMMAND", "") + + cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND") + setupGitEnv(cmd, "/tmp/foo.pem") + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + + actual := strings.TrimSpace(string(out)) + if actual != "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i /tmp/foo.pem" { + t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual) + } +} + +// gitRepo is a helper struct which controls a single temp git repo. +type gitRepo struct { + t *testing.T + url *url.URL + dir string +} + +// testGitRepo creates a new test git repository. +func testGitRepo(t *testing.T, name string) *gitRepo { + t.Helper() + dir, err := ioutil.TempDir("", "go-getter") + if err != nil { + t.Fatal(err) + } + dir = filepath.Join(dir, name) + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + + r := &gitRepo{ + t: t, + dir: dir, + } + + url, err := urlhelper.Parse("file://" + r.dir) + if err != nil { + t.Fatal(err) + } + r.url = url + + t.Logf("initializing git repo in %s", dir) + r.git("init") + r.git("config", "user.name", "go-getter") + r.git("config", "user.email", "go-getter@hashicorp.com") + + return r +} + +// git runs a git command against the repo. +func (r *gitRepo) git(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = r.dir + bfr := bytes.NewBuffer(nil) + cmd.Stderr = bfr + if err := cmd.Run(); err != nil { + r.t.Fatal(err, bfr.String()) + } +} + +// commitFile writes and commits a text file to the repo. +func (r *gitRepo) commitFile(file, content string) { + path := filepath.Join(r.dir, file) + if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil { + r.t.Fatal(err) + } + r.git("add", file) + r.git("commit", "-m", "Adding "+file) +} + +// This is a read-only deploy key for an empty test repository. +// Note: This is split over multiple lines to avoid being disabled by key +// scanners automatically. +var testGitToken = `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA9cHsxCl3Jjgu9DHpwvmfFOl1XEdY+ShHDR/cMnzJ5ddk5/oV +Wy6EWatvyHZfRSZMwzv4PtKeUPm6iXjqWp4xdWU9khlPzozyj+U9Fq70TRVUW9E5 +T1XdQVwJE421yffr4VMMwu60wBqjI1epapH2i2inYvw9Zl9X2MXq0+jTvFvDerbT +mDtfStDPljenELAIZtWVETSvbI46gALwbxbM2292ZUIL4D6jRz0aZMmyy/twYv8r +9WGJLwmYzU518Ie7zqKW/mCTdTrV0WRiDj0MeRaPgrGY9amuHE4r9iG/cJkwpKAO +Ccz0Hs6i89u9vZnTqZU9V7weJqRAQcMjXXR6yQIDAQABAoIBAQDBzICKnGxiTlHw +rd+6qqChnAy5jWYDbZjCJ8q8YZ3RS08+g/8NXZxvHftTqM0uOaq1FviHig3gq15H +hHvCpBc6jXDFYoKFzq6FfO/0kFkE5HoWweIgxwRow0xBCDJAJ+ryUEyy+Ay/pQHb +IAjwilRS0V+WdnVw4mTjBAhPvb4jPOo97Yfy3PYUyx2F3newkqXOZy+zx3G/ANoa +ncypfMGyy76sfCWKqw4J1gVkVQLwbB6gQkXUFGYwY9sRrxbG93kQw76Flc/E/s52 +62j4v1IM0fq0t/St+Y/+s6Lkw` + `aqt3ft1nsqWcRaVDdqvMfkzgJGXlw0bGzJG5MEQ +AIBq3dHRAoGBAP8OeG/DKG2Z1VmSfzuz1pas1fbZ+F7venOBrjez3sKlb3Pyl2aH +mt2wjaTUi5v10VrHgYtOEdqyhQeUSYydWXIBKNMag0NLLrfFUKZK+57wrHWFdFjn +VgpsdkLSNTOZpC8gA5OaJ+36IcOPfGqyyP9wuuRoaYnVT1KEzqLa9FEFAoGBAPaq +pglwhil2rxjJE4zq0afQLNpAfi7Xqcrepij+xvJIcIj7nawxXuPxqRFxONE/h3yX +zkybO8wLdbHX9Iw/wc1j50Uf1Z5gHdLf7/hQJoWKpz1RnkWRy6CYON8v1tpVp0tb +OAajR/kZnzebq2mfa7pyy5zDCX++2kp/dcFwHf31AoGAE8oupBVTZLWj7TBFuP8q +LkS40U92Sv9v09iDCQVmylmFvUxcXPM2m+7f/qMTNgWrucxzC7kB/6MMWVszHbrz +vrnCTibnemgx9sZTjKOSxHFOIEw7i85fSa3Cu0qOIDPSnmlwfZpfcMKQrhjLAYhf +uhooFiLX1X78iZ2OXup4PHUCgYEAsmBrm83sp1V1gAYBBlnVbXakyNv0pCk/Vz61 +iFXeRt1NzDGxLxGw3kQnED8BaIh5kQcyn8Fud7sdzJMv/LAqlT4Ww60mzNYTGyjo +H3jOsqm3ESfRvduWFreeAQBWbiOczGjV1i8D4EbAFfWT+tjXjchwKBf+6Yt5zn/o +Bw/uEHUCgYAFs+JPOR25oRyBs7ujrMo/OY1z/eXTVVgZxY+tYGe1FJqDeFyR7ytK ++JBB1MuDwQKGm2wSIXdCzTNoIx2B9zTseiPTwT8G7vqNFhXoIaTBp4P2xIQb45mJ +7GkTsMBHwpSMOXgX9Weq3v5xOJ2WxVtjENmd6qzxcYCO5lP15O17hA== +-----END RSA PRIVATE KEY-----` + +func assertContents(t *testing.T, path string, contents string) { + data, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(data, []byte(contents)) { + t.Fatalf("bad. expected:\n\n%s\n\nGot:\n\n%s", contents, string(data)) + } +} + +func tempDir(t *testing.T) string { + dir, err := ioutil.TempDir("", "tf") + if err != nil { + t.Fatalf("err: %s", err) + } + if err := os.RemoveAll(dir); err != nil { + t.Fatalf("err: %s", err) + } + + return dir +} + +func tempTestFile(t *testing.T) string { + dir := tempDir(t) + return filepath.Join(dir, "foo") +}