mirror of https://github.com/hashicorp/packer
feat: add http datasource (#11658)
teddylear
4 years ago
committed by
GitHub
parent
b19980c26e
commit
805225a113
@ -0,0 +1,157 @@
|
||||
//go:generate packer-sdc struct-markdown
|
||||
//go:generate packer-sdc mapstructure-to-hcl2 -type DatasourceOutput,Config
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/hashicorp/packer-plugin-sdk/common"
|
||||
"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
|
||||
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
|
||||
"github.com/hashicorp/packer-plugin-sdk/template/config"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
common.PackerConfig `mapstructure:",squash"`
|
||||
// The URL to request data from. This URL must respond with a `200 OK` response and a `text/*` or `application/json` Content-Type
|
||||
Url string `mapstructure:"url" required:"true"`
|
||||
// A map of strings representing additional HTTP headers to include in the request.
|
||||
Request_headers map[string]string `mapstructure:"request_headers" required:"false"`
|
||||
}
|
||||
|
||||
type Datasource struct {
|
||||
config Config
|
||||
}
|
||||
|
||||
type DatasourceOutput struct {
|
||||
// The URL the data was requested from.
|
||||
Url string `mapstructure:"url"`
|
||||
// The raw body of the HTTP response.
|
||||
Response_body string `mapstructure:"body"`
|
||||
// A map of strings representing the response HTTP headers.
|
||||
// Duplicate headers are contatenated with , according to [RFC2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2)
|
||||
Response_headers map[string]string `mapstructure:"request_headers"`
|
||||
}
|
||||
|
||||
func (d *Datasource) ConfigSpec() hcldec.ObjectSpec {
|
||||
return d.config.FlatMapstructure().HCL2Spec()
|
||||
}
|
||||
|
||||
func (d *Datasource) Configure(raws ...interface{}) error {
|
||||
err := config.Decode(&d.config, nil, raws...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs *packersdk.MultiError
|
||||
|
||||
if d.config.Url == "" {
|
||||
errs = packersdk.MultiErrorAppend(
|
||||
errs,
|
||||
fmt.Errorf("the `url` must be specified"))
|
||||
}
|
||||
|
||||
if errs != nil && len(errs.Errors) > 0 {
|
||||
return errs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Datasource) OutputSpec() hcldec.ObjectSpec {
|
||||
return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec()
|
||||
}
|
||||
|
||||
// This is to prevent potential issues w/ binary files
|
||||
// and generally unprintable characters
|
||||
// See https://github.com/hashicorp/terraform/pull/3858#issuecomment-156856738
|
||||
func isContentTypeText(contentType string) bool {
|
||||
|
||||
parsedType, params, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
allowedContentTypes := []*regexp.Regexp{
|
||||
regexp.MustCompile("^text/.+"),
|
||||
regexp.MustCompile("^application/json$"),
|
||||
regexp.MustCompile("^application/samlmetadata\\+xml"),
|
||||
}
|
||||
|
||||
for _, r := range allowedContentTypes {
|
||||
if r.MatchString(parsedType) {
|
||||
charset := strings.ToLower(params["charset"])
|
||||
return charset == "" || charset == "utf-8" || charset == "us-ascii"
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Most of this code comes from http terraform provider data source
|
||||
// https://github.com/hashicorp/terraform-provider-http/blob/main/internal/provider/data_source.go
|
||||
func (d *Datasource) Execute() (cty.Value, error) {
|
||||
ctx := context.TODO()
|
||||
url, headers := d.config.Url, d.config.Request_headers
|
||||
client := &http.Client{}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
// TODO: How to make a test case for this?
|
||||
if err != nil {
|
||||
fmt.Println("Error creating http request")
|
||||
return cty.NullVal(cty.EmptyObject), err
|
||||
}
|
||||
|
||||
for name, value := range headers {
|
||||
req.Header.Set(name, value)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
// TODO: How to make test case for this
|
||||
if err != nil {
|
||||
fmt.Println("Error making performing http request")
|
||||
return cty.NullVal(cty.EmptyObject), err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return cty.NullVal(cty.EmptyObject), fmt.Errorf("HTTP request error. Response code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" || isContentTypeText(contentType) == false {
|
||||
fmt.Println(fmt.Sprintf(
|
||||
"Content-Type is not recognized as a text type, got %q",
|
||||
contentType))
|
||||
fmt.Println("If the content is binary data, Packer may not properly handle the contents of the response.")
|
||||
}
|
||||
|
||||
bytes, err := ioutil.ReadAll(resp.Body)
|
||||
// TODO: How to make test case for this?
|
||||
if err != nil {
|
||||
fmt.Println("Error processing response body of call")
|
||||
return cty.NullVal(cty.EmptyObject), err
|
||||
}
|
||||
|
||||
responseHeaders := make(map[string]string)
|
||||
for k, v := range resp.Header {
|
||||
// Concatenate according to RFC2616
|
||||
// cf. https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
|
||||
responseHeaders[k] = strings.Join(v, ", ")
|
||||
}
|
||||
|
||||
output := DatasourceOutput{
|
||||
Url: d.config.Url,
|
||||
Response_headers: responseHeaders,
|
||||
Response_body: string(bytes),
|
||||
}
|
||||
return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
// Code generated by "packer-sdc mapstructure-to-hcl2"; DO NOT EDIT.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"github.com/hashicorp/hcl/v2/hcldec"
|
||||
"github.com/zclconf/go-cty/cty"
|
||||
)
|
||||
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
|
||||
type FlatConfig struct {
|
||||
PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"`
|
||||
PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"`
|
||||
PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"`
|
||||
PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"`
|
||||
PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"`
|
||||
PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"`
|
||||
PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
|
||||
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
|
||||
Url *string `mapstructure:"url" required:"true" cty:"url" hcl:"url"`
|
||||
Request_headers map[string]string `mapstructure:"request_headers" required:"false" cty:"request_headers" hcl:"request_headers"`
|
||||
}
|
||||
|
||||
// FlatMapstructure returns a new FlatConfig.
|
||||
// FlatConfig is an auto-generated flat version of Config.
|
||||
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
|
||||
func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
|
||||
return new(FlatConfig)
|
||||
}
|
||||
|
||||
// HCL2Spec returns the hcl spec of a Config.
|
||||
// This spec is used by HCL to read the fields of Config.
|
||||
// The decoded values from this spec will then be applied to a FlatConfig.
|
||||
func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
|
||||
s := map[string]hcldec.Spec{
|
||||
"packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false},
|
||||
"packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false},
|
||||
"packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false},
|
||||
"packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false},
|
||||
"packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false},
|
||||
"packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false},
|
||||
"packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
|
||||
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
|
||||
"url": &hcldec.AttrSpec{Name: "url", Type: cty.String, Required: false},
|
||||
"request_headers": &hcldec.AttrSpec{Name: "request_headers", Type: cty.Map(cty.String), Required: false},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput.
|
||||
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
|
||||
type FlatDatasourceOutput struct {
|
||||
Url *string `mapstructure:"url" cty:"url" hcl:"url"`
|
||||
Response_body *string `mapstructure:"body" cty:"body" hcl:"body"`
|
||||
Response_headers map[string]string `mapstructure:"request_headers" cty:"request_headers" hcl:"request_headers"`
|
||||
}
|
||||
|
||||
// FlatMapstructure returns a new FlatDatasourceOutput.
|
||||
// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput.
|
||||
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
|
||||
func (*DatasourceOutput) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
|
||||
return new(FlatDatasourceOutput)
|
||||
}
|
||||
|
||||
// HCL2Spec returns the hcl spec of a DatasourceOutput.
|
||||
// This spec is used by HCL to read the fields of DatasourceOutput.
|
||||
// The decoded values from this spec will then be applied to a FlatDatasourceOutput.
|
||||
func (*FlatDatasourceOutput) HCL2Spec() map[string]hcldec.Spec {
|
||||
s := map[string]hcldec.Spec{
|
||||
"url": &hcldec.AttrSpec{Name: "url", Type: cty.String, Required: false},
|
||||
"body": &hcldec.AttrSpec{Name: "body", Type: cty.String, Required: false},
|
||||
"request_headers": &hcldec.AttrSpec{Name: "request_headers", Type: cty.Map(cty.String), Required: false},
|
||||
}
|
||||
return s
|
||||
}
|
||||
@ -0,0 +1,109 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/packer-plugin-sdk/acctest"
|
||||
)
|
||||
|
||||
//go:embed test-fixtures/basic.pkr.hcl
|
||||
var testDatasourceBasic string
|
||||
|
||||
//go:embed test-fixtures/empty_url.pkr.hcl
|
||||
var testDatasourceEmptyUrl string
|
||||
|
||||
//go:embed test-fixtures/404_url.pkr.hcl
|
||||
var testDatasource404Url string
|
||||
|
||||
func TestHttpDataSource(t *testing.T) {
|
||||
tests := []struct {
|
||||
Name string
|
||||
Path string
|
||||
Error bool
|
||||
Outputs map[string]string
|
||||
}{
|
||||
{
|
||||
Name: "basic_test",
|
||||
Path: testDatasourceBasic,
|
||||
Error: false,
|
||||
Outputs: map[string]string{
|
||||
"url": "url is https://www.packer.io/",
|
||||
// Check that body is not empty
|
||||
"body": "body is true",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "url_is_empty",
|
||||
Path: testDatasourceEmptyUrl,
|
||||
Error: true,
|
||||
Outputs: map[string]string{
|
||||
"error": "the `url` must be specified",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "404_url",
|
||||
Path: testDatasource404Url,
|
||||
Error: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.Name, func(t *testing.T) {
|
||||
testCase := &acctest.PluginTestCase{
|
||||
Name: tt.Name,
|
||||
Setup: func() error {
|
||||
return nil
|
||||
},
|
||||
Teardown: func() error {
|
||||
return nil
|
||||
},
|
||||
Template: tt.Path,
|
||||
Type: "http",
|
||||
Check: func(buildCommand *exec.Cmd, logfile string) error {
|
||||
if buildCommand.ProcessState != nil {
|
||||
if buildCommand.ProcessState.ExitCode() != 0 && !tt.Error {
|
||||
return fmt.Errorf("Bad exit code. Logfile: %s", logfile)
|
||||
}
|
||||
if tt.Error && buildCommand.ProcessState.ExitCode() == 0 {
|
||||
return fmt.Errorf("Expected Bad exit code.")
|
||||
}
|
||||
}
|
||||
|
||||
if tt.Outputs != nil {
|
||||
logs, err := os.Open(logfile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable find %s", logfile)
|
||||
}
|
||||
defer logs.Close()
|
||||
|
||||
logsBytes, err := ioutil.ReadAll(logs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to read %s", logfile)
|
||||
}
|
||||
logsString := string(logsBytes)
|
||||
|
||||
for key, val := range tt.Outputs {
|
||||
if matched, _ := regexp.MatchString(val+".*", logsString); !matched {
|
||||
t.Fatalf(
|
||||
"logs doesn't contain expected log %v with value %v in %q",
|
||||
key,
|
||||
val,
|
||||
logsString)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
acctest.TestPlugin(t, testCase)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
|
||||
source "null" "example" {
|
||||
communicator = "none"
|
||||
}
|
||||
|
||||
data "http" "basic" {
|
||||
url = "https://www.packer.io/thisWillFail"
|
||||
}
|
||||
|
||||
locals {
|
||||
url = "${data.http.basic.url}"
|
||||
}
|
||||
|
||||
build {
|
||||
name = "mybuild"
|
||||
sources = [
|
||||
"source.null.example"
|
||||
]
|
||||
provisioner "shell-local" {
|
||||
inline = [
|
||||
"echo data is ${local.url}",
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
source "null" "example" {
|
||||
communicator = "none"
|
||||
}
|
||||
|
||||
data "http" "basic" {
|
||||
url = "https://www.packer.io/"
|
||||
}
|
||||
|
||||
locals {
|
||||
url = "${data.http.basic.url}"
|
||||
body = "${data.http.basic.body}" != ""
|
||||
}
|
||||
|
||||
build {
|
||||
name = "mybuild"
|
||||
sources = [
|
||||
"source.null.example"
|
||||
]
|
||||
provisioner "shell-local" {
|
||||
inline = [
|
||||
"echo url is ${local.url}",
|
||||
"echo body is ${local.body}"
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
source "null" "example" {
|
||||
communicator = "none"
|
||||
}
|
||||
|
||||
data "http" "basic" {
|
||||
url = ""
|
||||
}
|
||||
|
||||
locals {
|
||||
url = "${data.http.basic.url}"
|
||||
}
|
||||
|
||||
build {
|
||||
name = "mybuild"
|
||||
sources = [
|
||||
"source.null.example"
|
||||
]
|
||||
provisioner "shell-local" {
|
||||
inline = [
|
||||
"echo data is ${local.url}",
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
---
|
||||
description: |
|
||||
The http Data Source retrieves information from an http endpoint to be used
|
||||
during Packer builds
|
||||
page_title: Http - Data Sources
|
||||
---
|
||||
|
||||
<BadgesHeader>
|
||||
<PluginBadge type="official" />
|
||||
<PluginBadge type="hcp_packer_ready" />
|
||||
</BadgesHeader>
|
||||
|
||||
# Http Data Source
|
||||
|
||||
Type: `http`
|
||||
|
||||
The `http` data source makes an HTTP GET request to the given URL and exports information about the response.
|
||||
|
||||
|
||||
## Basic Example
|
||||
|
||||
```hcl
|
||||
data "http" "example" {
|
||||
url = "https://checkpoint-api.hashicorp.com/v1/check/terraform"
|
||||
|
||||
# Optional request headers
|
||||
request_headers = {
|
||||
Accept = "application/json"
|
||||
}
|
||||
}
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
Configuration options are organized below into two categories: required and
|
||||
optional. Within each category, the available options are alphabetized and
|
||||
described.
|
||||
|
||||
### Required:
|
||||
|
||||
@include 'datasource/http/Config-required.mdx'
|
||||
|
||||
### Not Required:
|
||||
@include 'datasource/http/Config-not-required.mdx'
|
||||
|
||||
## Datasource outputs
|
||||
|
||||
The outputs for this datasource are as follows:
|
||||
|
||||
@include 'datasource/http/DatasourceOutput.mdx'
|
||||
@ -0,0 +1,5 @@
|
||||
<!-- Code generated from the comments of the Config struct in datasource/http/data.go; DO NOT EDIT MANUALLY -->
|
||||
|
||||
- `request_headers` (map[string]string) - Request headers for call
|
||||
|
||||
<!-- End of code generated from the comments of the Config struct in datasource/http/data.go; -->
|
||||
@ -0,0 +1,5 @@
|
||||
<!-- Code generated from the comments of the Config struct in datasource/http/data.go; DO NOT EDIT MANUALLY -->
|
||||
|
||||
- `url` (string) - Url where should be getting things from
|
||||
|
||||
<!-- End of code generated from the comments of the Config struct in datasource/http/data.go; -->
|
||||
@ -0,0 +1,9 @@
|
||||
<!-- Code generated from the comments of the DatasourceOutput struct in datasource/http/data.go; DO NOT EDIT MANUALLY -->
|
||||
|
||||
- `url` (string) - Url
|
||||
|
||||
- `body` (string) - Response _ body
|
||||
|
||||
- `request_headers` (map[string]string) - Response _ headers
|
||||
|
||||
<!-- End of code generated from the comments of the DatasourceOutput struct in datasource/http/data.go; -->
|
||||
Loading…
Reference in new issue