From 95ed4443bbd6a1635f5a15cd74a22708c416ee99 Mon Sep 17 00:00:00 2001 From: Zachary Shilton <4624598+zchsh@users.noreply.github.com> Date: Fri, 23 Apr 2021 11:04:20 -0400 Subject: [PATCH] website: check for unlinked content, rm split-out vagrant content (#10958) --- .github/workflows/check-unlinked-content.js | 134 +++++++++ .github/workflows/check-unlinked-content.yml | 26 ++ .../docs/post-processors/vagrant-cloud.mdx | 211 -------------- .../content/docs/post-processors/vagrant.mdx | 272 ------------------ 4 files changed, 160 insertions(+), 483 deletions(-) create mode 100644 .github/workflows/check-unlinked-content.js create mode 100644 .github/workflows/check-unlinked-content.yml delete mode 100644 website/content/docs/post-processors/vagrant-cloud.mdx delete mode 100644 website/content/docs/post-processors/vagrant.mdx diff --git a/.github/workflows/check-unlinked-content.js b/.github/workflows/check-unlinked-content.js new file mode 100644 index 000000000..01dc2ed96 --- /dev/null +++ b/.github/workflows/check-unlinked-content.js @@ -0,0 +1,134 @@ +var fs = require("fs"); +var path = require("path"); + +const COLOR_RESET = "\x1b[0m"; +const COLOR_GREEN = "\x1b[32m"; +const COLOR_RED = "\x1b[31m"; + +runCheck([ + { + contentDir: "website/content/docs", + navDataFile: "website/data/docs-nav-data.json", + }, + { + contentDir: "website/content/guides", + navDataFile: "website/data/guides-nav-data.json", + }, + { + contentDir: "website/content/intro", + navDataFile: "website/data/intro-nav-data.json", + }, +]); + +async function runCheck(baseRoutes) { + const validatedBaseRoutes = await Promise.all( + baseRoutes.map(async ({ contentDir, navDataFile }) => { + const missingRoutes = await validateMissingRoutes( + contentDir, + navDataFile + ); + return { contentDir, navDataFile, missingRoutes }; + }) + ); + const allMissingRoutes = validatedBaseRoutes.reduce((acc, baseRoute) => { + return acc.concat(baseRoute.missingRoutes); + }, []); + if (allMissingRoutes.length == 0) { + console.log( + `\n${COLOR_GREEN}✓ All content files have routes, and are included in navigation data.${COLOR_RESET}\n` + ); + } else { + validatedBaseRoutes.forEach( + ({ contentDir, navDataFile, missingRoutes }) => { + if (missingRoutes.length == 0) return true; + console.log( + `\n${COLOR_RED}Error: Missing pages found in the ${contentDir} directory.\n\nPlease add these paths to ${navDataFile}, or remove the .mdx files.\n\n${JSON.stringify( + missingRoutes, + null, + 2 + )}${COLOR_RESET}\n\n` + ); + } + ); + process.exit(1); + } +} + +async function validateMissingRoutes(contentDir, navDataFile) { + // Read in nav-data.json, and make a flattened array of nodes + const navDataPath = path.join(process.cwd(), navDataFile); + const navData = JSON.parse(fs.readFileSync(navDataPath)); + const navDataFlat = flattenNodes(navData); + // Read all files in the content directory + const files = await walkAsync(contentDir); + // Filter out content files that are already + // included in nav-data.json + const missingPages = files + // Ignore non-.mdx files + .filter((filePath) => { + return path.extname(filePath) == ".mdx"; + }) + // Transform the filePath into an expected route + .map((filePath) => { + // Get the relative filepath, that's what we'll see in the route + const contentDirPath = path.join(process.cwd(), contentDir); + const relativePath = path.relative(contentDirPath, filePath); + // Remove extensions, these will not be in routes + const pathNoExt = relativePath.replace(/\.mdx$/, ""); + // Resolve /index routes, these will not have /index in their path + const routePath = pathNoExt.replace(/\/?index$/, ""); + return routePath; + }) + // Determine if there is a match in nav-data. + // If there is no match, then this is an unlinked content file. + .filter((pathToMatch) => { + // If it's the root path index page, we know + // it'll be rendered (hard-coded into docs-page/server.js) + const isIndexPage = pathToMatch === ""; + if (isIndexPage) return false; + // Otherwise, needs a path match in nav-data + const matches = navDataFlat.filter(({ path }) => path == pathToMatch); + return matches.length == 0; + }); + return missingPages; +} + +function flattenNodes(nodes) { + return nodes.reduce((acc, n) => { + if (!n.routes) return acc.concat(n); + return acc.concat(flattenNodes(n.routes)); + }, []); +} + +function walkAsync(relativeDir) { + const dirPath = path.join(process.cwd(), relativeDir); + return new Promise((resolve, reject) => { + walk(dirPath, function (err, result) { + if (err) reject(err); + resolve(result); + }); + }); +} + +function walk(dir, done) { + var results = []; + fs.readdir(dir, function (err, list) { + if (err) return done(err); + var pending = list.length; + if (!pending) return done(null, results); + list.forEach(function (file) { + file = path.resolve(dir, file); + fs.stat(file, function (err, stat) { + if (stat && stat.isDirectory()) { + walk(file, function (err, res) { + results = results.concat(res); + if (!--pending) done(null, results); + }); + } else { + results.push(file); + if (!--pending) done(null, results); + } + }); + }); + }); +} diff --git a/.github/workflows/check-unlinked-content.yml b/.github/workflows/check-unlinked-content.yml new file mode 100644 index 000000000..b90b02a10 --- /dev/null +++ b/.github/workflows/check-unlinked-content.yml @@ -0,0 +1,26 @@ +# +# This GitHub action checks that all .mdx files in the +# the website/content directory are being published. +# It fails if any of these files are not included +# in the expected nav-data.json file. +# +# To resolve failed checks, add the listed paths +# to the corresponding nav-data.json file +# in website/data. + +name: "website: Check unlinked content" +on: + pull_request: + paths: + - "website/**" + +jobs: + check-unlinked-content: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Node + uses: actions/setup-node@v1 + - name: Check that all content files are included in navigation + run: node .github/workflows/check-unlinked-content.js diff --git a/website/content/docs/post-processors/vagrant-cloud.mdx b/website/content/docs/post-processors/vagrant-cloud.mdx deleted file mode 100644 index a4d84791d..000000000 --- a/website/content/docs/post-processors/vagrant-cloud.mdx +++ /dev/null @@ -1,211 +0,0 @@ ---- -description: | - The Vagrant Cloud post-processor enables the upload of Vagrant boxes to - Vagrant Cloud. -page_title: Vagrant Cloud - Post-Processors ---- - -# Vagrant Cloud Post-Processor - -Type: `vagrant-cloud` -Artifact BuilderId: `pearkes.post-processor.vagrant-cloud` - -[Vagrant Cloud](https://app.vagrantup.com/boxes/search) hosts and serves boxes -to Vagrant, allowing you to version and distribute boxes to an organization in a -simple way. - -The Vagrant Cloud post-processor enables the upload of Vagrant boxes to Vagrant -Cloud. Currently, the Vagrant Cloud post-processor will accept and upload boxes -supplied to it from the [Vagrant](/docs/post-processors/vagrant) or -[Artifice](/docs/post-processors/artifice) post-processors and the -[Vagrant](/docs/builders/vagrant) builder. - -You'll need to be familiar with Vagrant Cloud, have an upgraded account to -enable box hosting, and be distributing your box via the [shorthand -name](https://docs.vagrantup.com/v2/cli/box.html) configuration. - -## Workflow - -It's important to understand the workflow that using this post-processor -enforces in order to take full advantage of Vagrant and Vagrant Cloud. - -The use of this processor assume that you currently distribute, or plan to -distribute, boxes via Vagrant Cloud. It also assumes you create Vagrant Boxes -and deliver them to your team in some fashion. - -Here is an example workflow: - -1. You use Packer to build a Vagrant Box for the `virtualbox` provider -2. The `vagrant-cloud` post-processor is configured to point to the box - `hashicorp/foobar` on Vagrant Cloud via the `box_tag` configuration -3. The post-processor receives the box from the `vagrant` post-processor -4. It then creates the configured version, or verifies the existence of it, on - Vagrant Cloud -5. A provider matching the name of the Vagrant provider is then created -6. The box is uploaded to Vagrant Cloud -7. The upload is verified -8. The version is released and available to users of the box - -~> The Vagrant Cloud box (`hashicorp/foobar` in this example) must already -exist. Packer will not create the box automatically. If running Packer in -automation, consider using the -[Vagrant Cloud API](https://www.vagrantup.com/docs/vagrant-cloud/api.html) -to create the Vagrant Cloud box if it doesn't already exist. - -## Configuration - -The configuration allows you to specify the target box that you have access to -on Vagrant Cloud, as well as authentication and version information. - -### Required: - -- `box_tag` (string) - The shorthand tag for your box that maps to Vagrant - Cloud, for example `hashicorp/precise64`, which is short for - `vagrantcloud.com/hashicorp/precise64`. This box must already exist in - Vagrant Cloud. Packer will not create the box automatically. - -- `version` (string) - The version number, typically incrementing a previous - version. The version string is validated based on [Semantic - Versioning](http://semver.org/). The string must match a pattern that could - be semver, and doesn't validate that the version comes after your previous - versions. - -- `access_token` (string) - Your access token for the Vagrant Cloud API. This - can be generated on your [tokens - page](https://app.vagrantup.com/settings/security). If not specified, the - environment will be searched. First, `VAGRANT_CLOUD_TOKEN` is checked, and - if nothing is found, finally `ATLAS_TOKEN` will be used. This is required - unless you are using a private hosting solution (i.e. `vagrant_cloud_url` - has been populated). - - **or** - -- `vagrant_cloud_url` (string) - Override the base URL for Vagrant Cloud. - This is useful if you're using Vagrant Private Cloud in your own network. - Defaults to `https://vagrantcloud.com/api/v1`. If this value is set to something - other than the default then `access_token` can be left blank and no - `Authorization` header will be added to requests sent by this post-processor. - -### Optional: - -- `no_release` (string) - If set to true, does not release the version on - Vagrant Cloud, making it active. You can manually release the version via - the API or Web UI. Defaults to `false`. - -- `insecure_skip_tls_verify` (boolean) - If set to true _and_ `vagrant_cloud_url` - is set to something different than its default, it will set TLS InsecureSkipVerify - to true. In other words, this will disable security checks of SSL. You may need - to set this option to true if your host at `vagrant_cloud_url` is using a - self-signed certificate. - -- `keep_input_artifact` (boolean) - When true, preserve the local box - after uploading to Vagrant cloud. Defaults to `true`. - -- `version_description` (string) - Optional Markdown text used as a - full-length and in-depth description of the version, typically for denoting - changes introduced - -- `box_download_url` (string) - Optional URL for a self-hosted box. If this - is set the box will not be uploaded to the Vagrant Cloud. - This is a [template engine](/docs/templates/legacy_json_templates/engine). Therefore, you - may use user variables and template functions in this field. - The following extra variables are also available in this engine: - - - `Provider`: The Vagrant provider the box is for - - `ArtifactId`: The ID of the input artifact. - -- `no_direct_upload` (boolean) - When `true`, upload the box artifact through - Vagrant Cloud instead of directly to the backend storage. - -## Use with the Vagrant Post-Processor - -An example configuration is shown below. Note the use of the nested array that -wraps both the Vagrant and Vagrant Cloud post-processors within the -post-processor section. Chaining the post-processors together in this way tells -Packer that the artifact produced by the Vagrant post-processor should be passed -directly to the Vagrant Cloud Post-Processor. It also sets the order in which -the post-processors should run. - -Failure to chain the post-processors together in this way will result in the -wrong artifact being supplied to the Vagrant Cloud post-processor. This will -likely cause the Vagrant Cloud post-processor to error and fail. - -```json -{ - "variables": { - "cloud_token": "{{ env `VAGRANT_CLOUD_TOKEN` }}", - "version": "1.0.{{timestamp}}" - }, - "post-processors": [ - { - "type": "shell-local", - "inline": ["echo Doing stuff..."] - }, - [ - { - "type": "vagrant", - "include": ["image.iso"], - "vagrantfile_template": "vagrantfile.tpl", - "output": "proxycore_{{.Provider}}.box" - }, - { - "type": "vagrant-cloud", - "box_tag": "hashicorp/precise64", - "access_token": "{{user `cloud_token`}}", - "version": "{{user `version`}}" - } - ] - ] -} -``` - -## Use with the Artifice Post-Processor - -An example configuration is shown below. Note the use of the nested array that -wraps both the Artifice and Vagrant Cloud post-processors within the -post-processor section. Chaining the post-processors together in this way tells -Packer that the artifact produced by the Artifice post-processor should be -passed directly to the Vagrant Cloud Post-Processor. It also sets the order in -which the post-processors should run. - -Failure to chain the post-processors together in this way will result in the -wrong artifact being supplied to the Vagrant Cloud post-processor. This will -likely cause the Vagrant Cloud post-processor to error and fail. - -Note that the Vagrant box specified in the Artifice post-processor `files` array -must end in the `.box` extension. It must also be the first file in the array. -Additional files bundled by the Artifice post-processor will be ignored. - -```json -{ - "variables": { - "cloud_token": "{{ env `VAGRANT_CLOUD_TOKEN` }}" - }, - - "builders": [ - { - "type": "null", - "communicator": "none" - } - ], - - "post-processors": [ - { - "type": "shell-local", - "inline": ["echo Doing stuff..."] - }, - [ - { - "type": "artifice", - "files": ["./path/to/my.box"] - }, - { - "type": "vagrant-cloud", - "box_tag": "myorganisation/mybox", - "access_token": "{{user `cloud_token`}}", - "version": "0.1.0" - } - ] - ] -} -``` diff --git a/website/content/docs/post-processors/vagrant.mdx b/website/content/docs/post-processors/vagrant.mdx deleted file mode 100644 index c1763771a..000000000 --- a/website/content/docs/post-processors/vagrant.mdx +++ /dev/null @@ -1,272 +0,0 @@ ---- -description: > - The Packer Vagrant post-processor takes a build and converts the artifact into - - a valid Vagrant box, if it can. This lets you use Packer to automatically - - create arbitrarily complex Vagrant boxes, and is in fact how the official - boxes - - distributed by Vagrant are created. -page_title: Vagrant - Post-Processors ---- - -# Vagrant Post-Processor - -Type: `vagrant` -Artifact BuilderId: `mitchellh.post-processor.vagrant` - -The Packer Vagrant post-processor takes a build and converts the artifact into -a valid [Vagrant](https://www.vagrantup.com) box, if it can. This lets you use -Packer to automatically create arbitrarily complex Vagrant boxes, and is in -fact how the official boxes distributed by Vagrant are created. - -If you've never used a post-processor before, please read the documentation on -[using post-processors](/docs/templates/legacy_json_templates/post-processors) in templates. -This knowledge will be expected for the remainder of this document. - -Because Vagrant boxes are -[provider-specific](https://www.vagrantup.com/docs/boxes/format), the -Vagrant post-processor is hardcoded to understand how to convert the artifacts -of certain builders into proper boxes for their respective providers. - -Currently, the Vagrant post-processor can create boxes for the following -providers. - -- AWS -- Azure -- DigitalOcean -- Docker -- Hyper-V -- LXC -- Parallels -- QEMU -- VirtualBox -- VMware - --> **Support for additional providers** is planned. If the Vagrant -post-processor doesn't support creating boxes for a provider you care about, -please help by contributing to Packer and adding support for it. - -Please note that if you are using the Vagrant builder, then the Vagrant -post-processor is unnecessary because the output of the Vagrant builder is -already a Vagrant box; using this post-processor with the Vagrant builder will -cause your build to fail. - -## Configuration - -The simplest way to use the post-processor is to just enable it. No -configuration is required by default. This will mostly do what you expect and -will build functioning boxes for many of the built-in builders of Packer. - -However, if you want to configure things a bit more, the post-processor does -expose some configuration options. The available options are listed below, with -more details about certain options in following sections. - -- `compression_level` (number) - An integer representing the compression - level to use when creating the Vagrant box. Valid values range from 0 to 9, - with 0 being no compression and 9 being the best compression. By default, - compression is enabled at level 6. - -- `include` (array of strings) - Paths to files to include in the Vagrant - box. These files will each be copied into the top level directory of the - Vagrant box (regardless of their paths). They can then be used from the - Vagrantfile. - -- `keep_input_artifact` (boolean) - When true, preserve the artifact we use to - create the vagrant box. Defaults to `false`, except when you set a cloud - provider (e.g. aws, azure, google, digitalocean). In these cases deleting - the input artifact would render the vagrant box useless, so we always keep - these artifacts -- even if you specifically set - `"keep_input_artifact":false` - -- `output` (string) - The full path to the box file that will be created by - this post-processor. This is a - [template engine](/docs/templates/legacy_json_templates/engine). Therefore, you may use user - variables and template functions in this field. The following extra - variables are also available in this engine: - - - `Provider`: The Vagrant provider the box is for - - `ArtifactId`: The ID of the input artifact. - - `BuildName`: The name of the build. - - By default, the value of this config is - `packer_{{.BuildName}}_{{.Provider}}.box`. - -- `provider_override` (string) - this option will override the internal logic - that decides which Vagrant provider to set for a particular Packer builder's - or post-processor's artifact. It is required when the artifact comes from the - Artifice post-processor, but is otherwise optional. Valid options are: - `digitalocean`, `virtualbox`, `azure`, `vmware`, `libvirt`, `docker`, - `lxc`, `scaleway`, `hyperv`, `parallels`, `aws`, or `google`. - -- `vagrantfile_template` (string) - Path to a template to use for the - Vagrantfile that is packaged with the box. This option supports the usage of the [template engine](/docs/templates/legacy_json_templates/engine) - for JSON and the [contextual variables](/docs/templates/hcl_templates/contextual-variables) for HCL2. - -- `vagrantfile_template_generated` (boolean) - By default, Packer will - exit with an error if the file specified using the - `vagrantfile_template` variable is not found. However, under certain - circumstances, it may be desirable to dynamically generate the - Vagrantfile during the course of the build. Setting this variable to - `true` skips the start up check and allows the user to script the - creation of the Vagrantfile at some previous point in the build. - Defaults to `false`. - -## Using together with the Artifice post-processor - -Sometimes you may want to run several builds in a pipeline rather than running -this post-processor inside a long-running Packer build. Here is an example of -how to do this: - - - - -```json -{ - "builders": [ - { - "type": "null", - "communicator": "none" - } - ], - "post-processors": [ - [ - { - "type": "artifice", - "files": [ - "output-virtualbox-iso/vbox-example-disk001.vmdk", - "output-virtualbox-iso/vbox-example.ovf" - ] - }, - { - "type": "vagrant", - "keep_input_artifact": true, - "provider_override": "virtualbox" - } - ] - ] -} -``` - - - - -```hcl -source "null" "example" { - communicator = "none" -} - -build { - sources = [ - "source.null.example" - ] - - post-processor "artifice" { - files = ["output-virtualbox-iso/vbox-example-disk001.vmdk", - "output-virtualbox-iso/vbox-example.ovf"] - } - - post-processor "vagrant" { - keep_input_artifact = true - provider_override = "virtualbox" - } -} -``` - - - - -## Provider-Specific Overrides - -If you have a Packer template with multiple builder types within it, you may -want to configure the box creation for each type a little differently. For -example, the contents of the Vagrantfile for a Vagrant box for AWS might be -different from the contents of the Vagrantfile you want for VMware. The -post-processor lets you do this. - -Specify overrides within the `override` configuration by provider name: - - - - -```json -{ - "type": "vagrant", - "compression_level": 1, - "override": { - "vmware": { - "compression_level": 0 - } - } -} -``` - - - - -```hcl -## This feature is not implemented in HCL. -``` - - - - -In the example above, the compression level will be set to 1 except for VMware, -where it will be set to 0. - -The available provider names are: - -- `aws` -- `azure` -- `digitalocean` -- `google` -- `hyperv` -- `parallels` -- `libvirt` -- `lxc` -- `scaleway` -- `virtualbox` -- `vmware` -- `docker` - -## Input Artifacts - -By default, Packer will delete the original input artifact, assuming you only -want the final Vagrant box as the result. If you wish to keep the input -artifact (the raw virtual machine, for example), then you must configure Packer -to keep it. - -Please see the [documentation on input -artifacts](/docs/templates/legacy_json_templates/post-processors#input-artifacts) for more information. - -### Docker - -Using a Docker input artifact will include a reference to the image in the -`Vagrantfile`. If the image tag is not specified in the post-processor, the -sha256 hash will be used. - -The following Docker input artifacts are supported: - -- `docker` builder with `commit: true`, always uses the sha256 hash -- `docker-import` -- `docker-tag` -- `docker-push` - -### QEMU/libvirt - -The `libvirt` provider supports QEMU artifacts built using any these -accelerators: none, kvm, tcg, or hvf. - -### VMWare - -If you are using the Vagrant post-processor with the `vmware-esxi` builder, you -must export the builder artifact locally; the Vagrant post-processor will -not work on remote artifacts. - -### Artifice - -If you are using this post-processor after defining an artifact using the -Artifice post-processor, then you must set the "provider_override" template -option so that the Vagrant post-processor knows what provider to use to create -the Vagrant box.