Files
coder/provisioner/terraform/provision.go
Kyle Carberry 013f028e55 feat: Add workspace application support (#1773)
* feat: Add app support

This adds apps as a property to a workspace agent.

The resource is added to the Terraform provider here:
https://github.com/coder/terraform-provider-coder/pull/17

Apps will be opened in the dashboard or via the CLI
with `coder open <name>`. If `command` is specified, a
terminal will appear locally and in the web. If `target`
is specified, the browser will open to an exposed instance
of that target.

* Compare fields in apps test

* Update Terraform provider to use relative path

* Add some basic structure for routing

* chore: Remove interface from coderd and lift API surface

Abstracting coderd into an interface added misdirection because
the interface was never intended to be fulfilled outside of a single
implementation.

This lifts the abstraction, and attaches all handlers to a root struct
named `*coderd.API`.

* Add basic proxy logic

* Add proxying based on path

* Add app proxying for wildcards

* Add wsconncache

* fix: Race when writing to a closed pipe

This is such an intermittent race it's difficult to track,
but regardless this is an improvement to the code.

* fix: Race when writing to a closed pipe

This is such an intermittent race it's difficult to track,
but regardless this is an improvement to the code.

* fix: Race when writing to a closed pipe

This is such an intermittent race it's difficult to track,
but regardless this is an improvement to the code.

* fix: Race when writing to a closed pipe

This is such an intermittent race it's difficult to track,
but regardless this is an improvement to the code.

* Add workspace route proxying endpoint

- Makes the workspace conn cache concurrency-safe
- Reduces unnecessary open checks in `peer.Channel`
- Fixes the use of a temporary context when dialing a workspace agent

* Add embed errors

* chore: Refactor site to improve testing

It was difficult to develop this package due to the
embed build tag being mandatory on the tests. The logic
to test doesn't require any embedded files.

* Add test for error handler

* Remove unused access url

* Add RBAC tests

* Fix dial agent syntax

* Fix linting errors

* Fix gen

* Fix icon required

* Adjust migration number

* Fix proxy error status code

* Fix empty db lookup
2022-06-04 15:13:37 -05:00

756 lines
21 KiB
Go

package terraform
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"github.com/awalterschulze/gographviz"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
"github.com/mitchellh/mapstructure"
"golang.org/x/xerrors"
"github.com/coder/coder/provisionersdk"
"github.com/coder/coder/provisionersdk/proto"
)
var (
// noStateRegex is matched against the output from `terraform state show`
noStateRegex = regexp.MustCompile(`(?i)State read error.*no state`)
)
// Provision executes `terraform apply`.
func (t *terraform) Provision(stream proto.DRPCProvisioner_ProvisionStream) error {
shutdown, shutdownFunc := context.WithCancel(stream.Context())
defer shutdownFunc()
request, err := stream.Recv()
if err != nil {
return err
}
if request.GetCancel() != nil {
return nil
}
// We expect the first message is start!
if request.GetStart() == nil {
return nil
}
go func() {
for {
request, err := stream.Recv()
if err != nil {
return
}
if request.GetCancel() == nil {
// This is only to process cancels!
continue
}
shutdownFunc()
return
}
}()
start := request.GetStart()
terraform, err := tfexec.NewTerraform(start.Directory, t.binaryPath)
if err != nil {
return xerrors.Errorf("create new terraform executor: %w", err)
}
version, _, err := terraform.Version(shutdown, false)
if err != nil {
return xerrors.Errorf("get terraform version: %w", err)
}
if !version.GreaterThanOrEqual(minimumTerraformVersion) {
return xerrors.Errorf("terraform version %q is too old. required >= %q", version.String(), minimumTerraformVersion.String())
}
statefilePath := filepath.Join(start.Directory, "terraform.tfstate")
if len(start.State) > 0 {
err := os.WriteFile(statefilePath, start.State, 0600)
if err != nil {
return xerrors.Errorf("write statefile %q: %w", statefilePath, err)
}
}
reader, writer := io.Pipe()
defer reader.Close()
defer writer.Close()
go func() {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
_ = stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Log{
Log: &proto.Log{
Level: proto.LogLevel_DEBUG,
Output: scanner.Text(),
},
},
})
}
}()
terraformEnv := map[string]string{}
// Required for "terraform init" to find "git" to
// clone Terraform modules.
for _, env := range os.Environ() {
parts := strings.SplitN(env, "=", 2)
if len(parts) < 2 {
continue
}
terraformEnv[parts[0]] = parts[1]
}
// Only Linux reliably works with the Terraform plugin
// cache directory. It's unknown why this is.
if t.cachePath != "" && runtime.GOOS == "linux" {
terraformEnv["TF_PLUGIN_CACHE_DIR"] = t.cachePath
}
err = terraform.SetEnv(terraformEnv)
if err != nil {
return xerrors.Errorf("set terraform env: %w", err)
}
terraform.SetStdout(writer)
t.logger.Debug(shutdown, "running initialization")
err = terraform.Init(shutdown)
if err != nil {
return xerrors.Errorf("initialize terraform: %w", err)
}
t.logger.Debug(shutdown, "ran initialization")
_ = reader.Close()
terraform.SetStdout(io.Discard)
env := os.Environ()
env = append(env,
"CODER_AGENT_URL="+start.Metadata.CoderUrl,
"CODER_WORKSPACE_TRANSITION="+strings.ToLower(start.Metadata.WorkspaceTransition.String()),
"CODER_WORKSPACE_NAME="+start.Metadata.WorkspaceName,
"CODER_WORKSPACE_OWNER="+start.Metadata.WorkspaceOwner,
"CODER_WORKSPACE_ID="+start.Metadata.WorkspaceId,
"CODER_WORKSPACE_OWNER_ID="+start.Metadata.WorkspaceOwnerId,
)
for key, value := range provisionersdk.AgentScriptEnv() {
env = append(env, key+"="+value)
}
vars := []string{}
for _, param := range start.ParameterValues {
switch param.DestinationScheme {
case proto.ParameterDestination_ENVIRONMENT_VARIABLE:
env = append(env, fmt.Sprintf("%s=%s", param.Name, param.Value))
case proto.ParameterDestination_PROVISIONER_VARIABLE:
vars = append(vars, fmt.Sprintf("%s=%s", param.Name, param.Value))
default:
return xerrors.Errorf("unsupported parameter type %q for %q", param.DestinationScheme, param.Name)
}
}
closeChan := make(chan struct{})
reader, writer = io.Pipe()
defer reader.Close()
defer writer.Close()
go func() {
defer close(closeChan)
decoder := json.NewDecoder(reader)
for {
var log terraformProvisionLog
err := decoder.Decode(&log)
if err != nil {
return
}
logLevel, err := convertTerraformLogLevel(log.Level)
if err != nil {
// Not a big deal, but we should handle this at some point!
continue
}
_ = stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Log{
Log: &proto.Log{
Level: logLevel,
Output: log.Message,
},
},
})
if log.Diagnostic == nil {
continue
}
// If the diagnostic is provided, let's provide a bit more info!
logLevel, err = convertTerraformLogLevel(log.Diagnostic.Severity)
if err != nil {
continue
}
_ = stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Log{
Log: &proto.Log{
Level: logLevel,
Output: log.Diagnostic.Detail,
},
},
})
}
}()
// If we're destroying, exit early if there's no state. This is necessary to
// avoid any cases where a workspace is "locked out" of terraform due to
// e.g. bad template param values and cannot be deleted. This is just for
// contingency, in the future we will try harder to prevent workspaces being
// broken this hard.
if start.Metadata.WorkspaceTransition == proto.WorkspaceTransition_DESTROY {
_, err := getTerraformState(shutdown, terraform, statefilePath)
if xerrors.Is(err, os.ErrNotExist) {
_ = stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Log{
Log: &proto.Log{
Level: proto.LogLevel_INFO,
Output: "The terraform state does not exist, there is nothing to do",
},
},
})
return stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{},
},
})
}
if err != nil {
err = xerrors.Errorf("get terraform state: %w", err)
_ = stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{
Error: err.Error(),
},
},
})
return err
}
}
planfilePath := filepath.Join(start.Directory, "terraform.tfplan")
var args []string
if start.DryRun {
args = []string{
"plan",
"-no-color",
"-input=false",
"-json",
"-refresh=true",
"-out=" + planfilePath,
}
} else {
args = []string{
"apply",
"-no-color",
"-auto-approve",
"-input=false",
"-json",
"-refresh=true",
}
}
if start.Metadata.WorkspaceTransition == proto.WorkspaceTransition_DESTROY {
args = append(args, "-destroy")
}
for _, variable := range vars {
args = append(args, "-var", variable)
}
// #nosec
cmd := exec.CommandContext(stream.Context(), t.binaryPath, args...)
go func() {
select {
case <-stream.Context().Done():
return
case <-shutdown.Done():
_ = cmd.Process.Signal(os.Interrupt)
}
}()
cmd.Stdout = writer
cmd.Env = env
cmd.Dir = terraform.WorkingDir()
err = cmd.Run()
if err != nil {
if start.DryRun {
if shutdown.Err() != nil {
return stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{
Error: err.Error(),
},
},
})
}
return xerrors.Errorf("plan terraform: %w", err)
}
errorMessage := err.Error()
// Terraform can fail and apply and still need to store it's state.
// In this case, we return Complete with an explicit error message.
stateData, _ := os.ReadFile(statefilePath)
return stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{
State: stateData,
Error: errorMessage,
},
},
})
}
_ = reader.Close()
<-closeChan
var resp *proto.Provision_Response
if start.DryRun {
resp, err = parseTerraformPlan(stream.Context(), terraform, planfilePath)
} else {
resp, err = parseTerraformApply(stream.Context(), terraform, statefilePath)
}
if err != nil {
return err
}
return stream.Send(resp)
}
func parseTerraformPlan(ctx context.Context, terraform *tfexec.Terraform, planfilePath string) (*proto.Provision_Response, error) {
plan, err := terraform.ShowPlanFile(ctx, planfilePath)
if err != nil {
return nil, xerrors.Errorf("show terraform plan file: %w", err)
}
rawGraph, err := terraform.Graph(ctx)
if err != nil {
return nil, xerrors.Errorf("graph: %w", err)
}
resourceDependencies, err := findDirectDependencies(rawGraph)
if err != nil {
return nil, xerrors.Errorf("find dependencies: %w", err)
}
resources := make([]*proto.Resource, 0)
agents := map[string]*proto.Agent{}
tfResources := make([]*tfjson.ConfigResource, 0)
var appendResources func(mod *tfjson.ConfigModule)
appendResources = func(mod *tfjson.ConfigModule) {
for _, module := range mod.ModuleCalls {
appendResources(module.Module)
}
tfResources = append(tfResources, mod.Resources...)
}
appendResources(plan.Config.RootModule)
// Store all agents inside the maps!
for _, resource := range tfResources {
if resource.Type != "coder_agent" {
continue
}
agent := &proto.Agent{
Name: resource.Name,
Auth: &proto.Agent_Token{},
}
if operatingSystemRaw, has := resource.Expressions["os"]; has {
operatingSystem, ok := operatingSystemRaw.ConstantValue.(string)
if ok {
agent.OperatingSystem = operatingSystem
}
}
if archRaw, has := resource.Expressions["arch"]; has {
arch, ok := archRaw.ConstantValue.(string)
if ok {
agent.Architecture = arch
}
}
if envRaw, has := resource.Expressions["env"]; has {
env, ok := envRaw.ConstantValue.(map[string]interface{})
if ok {
agent.Env = map[string]string{}
for key, valueRaw := range env {
value, valid := valueRaw.(string)
if !valid {
continue
}
agent.Env[key] = value
}
}
}
if startupScriptRaw, has := resource.Expressions["startup_script"]; has {
startupScript, ok := startupScriptRaw.ConstantValue.(string)
if ok {
agent.StartupScript = startupScript
}
}
if directoryRaw, has := resource.Expressions["dir"]; has {
dir, ok := directoryRaw.ConstantValue.(string)
if ok {
agent.Directory = dir
}
}
agents[convertAddressToLabel(resource.Address)] = agent
}
for _, resource := range tfResources {
if resource.Mode == tfjson.DataResourceMode {
continue
}
if resource.Type == "coder_agent" || resource.Type == "coder_agent_instance" || resource.Type == "coder_app" {
continue
}
resources = append(resources, &proto.Resource{
Name: resource.Name,
Type: resource.Type,
Agents: findAgents(resourceDependencies, agents, convertAddressToLabel(resource.Address)),
})
}
return &proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{
Resources: resources,
},
},
}, nil
}
func parseTerraformApply(ctx context.Context, terraform *tfexec.Terraform, statefilePath string) (*proto.Provision_Response, error) {
_, err := os.Stat(statefilePath)
statefileExisted := err == nil
state, err := getTerraformState(ctx, terraform, statefilePath)
if err != nil {
return nil, xerrors.Errorf("get terraform state: %w", err)
}
resources := make([]*proto.Resource, 0)
if state.Values != nil {
rawGraph, err := terraform.Graph(ctx)
if err != nil {
return nil, xerrors.Errorf("graph: %w", err)
}
resourceDependencies, err := findDirectDependencies(rawGraph)
if err != nil {
return nil, xerrors.Errorf("find dependencies: %w", err)
}
type agentAttributes struct {
Auth string `mapstructure:"auth"`
OperatingSystem string `mapstructure:"os"`
Architecture string `mapstructure:"arch"`
Directory string `mapstructure:"dir"`
ID string `mapstructure:"id"`
Token string `mapstructure:"token"`
Env map[string]string `mapstructure:"env"`
StartupScript string `mapstructure:"startup_script"`
}
agents := map[string]*proto.Agent{}
tfResources := make([]*tfjson.StateResource, 0)
var appendResources func(resource *tfjson.StateModule)
appendResources = func(mod *tfjson.StateModule) {
for _, module := range mod.ChildModules {
appendResources(module)
}
tfResources = append(tfResources, mod.Resources...)
}
appendResources(state.Values.RootModule)
// Store all agents inside the maps!
for _, resource := range tfResources {
if resource.Type != "coder_agent" {
continue
}
var attrs agentAttributes
err = mapstructure.Decode(resource.AttributeValues, &attrs)
if err != nil {
return nil, xerrors.Errorf("decode agent attributes: %w", err)
}
agent := &proto.Agent{
Name: resource.Name,
Id: attrs.ID,
Env: attrs.Env,
StartupScript: attrs.StartupScript,
OperatingSystem: attrs.OperatingSystem,
Architecture: attrs.Architecture,
Directory: attrs.Directory,
}
switch attrs.Auth {
case "token":
agent.Auth = &proto.Agent_Token{
Token: attrs.Token,
}
default:
agent.Auth = &proto.Agent_InstanceId{}
}
agents[convertAddressToLabel(resource.Address)] = agent
}
// Manually associate agents with instance IDs.
for _, resource := range tfResources {
if resource.Type != "coder_agent_instance" {
continue
}
agentIDRaw, valid := resource.AttributeValues["agent_id"]
if !valid {
continue
}
agentID, valid := agentIDRaw.(string)
if !valid {
continue
}
instanceIDRaw, valid := resource.AttributeValues["instance_id"]
if !valid {
continue
}
instanceID, valid := instanceIDRaw.(string)
if !valid {
continue
}
for _, agent := range agents {
if agent.Id != agentID {
continue
}
agent.Auth = &proto.Agent_InstanceId{
InstanceId: instanceID,
}
break
}
}
type appAttributes struct {
AgentID string `mapstructure:"agent_id"`
Name string `mapstructure:"name"`
Icon string `mapstructure:"icon"`
URL string `mapstructure:"url"`
Command string `mapstructure:"command"`
RelativePath bool `mapstructure:"relative_path"`
}
// Associate Apps with agents.
for _, resource := range state.Values.RootModule.Resources {
if resource.Type != "coder_app" {
continue
}
var attrs appAttributes
err = mapstructure.Decode(resource.AttributeValues, &attrs)
if err != nil {
return nil, xerrors.Errorf("decode app attributes: %w", err)
}
if attrs.Name == "" {
// Default to the resource name if none is set!
attrs.Name = resource.Name
}
for _, agent := range agents {
if agent.Id != attrs.AgentID {
continue
}
agent.Apps = append(agent.Apps, &proto.App{
Name: attrs.Name,
Command: attrs.Command,
Url: attrs.URL,
Icon: attrs.Icon,
RelativePath: attrs.RelativePath,
})
}
}
for _, resource := range tfResources {
if resource.Mode == tfjson.DataResourceMode {
continue
}
if resource.Type == "coder_agent" || resource.Type == "coder_agent_instance" || resource.Type == "coder_app" {
continue
}
resourceAgents := findAgents(resourceDependencies, agents, convertAddressToLabel(resource.Address))
for _, agent := range resourceAgents {
// Didn't use instance identity.
if agent.GetToken() != "" {
continue
}
key, isValid := map[string]string{
"google_compute_instance": "instance_id",
"aws_instance": "id",
"azurerm_linux_virtual_machine": "id",
"azurerm_windows_virtual_machine": "id",
}[resource.Type]
if !isValid {
// The resource type doesn't support
// automatically setting the instance ID.
continue
}
instanceIDRaw, valid := resource.AttributeValues[key]
if !valid {
continue
}
instanceID, valid := instanceIDRaw.(string)
if !valid {
continue
}
agent.Auth = &proto.Agent_InstanceId{
InstanceId: instanceID,
}
}
resources = append(resources, &proto.Resource{
Name: resource.Name,
Type: resource.Type,
Agents: resourceAgents,
})
}
}
var stateContent []byte
// We only want to restore state if it's not hosted remotely.
if statefileExisted {
stateContent, err = os.ReadFile(statefilePath)
if err != nil {
return nil, xerrors.Errorf("read statefile %q: %w", statefilePath, err)
}
}
return &proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{
State: stateContent,
Resources: resources,
},
},
}, nil
}
// getTerraformState pulls and merges any remote terraform state into the given
// path and reads the merged state. If there is no state, `os.ErrNotExist` will
// be returned.
func getTerraformState(ctx context.Context, terraform *tfexec.Terraform, statefilePath string) (*tfjson.State, error) {
statefile, err := os.OpenFile(statefilePath, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, xerrors.Errorf("open statefile %q: %w", statefilePath, err)
}
defer statefile.Close()
// #nosec
cmd := exec.CommandContext(ctx, terraform.ExecPath(), "state", "pull")
cmd.Dir = terraform.WorkingDir()
cmd.Stdout = statefile
err = cmd.Run()
if err != nil {
return nil, xerrors.Errorf("pull terraform state: %w", err)
}
state, err := terraform.ShowStateFile(ctx, statefilePath)
if err != nil {
if noStateRegex.MatchString(err.Error()) {
return nil, os.ErrNotExist
}
return nil, xerrors.Errorf("show terraform state: %w", err)
}
return state, nil
}
type terraformProvisionLog struct {
Level string `json:"@level"`
Message string `json:"@message"`
Diagnostic *terraformProvisionLogDiagnostic `json:"diagnostic"`
}
type terraformProvisionLogDiagnostic struct {
Severity string `json:"severity"`
Summary string `json:"summary"`
Detail string `json:"detail"`
}
func convertTerraformLogLevel(logLevel string) (proto.LogLevel, error) {
switch strings.ToLower(logLevel) {
case "trace":
return proto.LogLevel_TRACE, nil
case "debug":
return proto.LogLevel_DEBUG, nil
case "info":
return proto.LogLevel_INFO, nil
case "warn":
return proto.LogLevel_WARN, nil
case "error":
return proto.LogLevel_ERROR, nil
default:
return proto.LogLevel(0), xerrors.Errorf("invalid log level %q", logLevel)
}
}
// findDirectDependencies maps Terraform resources to their children nodes.
// This parses GraphViz output from Terraform which isn't ideal, but seems reliable.
func findDirectDependencies(rawGraph string) (map[string][]string, error) {
parsedGraph, err := gographviz.ParseString(rawGraph)
if err != nil {
return nil, xerrors.Errorf("parse graph: %w", err)
}
graph, err := gographviz.NewAnalysedGraph(parsedGraph)
if err != nil {
return nil, xerrors.Errorf("analyze graph: %w", err)
}
direct := map[string][]string{}
for _, node := range graph.Nodes.Nodes {
label, exists := node.Attrs["label"]
if !exists {
continue
}
label = strings.Trim(label, `"`)
direct[label] = findDependenciesWithLabels(graph, node.Name)
}
return direct, nil
}
// findDependenciesWithLabels recursively finds nodes with labels (resource and data nodes)
// to build a dependency tree.
func findDependenciesWithLabels(graph *gographviz.Graph, nodeName string) []string {
dependencies := make([]string, 0)
for destination := range graph.Edges.SrcToDsts[nodeName] {
dependencyNode, exists := graph.Nodes.Lookup[destination]
if !exists {
continue
}
label, exists := dependencyNode.Attrs["label"]
if !exists {
dependencies = append(dependencies, findDependenciesWithLabels(graph, dependencyNode.Name)...)
continue
}
label = strings.Trim(label, `"`)
dependencies = append(dependencies, label)
}
return dependencies
}
// findAgents recursively searches through resource dependencies
// to find associated agents. Nested is required for indirect
// dependency matching.
func findAgents(resourceDependencies map[string][]string, agents map[string]*proto.Agent, resourceLabel string) []*proto.Agent {
resourceNode, exists := resourceDependencies[resourceLabel]
if !exists {
return []*proto.Agent{}
}
// Associate resources that depend on an agent.
resourceAgents := make([]*proto.Agent, 0)
for _, dep := range resourceNode {
var has bool
agent, has := agents[dep]
if !has {
resourceAgents = append(resourceAgents, findAgents(resourceDependencies, agents, dep)...)
continue
}
resourceAgents = append(resourceAgents, agent)
}
return resourceAgents
}
// convertAddressToLabel returns the Terraform address without the count
// specifier. eg. "module.ec2_dev.ec2_instance.dev[0]" becomes "module.ec2_dev.ec2_instance.dev"
func convertAddressToLabel(address string) string {
return strings.Split(address, "[")[0]
}