package terraform import ( "bufio" "context" "encoding/json" "fmt" "io" "os" "os/exec" "path/filepath" "runtime" "strings" "github.com/awalterschulze/gographviz" "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/mapstructure" "golang.org/x/xerrors" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/provisionersdk/proto" ) // Provision executes `terraform apply`. func (t *terraform) Provision(stream proto.DRPCProvisioner_ProvisionStream) error { shutdown, shutdownFunc := context.WithCancel(stream.Context()) defer shutdownFunc() request, err := stream.Recv() if err != nil { return err } if request.GetCancel() != nil { return nil } // We expect the first message is start! if request.GetStart() == nil { return nil } go func() { for { request, err := stream.Recv() if err != nil { return } if request.GetCancel() == nil { // This is only to process cancels! continue } shutdownFunc() return } }() start := request.GetStart() statefilePath := filepath.Join(start.Directory, "terraform.tfstate") if len(start.State) > 0 { err := os.WriteFile(statefilePath, start.State, 0600) if err != nil { return xerrors.Errorf("write statefile %q: %w", statefilePath, err) } } terraform, err := tfexec.NewTerraform(start.Directory, t.binaryPath) if err != nil { return xerrors.Errorf("create new terraform executor: %w", err) } version, _, err := terraform.Version(shutdown, false) if err != nil { return xerrors.Errorf("get terraform version: %w", err) } if !version.GreaterThanOrEqual(minimumTerraformVersion) { return xerrors.Errorf("terraform version %q is too old. required >= %q", version.String(), minimumTerraformVersion.String()) } reader, writer := io.Pipe() defer reader.Close() defer writer.Close() go func() { scanner := bufio.NewScanner(reader) for scanner.Scan() { _ = stream.Send(&proto.Provision_Response{ Type: &proto.Provision_Response_Log{ Log: &proto.Log{ Level: proto.LogLevel_DEBUG, Output: scanner.Text(), }, }, }) } }() terraformEnv := map[string]string{} // Required for "terraform init" to find "git" to // clone Terraform modules. for _, env := range os.Environ() { parts := strings.SplitN(env, "=", 2) if len(parts) < 2 { continue } terraformEnv[parts[0]] = parts[1] } // Only Linux reliably works with the Terraform plugin // cache directory. It's unknown why this is. if t.cachePath != "" && runtime.GOOS == "linux" { terraformEnv["TF_PLUGIN_CACHE_DIR"] = t.cachePath } err = terraform.SetEnv(terraformEnv) if err != nil { return xerrors.Errorf("set terraform env: %w", err) } terraform.SetStdout(writer) t.logger.Debug(shutdown, "running initialization") err = terraform.Init(shutdown) if err != nil { return xerrors.Errorf("initialize terraform: %w", err) } t.logger.Debug(shutdown, "ran initialization") _ = reader.Close() terraform.SetStdout(io.Discard) env := os.Environ() env = append(env, "CODER_URL="+start.Metadata.CoderUrl, "CODER_WORKSPACE_TRANSITION="+strings.ToLower(start.Metadata.WorkspaceTransition.String()), "CODER_WORKSPACE_NAME="+start.Metadata.WorkspaceName, "CODER_WORKSPACE_OWNER="+start.Metadata.WorkspaceOwner, ) for key, value := range provisionersdk.AgentScriptEnv() { env = append(env, key+"="+value) } vars := []string{} for _, param := range start.ParameterValues { switch param.DestinationScheme { case proto.ParameterDestination_ENVIRONMENT_VARIABLE: env = append(env, fmt.Sprintf("%s=%s", param.Name, param.Value)) case proto.ParameterDestination_PROVISIONER_VARIABLE: vars = append(vars, fmt.Sprintf("%s=%s", param.Name, param.Value)) default: return xerrors.Errorf("unsupported parameter type %q for %q", param.DestinationScheme, param.Name) } } closeChan := make(chan struct{}) reader, writer = io.Pipe() defer reader.Close() defer writer.Close() go func() { defer close(closeChan) decoder := json.NewDecoder(reader) for { var log terraformProvisionLog err := decoder.Decode(&log) if err != nil { return } logLevel, err := convertTerraformLogLevel(log.Level) if err != nil { // Not a big deal, but we should handle this at some point! continue } _ = stream.Send(&proto.Provision_Response{ Type: &proto.Provision_Response_Log{ Log: &proto.Log{ Level: logLevel, Output: log.Message, }, }, }) if log.Diagnostic == nil { continue } // If the diagnostic is provided, let's provide a bit more info! logLevel, err = convertTerraformLogLevel(log.Diagnostic.Severity) if err != nil { continue } _ = stream.Send(&proto.Provision_Response{ Type: &proto.Provision_Response_Log{ Log: &proto.Log{ Level: logLevel, Output: log.Diagnostic.Detail, }, }, }) } }() planfilePath := filepath.Join(start.Directory, "terraform.tfplan") var args []string if start.DryRun { args = []string{ "plan", "-no-color", "-input=false", "-json", "-refresh=true", "-out=" + planfilePath, } } else { args = []string{ "apply", "-no-color", "-auto-approve", "-input=false", "-json", "-refresh=true", } } if start.Metadata.WorkspaceTransition == proto.WorkspaceTransition_DESTROY { args = append(args, "-destroy") } for _, variable := range vars { args = append(args, "-var", variable) } // #nosec cmd := exec.CommandContext(stream.Context(), t.binaryPath, args...) go func() { select { case <-stream.Context().Done(): return case <-shutdown.Done(): _ = cmd.Process.Signal(os.Interrupt) } }() cmd.Stdout = writer cmd.Env = env cmd.Dir = terraform.WorkingDir() err = cmd.Run() if err != nil { if start.DryRun { if shutdown.Err() != nil { return stream.Send(&proto.Provision_Response{ Type: &proto.Provision_Response_Complete{ Complete: &proto.Provision_Complete{ Error: err.Error(), }, }, }) } return xerrors.Errorf("plan terraform: %w", err) } errorMessage := err.Error() // Terraform can fail and apply and still need to store it's state. // In this case, we return Complete with an explicit error message. statefileContent, err := os.ReadFile(statefilePath) if err != nil { return xerrors.Errorf("read file %q: %w", statefilePath, err) } return stream.Send(&proto.Provision_Response{ Type: &proto.Provision_Response_Complete{ Complete: &proto.Provision_Complete{ State: statefileContent, Error: errorMessage, }, }, }) } _ = reader.Close() <-closeChan var resp *proto.Provision_Response if start.DryRun { resp, err = parseTerraformPlan(stream.Context(), terraform, planfilePath) } else { resp, err = parseTerraformApply(stream.Context(), terraform, statefilePath) } if err != nil { return err } return stream.Send(resp) } func parseTerraformPlan(ctx context.Context, terraform *tfexec.Terraform, planfilePath string) (*proto.Provision_Response, error) { plan, err := terraform.ShowPlanFile(ctx, planfilePath) if err != nil { return nil, xerrors.Errorf("show terraform plan file: %w", err) } rawGraph, err := terraform.Graph(ctx) if err != nil { return nil, xerrors.Errorf("graph: %w", err) } resourceDependencies, err := findDirectDependencies(rawGraph) if err != nil { return nil, xerrors.Errorf("find dependencies: %w", err) } resources := make([]*proto.Resource, 0) agents := map[string]*proto.Agent{} // Store all agents inside the maps! for _, resource := range plan.Config.RootModule.Resources { if resource.Type != "coder_agent" { continue } agent := &proto.Agent{ Name: resource.Name, Auth: &proto.Agent_Token{}, } if operatingSystemRaw, has := resource.Expressions["os"]; has { operatingSystem, ok := operatingSystemRaw.ConstantValue.(string) if ok { agent.OperatingSystem = operatingSystem } } if archRaw, has := resource.Expressions["arch"]; has { arch, ok := archRaw.ConstantValue.(string) if ok { agent.Architecture = arch } } if envRaw, has := resource.Expressions["env"]; has { env, ok := envRaw.ConstantValue.(map[string]interface{}) if ok { agent.Env = map[string]string{} for key, valueRaw := range env { value, valid := valueRaw.(string) if !valid { continue } agent.Env[key] = value } } } if startupScriptRaw, has := resource.Expressions["startup_script"]; has { startupScript, ok := startupScriptRaw.ConstantValue.(string) if ok { agent.StartupScript = startupScript } } agents[resource.Address] = agent } for _, resource := range plan.PlannedValues.RootModule.Resources { if resource.Mode == tfjson.DataResourceMode { continue } if resource.Type == "coder_agent" || resource.Type == "coder_agent_instance" { continue } resourceKey := strings.Join([]string{resource.Type, resource.Name}, ".") resources = append(resources, &proto.Resource{ Name: resource.Name, Type: resource.Type, Agents: findAgents(resourceDependencies, agents, resourceKey), }) } return &proto.Provision_Response{ Type: &proto.Provision_Response_Complete{ Complete: &proto.Provision_Complete{ Resources: resources, }, }, }, nil } func parseTerraformApply(ctx context.Context, terraform *tfexec.Terraform, statefilePath string) (*proto.Provision_Response, error) { statefileContent, err := os.ReadFile(statefilePath) if err != nil { return nil, xerrors.Errorf("read file %q: %w", statefilePath, err) } state, err := terraform.ShowStateFile(ctx, statefilePath) if err != nil { return nil, xerrors.Errorf("show state file %q: %w", statefilePath, err) } resources := make([]*proto.Resource, 0) if state.Values != nil { rawGraph, err := terraform.Graph(ctx) if err != nil { return nil, xerrors.Errorf("graph: %w", err) } resourceDependencies, err := findDirectDependencies(rawGraph) if err != nil { return nil, xerrors.Errorf("find dependencies: %w", err) } type agentAttributes struct { Auth string `mapstructure:"auth"` OperatingSystem string `mapstructure:"os"` Architecture string `mapstructure:"arch"` ID string `mapstructure:"id"` Token string `mapstructure:"token"` Env map[string]string `mapstructure:"env"` StartupScript string `mapstructure:"startup_script"` } agents := map[string]*proto.Agent{} // Store all agents inside the maps! for _, resource := range state.Values.RootModule.Resources { if resource.Type != "coder_agent" { continue } var attrs agentAttributes err = mapstructure.Decode(resource.AttributeValues, &attrs) if err != nil { return nil, xerrors.Errorf("decode agent attributes: %w", err) } agent := &proto.Agent{ Name: resource.Name, Id: attrs.ID, Env: attrs.Env, StartupScript: attrs.StartupScript, OperatingSystem: attrs.OperatingSystem, Architecture: attrs.Architecture, } switch attrs.Auth { case "token": agent.Auth = &proto.Agent_Token{ Token: attrs.Token, } default: agent.Auth = &proto.Agent_InstanceId{} } resourceKey := strings.Join([]string{resource.Type, resource.Name}, ".") agents[resourceKey] = agent } // Manually associate agents with instance IDs. for _, resource := range state.Values.RootModule.Resources { if resource.Type != "coder_agent_instance" { continue } agentIDRaw, valid := resource.AttributeValues["agent_id"] if !valid { continue } agentID, valid := agentIDRaw.(string) if !valid { continue } instanceIDRaw, valid := resource.AttributeValues["instance_id"] if !valid { continue } instanceID, valid := instanceIDRaw.(string) if !valid { continue } for _, agent := range agents { if agent.Id != agentID { continue } agent.Auth = &proto.Agent_InstanceId{ InstanceId: instanceID, } break } } for _, resource := range state.Values.RootModule.Resources { if resource.Mode == tfjson.DataResourceMode { continue } if resource.Type == "coder_agent" || resource.Type == "coder_agent_instance" { continue } resourceKey := strings.Join([]string{resource.Type, resource.Name}, ".") resourceAgents := findAgents(resourceDependencies, agents, resourceKey) for _, agent := range resourceAgents { // Didn't use instance identity. if agent.GetToken() != "" { continue } key, isValid := map[string]string{ "google_compute_instance": "instance_id", "aws_instance": "id", "azurerm_linux_virtual_machine": "id", "azurerm_windows_virtual_machine": "id", }[resource.Type] if !isValid { // The resource type doesn't support // automatically setting the instance ID. continue } instanceIDRaw, valid := resource.AttributeValues[key] if !valid { continue } instanceID, valid := instanceIDRaw.(string) if !valid { continue } agent.Auth = &proto.Agent_InstanceId{ InstanceId: instanceID, } } resources = append(resources, &proto.Resource{ Name: resource.Name, Type: resource.Type, Agents: resourceAgents, }) } } return &proto.Provision_Response{ Type: &proto.Provision_Response_Complete{ Complete: &proto.Provision_Complete{ State: statefileContent, Resources: resources, }, }, }, nil } type terraformProvisionLog struct { Level string `json:"@level"` Message string `json:"@message"` Diagnostic *terraformProvisionLogDiagnostic `json:"diagnostic"` } type terraformProvisionLogDiagnostic struct { Severity string `json:"severity"` Summary string `json:"summary"` Detail string `json:"detail"` } func convertTerraformLogLevel(logLevel string) (proto.LogLevel, error) { switch strings.ToLower(logLevel) { case "trace": return proto.LogLevel_TRACE, nil case "debug": return proto.LogLevel_DEBUG, nil case "info": return proto.LogLevel_INFO, nil case "warn": return proto.LogLevel_WARN, nil case "error": return proto.LogLevel_ERROR, nil default: return proto.LogLevel(0), xerrors.Errorf("invalid log level %q", logLevel) } } // findDirectDependencies maps Terraform resources to their children nodes. // This parses GraphViz output from Terraform which isn't ideal, but seems reliable. func findDirectDependencies(rawGraph string) (map[string][]string, error) { parsedGraph, err := gographviz.ParseString(rawGraph) if err != nil { return nil, xerrors.Errorf("parse graph: %w", err) } graph, err := gographviz.NewAnalysedGraph(parsedGraph) if err != nil { return nil, xerrors.Errorf("analyze graph: %w", err) } direct := map[string][]string{} for _, node := range graph.Nodes.Nodes { label, exists := node.Attrs["label"] if !exists { continue } label = strings.Trim(label, `"`) direct[label] = findDependenciesWithLabels(graph, node.Name) } return direct, nil } // findDependenciesWithLabels recursively finds nodes with labels (resource and data nodes) // to build a dependency tree. func findDependenciesWithLabels(graph *gographviz.Graph, nodeName string) []string { dependencies := make([]string, 0) for destination := range graph.Edges.SrcToDsts[nodeName] { dependencyNode, exists := graph.Nodes.Lookup[destination] if !exists { continue } label, exists := dependencyNode.Attrs["label"] if !exists { dependencies = append(dependencies, findDependenciesWithLabels(graph, dependencyNode.Name)...) continue } label = strings.Trim(label, `"`) dependencies = append(dependencies, label) } return dependencies } // findAgents recursively searches through resource dependencies // to find associated agents. Nested is required for indirect // dependency matching. func findAgents(resourceDependencies map[string][]string, agents map[string]*proto.Agent, resourceKey string) []*proto.Agent { resourceNode, exists := resourceDependencies[resourceKey] if !exists { return []*proto.Agent{} } // Associate resources that depend on an agent. resourceAgents := make([]*proto.Agent, 0) for _, dep := range resourceNode { var has bool agent, has := agents[dep] if !has { resourceAgents = append(resourceAgents, findAgents(resourceDependencies, agents, dep)...) continue } resourceAgents = append(resourceAgents, agent) } return resourceAgents }