synced/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2025-07-15 22:20:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Allow terraform provisions to be gracefully cancelled (#3526)

Browse Source 
* fix: Allow terraform provisions to be gracefully cancelled

This change allows terraform commands to be gracefully cancelled on
Unix-like platforms by signaling interrupt on provision cancellation.

One implementation detail to note is that we do not necessarily kill a
running terraform command immediately even if the stream is closed. The
reason for this is to allow for graceful cancellation even in such an
event. Currently the timeout is set to 5 minutes by default.

Related: #2683

The above issue may be partially or fully fixed by this change.

* fix: Remove incorrect minimumTerraformVersion variable

* Allow init to return provision complete response
This commit is contained in:
Mathias Fredriksson
2022-08-18 17:03:55 +03:00
committed by GitHub
parent 6a0f8ae9cc
commit f1423450bd
6 changed files with 357 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
provisioner/terraform/executor.go
View File

@ -41,7 +41,7 @@ func (e executor) basicEnv() []string {
return env
}
func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOutWriter, stdErrWriter io.WriteCloser) (err error) {
func (e executor) execWriteOutput(ctx, killCtx context.Context, args, env []string, stdOutWriter, stdErrWriter io.WriteCloser) (err error) {
defer func() {
closeErr := stdOutWriter.Close()
if err == nil && closeErr != nil {
@ -52,8 +52,12 @@ func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOu
err = closeErr
}
}()
if ctx.Err() != nil {
return ctx.Err()
}
// #nosec
cmd := exec.CommandContext(ctx, e.binaryPath, args...)
cmd := exec.CommandContext(killCtx, e.binaryPath, args...)
cmd.Dir = e.workdir
cmd.Env = env
@ -63,19 +67,36 @@ func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOu
cmd.Stdout = syncWriter{mut, stdOutWriter}
cmd.Stderr = syncWriter{mut, stdErrWriter}
return cmd.Run()
err = cmd.Start()
if err != nil {
return err
}
interruptCommandOnCancel(ctx, killCtx, cmd)
return cmd.Wait()
}
func (e executor) execParseJSON(ctx context.Context, args, env []string, v interface{}) error {
func (e executor) execParseJSON(ctx, killCtx context.Context, args, env []string, v interface{}) error {
if ctx.Err() != nil {
return ctx.Err()
}
// #nosec
cmd := exec.CommandContext(ctx, e.binaryPath, args...)
cmd := exec.CommandContext(killCtx, e.binaryPath, args...)
cmd.Dir = e.workdir
cmd.Env = env
out := &bytes.Buffer{}
stdErr := &bytes.Buffer{}
cmd.Stdout = out
cmd.Stderr = stdErr
err := cmd.Run()
err := cmd.Start()
if err != nil {
return err
}
interruptCommandOnCancel(ctx, killCtx, cmd)
err = cmd.Wait()
if err != nil {
errString, _ := io.ReadAll(stdErr)
return xerrors.Errorf("%s: %w", errString, err)
@ -95,11 +116,11 @@ func (e executor) checkMinVersion(ctx context.Context) error {
if err != nil {
return err
}
if !v.GreaterThanOrEqual(minimumTerraformVersion) {
if !v.GreaterThanOrEqual(minTerraformVersion) {
return xerrors.Errorf(
"terraform version %q is too old. required >= %q",
v.String(),
minimumTerraformVersion.String())
minTerraformVersion.String())
}
return nil
}
@ -109,6 +130,10 @@ func (e executor) version(ctx context.Context) (*version.Version, error) {
}
func versionFromBinaryPath(ctx context.Context, binaryPath string) (*version.Version, error) {
if ctx.Err() != nil {
return nil, ctx.Err()
}
// #nosec
cmd := exec.CommandContext(ctx, binaryPath, "version", "-json")
out, err := cmd.Output()
@ -130,7 +155,7 @@ func versionFromBinaryPath(ctx context.Context, binaryPath string) (*version.Ver
return version.NewVersion(vj.Version)
}
func (e executor) init(ctx context.Context, logr logger) error {
func (e executor) init(ctx, killCtx context.Context, logr logger) error {
outWriter, doneOut := logWriter(logr, proto.LogLevel_DEBUG)
errWriter, doneErr := logWriter(logr, proto.LogLevel_ERROR)
defer func() {
@ -156,11 +181,11 @@ func (e executor) init(ctx context.Context, logr logger) error {
defer e.initMu.Unlock()
}
return e.execWriteOutput(ctx, args, e.basicEnv(), outWriter, errWriter)
return e.execWriteOutput(ctx, killCtx, args, e.basicEnv(), outWriter, errWriter)
}
// revive:disable-next-line:flag-parameter
func (e executor) plan(ctx context.Context, env, vars []string, logr logger, destroy bool) (*proto.Provision_Response, error) {
func (e executor) plan(ctx, killCtx context.Context, env, vars []string, logr logger, destroy bool) (*proto.Provision_Response, error) {
planfilePath := filepath.Join(e.workdir, "terraform.tfplan")
args := []string{
"plan",
@ -184,11 +209,11 @@ func (e executor) plan(ctx context.Context, env, vars []string, logr logger, des
<-doneErr
}()
err := e.execWriteOutput(ctx, args, env, outWriter, errWriter)
err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter)
if err != nil {
return nil, xerrors.Errorf("terraform plan: %w", err)
}
resources, err := e.planResources(ctx, planfilePath)
resources, err := e.planResources(ctx, killCtx, planfilePath)
if err != nil {
return nil, err
}
@ -201,40 +226,52 @@ func (e executor) plan(ctx context.Context, env, vars []string, logr logger, des
}, nil
}
func (e executor) planResources(ctx context.Context, planfilePath string) ([]*proto.Resource, error) {
plan, err := e.showPlan(ctx, planfilePath)
func (e executor) planResources(ctx, killCtx context.Context, planfilePath string) ([]*proto.Resource, error) {
plan, err := e.showPlan(ctx, killCtx, planfilePath)
if err != nil {
return nil, xerrors.Errorf("show terraform plan file: %w", err)
}
rawGraph, err := e.graph(ctx)
rawGraph, err := e.graph(ctx, killCtx)
if err != nil {
return nil, xerrors.Errorf("graph: %w", err)
}
return ConvertResources(plan.PlannedValues.RootModule, rawGraph)
}
func (e executor) showPlan(ctx context.Context, planfilePath string) (*tfjson.Plan, error) {
func (e executor) showPlan(ctx, killCtx context.Context, planfilePath string) (*tfjson.Plan, error) {
args := []string{"show", "-json", "-no-color", planfilePath}
p := new(tfjson.Plan)
err := e.execParseJSON(ctx, args, e.basicEnv(), p)
err := e.execParseJSON(ctx, killCtx, args, e.basicEnv(), p)
return p, err
}
func (e executor) graph(ctx context.Context) (string, error) {
// #nosec
cmd := exec.CommandContext(ctx, e.binaryPath, "graph")
func (e executor) graph(ctx, killCtx context.Context) (string, error) {
if ctx.Err() != nil {
return "", ctx.Err()
}
var out bytes.Buffer
cmd := exec.CommandContext(killCtx, e.binaryPath, "graph") // #nosec
cmd.Stdout = &out
cmd.Dir = e.workdir
cmd.Env = e.basicEnv()
out, err := cmd.Output()
err := cmd.Start()
if err != nil {
return "", err
}
interruptCommandOnCancel(ctx, killCtx, cmd)
err = cmd.Wait()
if err != nil {
return "", xerrors.Errorf("graph: %w", err)
}
return string(out), nil
return out.String(), nil
}
// revive:disable-next-line:flag-parameter
func (e executor) apply(ctx context.Context, env, vars []string, logr logger, destroy bool,
func (e executor) apply(ctx, killCtx context.Context, env, vars []string, logr logger, destroy bool,
) (*proto.Provision_Response, error) {
args := []string{
"apply",
@ -258,11 +295,11 @@ func (e executor) apply(ctx context.Context, env, vars []string, logr logger, de
<-doneErr
}()
err := e.execWriteOutput(ctx, args, env, outWriter, errWriter)
err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter)
if err != nil {
return nil, xerrors.Errorf("terraform apply: %w", err)
}
resources, err := e.stateResources(ctx)
resources, err := e.stateResources(ctx, killCtx)
if err != nil {
return nil, err
}
@ -281,12 +318,12 @@ func (e executor) apply(ctx context.Context, env, vars []string, logr logger, de
}, nil
}
func (e executor) stateResources(ctx context.Context) ([]*proto.Resource, error) {
state, err := e.state(ctx)
func (e executor) stateResources(ctx, killCtx context.Context) ([]*proto.Resource, error) {
state, err := e.state(ctx, killCtx)
if err != nil {
return nil, err
}
rawGraph, err := e.graph(ctx)
rawGraph, err := e.graph(ctx, killCtx)
if err != nil {
return nil, xerrors.Errorf("get terraform graph: %w", err)
}
@ -300,16 +337,33 @@ func (e executor) stateResources(ctx context.Context) ([]*proto.Resource, error)
return resources, nil
}
func (e executor) state(ctx context.Context) (*tfjson.State, error) {
func (e executor) state(ctx, killCtx context.Context) (*tfjson.State, error) {
args := []string{"show", "-json"}
state := &tfjson.State{}
err := e.execParseJSON(ctx, args, e.basicEnv(), state)
err := e.execParseJSON(ctx, killCtx, args, e.basicEnv(), state)
if err != nil {
return nil, xerrors.Errorf("terraform show state: %w", err)
}
return state, nil
}
func interruptCommandOnCancel(ctx, killCtx context.Context, cmd *exec.Cmd) {
go func() {
select {
case <-ctx.Done():
switch runtime.GOOS {
case "windows":
// Interrupts aren't supported by Windows.
_ = cmd.Process.Kill()
default:
_ = cmd.Process.Signal(os.Interrupt)
}
case <-killCtx.Done():
}
}()
}
type logger interface {
Log(*proto.Log) error
}
@ -381,9 +435,6 @@ func provisionReadAndLog(logr logger, reader io.Reader, done chan<- any) {
// If the diagnostic is provided, let's provide a bit more info!
logLevel = convertTerraformLogLevel(log.Diagnostic.Severity, logr)
if err != nil {
continue
}
err = logr.Log(&proto.Log{Level: logLevel, Output: log.Diagnostic.Detail})
if err != nil {
// Not much we can do. We can't log because logging is itself breaking!