feat: Add graceful exits to provisionerd (#372)

* ci: Update DataDog GitHub branch to fallback to GITHUB_REF

This was detecting branches, but not our "main" branch before.
Hopefully this fixes it!

* Add basic Terraform Provider

* Rename post files to upload

* Add tests for resources

* Skip instance identity test

* Add tests for ensuring agent get's passed through properly

* Fix linting errors

* Add echo path

* Fix agent authentication

* fix: Convert all jobs to use a common resource and agent type

This enables a consistent API for project import and provisioned resources.

* Add "coder_workspace" data source

* feat: Remove magical parameters from being injected

This is a much cleaner abstraction. Explicitly declaring the user
parameters for each provisioner makes for significantly simpler
testing.

* feat: Add graceful exits to provisionerd

Terraform (or other provisioners) may need to cleanup state, or
cancel actions before exit. This adds the ability to gracefully
exit provisionerd.

* Fix cancel error check
This commit is contained in:
Kyle Carberry
2022-02-28 12:40:49 -06:00
committed by GitHub
parent e5c95552cd
commit 9d2803e07a
13 changed files with 1091 additions and 537 deletions

View File

@ -404,8 +404,8 @@ func (server *provisionerdServer) UpdateJob(ctx context.Context, request *proto.
return &proto.UpdateJobResponse{}, nil return &proto.UpdateJobResponse{}, nil
} }
func (server *provisionerdServer) CancelJob(ctx context.Context, cancelJob *proto.CancelledJob) (*proto.Empty, error) { func (server *provisionerdServer) FailJob(ctx context.Context, failJob *proto.FailedJob) (*proto.Empty, error) {
jobID, err := uuid.Parse(cancelJob.JobId) jobID, err := uuid.Parse(failJob.JobId)
if err != nil { if err != nil {
return nil, xerrors.Errorf("parse job id: %w", err) return nil, xerrors.Errorf("parse job id: %w", err)
} }
@ -422,19 +422,35 @@ func (server *provisionerdServer) CancelJob(ctx context.Context, cancelJob *prot
Time: database.Now(), Time: database.Now(),
Valid: true, Valid: true,
}, },
CancelledAt: sql.NullTime{
Time: database.Now(),
Valid: true,
},
UpdatedAt: database.Now(), UpdatedAt: database.Now(),
Error: sql.NullString{ Error: sql.NullString{
String: cancelJob.Error, String: failJob.Error,
Valid: cancelJob.Error != "", Valid: failJob.Error != "",
}, },
}) })
if err != nil { if err != nil {
return nil, xerrors.Errorf("update provisioner job: %w", err) return nil, xerrors.Errorf("update provisioner job: %w", err)
} }
switch jobType := failJob.Type.(type) {
case *proto.FailedJob_WorkspaceProvision_:
if jobType.WorkspaceProvision.State == nil {
break
}
var input workspaceProvisionJob
err = json.Unmarshal(job.Input, &input)
if err != nil {
return nil, xerrors.Errorf("unmarshal workspace provision input: %w", err)
}
err = server.Database.UpdateWorkspaceHistoryByID(ctx, database.UpdateWorkspaceHistoryByIDParams{
ID: jobID,
UpdatedAt: database.Now(),
ProvisionerState: jobType.WorkspaceProvision.State,
})
if err != nil {
return nil, xerrors.Errorf("update workspace history state: %w", err)
}
case *proto.FailedJob_ProjectImport_:
}
return &proto.Empty{}, nil return &proto.Empty{}, nil
} }

View File

@ -276,7 +276,11 @@ func convertProvisionerJob(provisionerJob database.ProvisionerJob) ProvisionerJo
case !provisionerJob.StartedAt.Valid: case !provisionerJob.StartedAt.Valid:
job.Status = ProvisionerJobStatusPending job.Status = ProvisionerJobStatusPending
case provisionerJob.CompletedAt.Valid: case provisionerJob.CompletedAt.Valid:
job.Status = ProvisionerJobStatusSucceeded if job.Error == "" {
job.Status = ProvisionerJobStatusSucceeded
} else {
job.Status = ProvisionerJobStatusFailed
}
case database.Now().Sub(provisionerJob.UpdatedAt) > 30*time.Second: case database.Now().Sub(provisionerJob.UpdatedAt) > 30*time.Second:
job.Status = ProvisionerJobStatusFailed job.Status = ProvisionerJobStatusFailed
job.Error = "Worker failed to update job in time." job.Error = "Worker failed to update job in time."
@ -284,10 +288,6 @@ func convertProvisionerJob(provisionerJob database.ProvisionerJob) ProvisionerJo
job.Status = ProvisionerJobStatusRunning job.Status = ProvisionerJobStatusRunning
} }
if !provisionerJob.CancelledAt.Valid && job.Error != "" {
job.Status = ProvisionerJobStatusFailed
}
return job return job
} }

View File

@ -102,6 +102,10 @@ func (*echo) Provision(request *proto.Provision_Request, stream proto.DRPCProvis
return stream.Context().Err() return stream.Context().Err()
} }
func (*echo) Shutdown(_ context.Context, _ *proto.Empty) (*proto.Empty, error) {
return &proto.Empty{}, nil
}
type Responses struct { type Responses struct {
Parse []*proto.Parse_Response Parse []*proto.Parse_Response
Provision []*proto.Provision_Response Provision []*proto.Provision_Response

View File

@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"reflect" "reflect"
"strings" "strings"
@ -253,25 +254,21 @@ func (t *terraform) runTerraformPlan(ctx context.Context, terraform *tfexec.Terr
} }
func (t *terraform) runTerraformApply(ctx context.Context, terraform *tfexec.Terraform, request *proto.Provision_Request, stream proto.DRPCProvisioner_ProvisionStream, statefilePath string) error { func (t *terraform) runTerraformApply(ctx context.Context, terraform *tfexec.Terraform, request *proto.Provision_Request, stream proto.DRPCProvisioner_ProvisionStream, statefilePath string) error {
env := map[string]string{ env := []string{
"CODER_URL": request.Metadata.CoderUrl, "CODER_URL=" + request.Metadata.CoderUrl,
"CODER_WORKSPACE_TRANSITION": strings.ToLower(request.Metadata.WorkspaceTransition.String()), "CODER_WORKSPACE_TRANSITION=" + strings.ToLower(request.Metadata.WorkspaceTransition.String()),
} }
options := []tfexec.ApplyOption{tfexec.JSON(true)} vars := []string{}
for _, param := range request.ParameterValues { for _, param := range request.ParameterValues {
switch param.DestinationScheme { switch param.DestinationScheme {
case proto.ParameterDestination_ENVIRONMENT_VARIABLE: case proto.ParameterDestination_ENVIRONMENT_VARIABLE:
env[param.Name] = param.Value env = append(env, fmt.Sprintf("%s=%s", param.Name, param.Value))
case proto.ParameterDestination_PROVISIONER_VARIABLE: case proto.ParameterDestination_PROVISIONER_VARIABLE:
options = append(options, tfexec.Var(fmt.Sprintf("%s=%s", param.Name, param.Value))) vars = append(vars, fmt.Sprintf("%s=%s", param.Name, param.Value))
default: default:
return xerrors.Errorf("unsupported parameter type %q for %q", param.DestinationScheme, param.Name) return xerrors.Errorf("unsupported parameter type %q for %q", param.DestinationScheme, param.Name)
} }
} }
err := terraform.SetEnv(env)
if err != nil {
return xerrors.Errorf("apply environment variables: %w", err)
}
reader, writer := io.Pipe() reader, writer := io.Pipe()
defer reader.Close() defer reader.Close()
@ -319,11 +316,24 @@ func (t *terraform) runTerraformApply(ctx context.Context, terraform *tfexec.Ter
} }
}() }()
terraform.SetStdout(writer) t.logger.Debug(ctx, "running apply", slog.F("vars", len(vars)), slog.F("env", len(env)))
t.logger.Debug(ctx, "running apply", slog.F("options", options)) err := runApplyCommand(ctx, t.shutdownCtx, terraform.ExecPath(), terraform.WorkingDir(), writer, env, vars)
err = terraform.Apply(ctx, options...)
if err != nil { if err != nil {
return xerrors.Errorf("apply terraform: %w", err) errorMessage := err.Error()
// Terraform can fail and apply and still need to store it's state.
// In this case, we return Complete with an explicit error message.
statefileContent, err := os.ReadFile(statefilePath)
if err != nil {
return xerrors.Errorf("read file %q: %w", statefilePath, err)
}
return stream.Send(&proto.Provision_Response{
Type: &proto.Provision_Response_Complete{
Complete: &proto.Provision_Complete{
State: statefileContent,
Error: errorMessage,
},
},
})
} }
t.logger.Debug(ctx, "ran apply") t.logger.Debug(ctx, "ran apply")
@ -428,6 +438,35 @@ func (t *terraform) runTerraformApply(ctx context.Context, terraform *tfexec.Ter
}) })
} }
// This couldn't use terraform-exec, because it doesn't support cancellation, and there didn't appear
// to be a straight-forward way to add it.
func runApplyCommand(ctx, shutdownCtx context.Context, bin, dir string, stdout io.Writer, env, vars []string) error {
args := []string{
"apply",
"-no-color",
"-auto-approve",
"-input=false",
"-json",
"-refresh=true",
}
for _, variable := range vars {
args = append(args, "-var", variable)
}
cmd := exec.CommandContext(ctx, bin, args...)
go func() {
select {
case <-ctx.Done():
return
case <-shutdownCtx.Done():
_ = cmd.Process.Signal(os.Kill)
}
}()
cmd.Stdout = stdout
cmd.Env = env
cmd.Dir = dir
return cmd.Run()
}
type terraformProvisionLog struct { type terraformProvisionLog struct {
Level string `json:"@level"` Level string `json:"@level"`
Message string `json:"@message"` Message string `json:"@message"`

View File

@ -10,6 +10,7 @@ import (
"cdr.dev/slog" "cdr.dev/slog"
"github.com/coder/coder/provisionersdk" "github.com/coder/coder/provisionersdk"
"github.com/coder/coder/provisionersdk/proto"
) )
var ( var (
@ -43,14 +44,25 @@ func Serve(ctx context.Context, options *ServeOptions) error {
} }
options.BinaryPath = binaryPath options.BinaryPath = binaryPath
} }
shutdownCtx, shutdownCancel := context.WithCancel(ctx)
return provisionersdk.Serve(ctx, &terraform{ return provisionersdk.Serve(ctx, &terraform{
binaryPath: options.BinaryPath, binaryPath: options.BinaryPath,
logger: options.Logger, logger: options.Logger,
shutdownCtx: shutdownCtx,
shutdownCancel: shutdownCancel,
}, options.ServeOptions) }, options.ServeOptions)
} }
type terraform struct { type terraform struct {
binaryPath string binaryPath string
logger slog.Logger logger slog.Logger
shutdownCtx context.Context
shutdownCancel context.CancelFunc
}
// Shutdown signals to begin graceful shutdown of any running operations.
func (t *terraform) Shutdown(_ context.Context, _ *proto.Empty) (*proto.Empty, error) {
t.shutdownCancel()
return &proto.Empty{}, nil
} }

View File

@ -228,17 +228,21 @@ func (*AcquiredJob_WorkspaceProvision_) isAcquiredJob_Type() {}
func (*AcquiredJob_ProjectImport_) isAcquiredJob_Type() {} func (*AcquiredJob_ProjectImport_) isAcquiredJob_Type() {}
type CancelledJob struct { type FailedJob struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
// Types that are assignable to Type:
// *FailedJob_WorkspaceProvision_
// *FailedJob_ProjectImport_
Type isFailedJob_Type `protobuf_oneof:"type"`
} }
func (x *CancelledJob) Reset() { func (x *FailedJob) Reset() {
*x = CancelledJob{} *x = FailedJob{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[2] mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@ -246,13 +250,13 @@ func (x *CancelledJob) Reset() {
} }
} }
func (x *CancelledJob) String() string { func (x *FailedJob) String() string {
return protoimpl.X.MessageStringOf(x) return protoimpl.X.MessageStringOf(x)
} }
func (*CancelledJob) ProtoMessage() {} func (*FailedJob) ProtoMessage() {}
func (x *CancelledJob) ProtoReflect() protoreflect.Message { func (x *FailedJob) ProtoReflect() protoreflect.Message {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[2] mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@ -264,25 +268,62 @@ func (x *CancelledJob) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x) return mi.MessageOf(x)
} }
// Deprecated: Use CancelledJob.ProtoReflect.Descriptor instead. // Deprecated: Use FailedJob.ProtoReflect.Descriptor instead.
func (*CancelledJob) Descriptor() ([]byte, []int) { func (*FailedJob) Descriptor() ([]byte, []int) {
return file_provisionerd_proto_provisionerd_proto_rawDescGZIP(), []int{2} return file_provisionerd_proto_provisionerd_proto_rawDescGZIP(), []int{2}
} }
func (x *CancelledJob) GetJobId() string { func (x *FailedJob) GetJobId() string {
if x != nil { if x != nil {
return x.JobId return x.JobId
} }
return "" return ""
} }
func (x *CancelledJob) GetError() string { func (x *FailedJob) GetError() string {
if x != nil { if x != nil {
return x.Error return x.Error
} }
return "" return ""
} }
func (m *FailedJob) GetType() isFailedJob_Type {
if m != nil {
return m.Type
}
return nil
}
func (x *FailedJob) GetWorkspaceProvision() *FailedJob_WorkspaceProvision {
if x, ok := x.GetType().(*FailedJob_WorkspaceProvision_); ok {
return x.WorkspaceProvision
}
return nil
}
func (x *FailedJob) GetProjectImport() *FailedJob_ProjectImport {
if x, ok := x.GetType().(*FailedJob_ProjectImport_); ok {
return x.ProjectImport
}
return nil
}
type isFailedJob_Type interface {
isFailedJob_Type()
}
type FailedJob_WorkspaceProvision_ struct {
WorkspaceProvision *FailedJob_WorkspaceProvision `protobuf:"bytes,3,opt,name=workspace_provision,json=workspaceProvision,proto3,oneof"`
}
type FailedJob_ProjectImport_ struct {
ProjectImport *FailedJob_ProjectImport `protobuf:"bytes,4,opt,name=project_import,json=projectImport,proto3,oneof"`
}
func (*FailedJob_WorkspaceProvision_) isFailedJob_Type() {}
func (*FailedJob_ProjectImport_) isFailedJob_Type() {}
// CompletedJob is sent when the provisioner daemon completes a job. // CompletedJob is sent when the provisioner daemon completes a job.
type CompletedJob struct { type CompletedJob struct {
state protoimpl.MessageState state protoimpl.MessageState
@ -683,6 +724,91 @@ func (x *AcquiredJob_ProjectImport) GetMetadata() *proto.Provision_Metadata {
return nil return nil
} }
type FailedJob_WorkspaceProvision struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
State []byte `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"`
}
func (x *FailedJob_WorkspaceProvision) Reset() {
*x = FailedJob_WorkspaceProvision{}
if protoimpl.UnsafeEnabled {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FailedJob_WorkspaceProvision) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FailedJob_WorkspaceProvision) ProtoMessage() {}
func (x *FailedJob_WorkspaceProvision) ProtoReflect() protoreflect.Message {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FailedJob_WorkspaceProvision.ProtoReflect.Descriptor instead.
func (*FailedJob_WorkspaceProvision) Descriptor() ([]byte, []int) {
return file_provisionerd_proto_provisionerd_proto_rawDescGZIP(), []int{2, 0}
}
func (x *FailedJob_WorkspaceProvision) GetState() []byte {
if x != nil {
return x.State
}
return nil
}
type FailedJob_ProjectImport struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *FailedJob_ProjectImport) Reset() {
*x = FailedJob_ProjectImport{}
if protoimpl.UnsafeEnabled {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FailedJob_ProjectImport) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FailedJob_ProjectImport) ProtoMessage() {}
func (x *FailedJob_ProjectImport) ProtoReflect() protoreflect.Message {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[10]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FailedJob_ProjectImport.ProtoReflect.Descriptor instead.
func (*FailedJob_ProjectImport) Descriptor() ([]byte, []int) {
return file_provisionerd_proto_provisionerd_proto_rawDescGZIP(), []int{2, 1}
}
type CompletedJob_WorkspaceProvision struct { type CompletedJob_WorkspaceProvision struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
@ -695,7 +821,7 @@ type CompletedJob_WorkspaceProvision struct {
func (x *CompletedJob_WorkspaceProvision) Reset() { func (x *CompletedJob_WorkspaceProvision) Reset() {
*x = CompletedJob_WorkspaceProvision{} *x = CompletedJob_WorkspaceProvision{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[9] mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@ -708,7 +834,7 @@ func (x *CompletedJob_WorkspaceProvision) String() string {
func (*CompletedJob_WorkspaceProvision) ProtoMessage() {} func (*CompletedJob_WorkspaceProvision) ProtoMessage() {}
func (x *CompletedJob_WorkspaceProvision) ProtoReflect() protoreflect.Message { func (x *CompletedJob_WorkspaceProvision) ProtoReflect() protoreflect.Message {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[9] mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[11]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@ -750,7 +876,7 @@ type CompletedJob_ProjectImport struct {
func (x *CompletedJob_ProjectImport) Reset() { func (x *CompletedJob_ProjectImport) Reset() {
*x = CompletedJob_ProjectImport{} *x = CompletedJob_ProjectImport{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[10] mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@ -763,7 +889,7 @@ func (x *CompletedJob_ProjectImport) String() string {
func (*CompletedJob_ProjectImport) ProtoMessage() {} func (*CompletedJob_ProjectImport) ProtoMessage() {}
func (x *CompletedJob_ProjectImport) ProtoReflect() protoreflect.Message { func (x *CompletedJob_ProjectImport) ProtoReflect() protoreflect.Message {
mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[10] mi := &file_provisionerd_proto_provisionerd_proto_msgTypes[12]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@ -847,90 +973,105 @@ var file_provisionerd_proto_provisionerd_proto_rawDesc = []byte{
0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x06, 0x0a, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x06, 0x0a,
0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3b, 0x0a, 0x0c, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xac, 0x02, 0x0a, 0x09, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64,
0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05,
0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72,
0x6f, 0x72, 0x22, 0xd3, 0x03, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64,
0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x60, 0x0a, 0x13, 0x77, 0x6f, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72,
0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72,
0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x12, 0x5d, 0x0a, 0x13, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x72,
0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e,
0x4a, 0x6f, 0x62, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x46, 0x61, 0x69,
0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x12, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65,
0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x51, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x12, 0x77, 0x6f, 0x72,
0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x6d, 0x70, 0x6f, 0x72,
0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73,
0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62,
0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x00, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x00,
0x52, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x1a,
0x5f, 0x0a, 0x12, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x2a, 0x0a, 0x12, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76,
0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x33, 0x0a, 0x09, 0x72, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x0f, 0x0a, 0x0d, 0x50,
0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x42, 0x06, 0x0a, 0x04,
0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x79, 0x70, 0x65, 0x22, 0xd3, 0x03, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18,
0x1a, 0x8d, 0x01, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x60, 0x0a, 0x13,
0x72, 0x74, 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73,
0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x76,
0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74,
0x63, 0x65, 0x52, 0x0e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50,
0x65, 0x73, 0x12, 0x3c, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x12, 0x77, 0x6f, 0x72, 0x6b,
0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x51,
0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74,
0x65, 0x52, 0x0d, 0x73, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69,
0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x9a, 0x01, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a,
0x12, 0x2f, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x6f, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74,
0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d, 0x70, 0x6f, 0x72,
0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x74, 0x1a, 0x5f, 0x0a, 0x12, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x72,
0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65,
0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x33, 0x0a,
0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1d, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52,
0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63,
0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x65, 0x73, 0x1a, 0x8d, 0x01, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x6d,
0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0x9b, 0x01, 0x0a, 0x10, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x72, 0x65,
0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e,
0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f,
0x64, 0x12, 0x25, 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75,
0x11, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x72, 0x63, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x72, 0x65, 0x73,
0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x49, 0x0a, 0x11, 0x70, 0x61, 0x72, 0x61, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70,
0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18, 0x03, 0x20, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75,
0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x63, 0x65, 0x52, 0x0d, 0x73, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63,
0x72, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x9a, 0x01, 0x0a, 0x03, 0x4c,
0x61, 0x52, 0x10, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, 0x68, 0x65, 0x6f, 0x67, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01,
0x6d, 0x61, 0x73, 0x22, 0x5b, 0x0a, 0x11, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x10, 0x70, 0x61, 0x72, 0x61, 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75,
0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x72, 0x63, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c,
0x0f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03,
0x2a, 0x34, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12,
0x12, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x45, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
0x4d, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0x9b, 0x01, 0x0a, 0x10, 0x55, 0x70, 0x64, 0x61,
0x4f, 0x4e, 0x45, 0x52, 0x10, 0x01, 0x32, 0x9d, 0x02, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x15, 0x0a, 0x06,
0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x12, 0x3c, 0x0a, 0x0a, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f,
0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
0x19, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x49, 0x0a, 0x11, 0x70, 0x61,
0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x4c, 0x0a, 0x09, 0x55, 0x70, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18,
0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x6e, 0x65, 0x72, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63, 0x68,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x65, 0x6d, 0x61, 0x52, 0x10, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x53, 0x63,
0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x22, 0x5b, 0x0a, 0x11, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x43, 0x61, 0x6e, 0x63, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x10, 0x70, 0x61,
0x65, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01,
0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e,
0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3e, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x65, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75,
0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x65, 0x73, 0x2a, 0x34, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12,
0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x16, 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x5f, 0x44,
0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x41, 0x45, 0x4d, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x56, 0x49,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x10, 0x01, 0x32, 0x98, 0x02, 0x0a, 0x11, 0x50, 0x72, 0x6f,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x12, 0x3c,
0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x70, 0x72, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x13, 0x2e, 0x70,
0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x4c, 0x0a, 0x09,
0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x76,
0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a,
0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x76,
0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a,
0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x46, 0x61,
0x69, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
0x6e, 0x65, 0x72, 0x64, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13,
0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x12, 0x3e, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4a,
0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13,
0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, 0x72,
0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
} }
var ( var (
@ -946,55 +1087,59 @@ func file_provisionerd_proto_provisionerd_proto_rawDescGZIP() []byte {
} }
var file_provisionerd_proto_provisionerd_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_provisionerd_proto_provisionerd_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_provisionerd_proto_provisionerd_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_provisionerd_proto_provisionerd_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
var file_provisionerd_proto_provisionerd_proto_goTypes = []interface{}{ var file_provisionerd_proto_provisionerd_proto_goTypes = []interface{}{
(LogSource)(0), // 0: provisionerd.LogSource (LogSource)(0), // 0: provisionerd.LogSource
(*Empty)(nil), // 1: provisionerd.Empty (*Empty)(nil), // 1: provisionerd.Empty
(*AcquiredJob)(nil), // 2: provisionerd.AcquiredJob (*AcquiredJob)(nil), // 2: provisionerd.AcquiredJob
(*CancelledJob)(nil), // 3: provisionerd.CancelledJob (*FailedJob)(nil), // 3: provisionerd.FailedJob
(*CompletedJob)(nil), // 4: provisionerd.CompletedJob (*CompletedJob)(nil), // 4: provisionerd.CompletedJob
(*Log)(nil), // 5: provisionerd.Log (*Log)(nil), // 5: provisionerd.Log
(*UpdateJobRequest)(nil), // 6: provisionerd.UpdateJobRequest (*UpdateJobRequest)(nil), // 6: provisionerd.UpdateJobRequest
(*UpdateJobResponse)(nil), // 7: provisionerd.UpdateJobResponse (*UpdateJobResponse)(nil), // 7: provisionerd.UpdateJobResponse
(*AcquiredJob_WorkspaceProvision)(nil), // 8: provisionerd.AcquiredJob.WorkspaceProvision (*AcquiredJob_WorkspaceProvision)(nil), // 8: provisionerd.AcquiredJob.WorkspaceProvision
(*AcquiredJob_ProjectImport)(nil), // 9: provisionerd.AcquiredJob.ProjectImport (*AcquiredJob_ProjectImport)(nil), // 9: provisionerd.AcquiredJob.ProjectImport
(*CompletedJob_WorkspaceProvision)(nil), // 10: provisionerd.CompletedJob.WorkspaceProvision (*FailedJob_WorkspaceProvision)(nil), // 10: provisionerd.FailedJob.WorkspaceProvision
(*CompletedJob_ProjectImport)(nil), // 11: provisionerd.CompletedJob.ProjectImport (*FailedJob_ProjectImport)(nil), // 11: provisionerd.FailedJob.ProjectImport
(proto.LogLevel)(0), // 12: provisioner.LogLevel (*CompletedJob_WorkspaceProvision)(nil), // 12: provisionerd.CompletedJob.WorkspaceProvision
(*proto.ParameterSchema)(nil), // 13: provisioner.ParameterSchema (*CompletedJob_ProjectImport)(nil), // 13: provisionerd.CompletedJob.ProjectImport
(*proto.ParameterValue)(nil), // 14: provisioner.ParameterValue (proto.LogLevel)(0), // 14: provisioner.LogLevel
(*proto.Provision_Metadata)(nil), // 15: provisioner.Provision.Metadata (*proto.ParameterSchema)(nil), // 15: provisioner.ParameterSchema
(*proto.Resource)(nil), // 16: provisioner.Resource (*proto.ParameterValue)(nil), // 16: provisioner.ParameterValue
(*proto.Provision_Metadata)(nil), // 17: provisioner.Provision.Metadata
(*proto.Resource)(nil), // 18: provisioner.Resource
} }
var file_provisionerd_proto_provisionerd_proto_depIdxs = []int32{ var file_provisionerd_proto_provisionerd_proto_depIdxs = []int32{
8, // 0: provisionerd.AcquiredJob.workspace_provision:type_name -> provisionerd.AcquiredJob.WorkspaceProvision 8, // 0: provisionerd.AcquiredJob.workspace_provision:type_name -> provisionerd.AcquiredJob.WorkspaceProvision
9, // 1: provisionerd.AcquiredJob.project_import:type_name -> provisionerd.AcquiredJob.ProjectImport 9, // 1: provisionerd.AcquiredJob.project_import:type_name -> provisionerd.AcquiredJob.ProjectImport
10, // 2: provisionerd.CompletedJob.workspace_provision:type_name -> provisionerd.CompletedJob.WorkspaceProvision 10, // 2: provisionerd.FailedJob.workspace_provision:type_name -> provisionerd.FailedJob.WorkspaceProvision
11, // 3: provisionerd.CompletedJob.project_import:type_name -> provisionerd.CompletedJob.ProjectImport 11, // 3: provisionerd.FailedJob.project_import:type_name -> provisionerd.FailedJob.ProjectImport
0, // 4: provisionerd.Log.source:type_name -> provisionerd.LogSource 12, // 4: provisionerd.CompletedJob.workspace_provision:type_name -> provisionerd.CompletedJob.WorkspaceProvision
12, // 5: provisionerd.Log.level:type_name -> provisioner.LogLevel 13, // 5: provisionerd.CompletedJob.project_import:type_name -> provisionerd.CompletedJob.ProjectImport
5, // 6: provisionerd.UpdateJobRequest.logs:type_name -> provisionerd.Log 0, // 6: provisionerd.Log.source:type_name -> provisionerd.LogSource
13, // 7: provisionerd.UpdateJobRequest.parameter_schemas:type_name -> provisioner.ParameterSchema 14, // 7: provisionerd.Log.level:type_name -> provisioner.LogLevel
14, // 8: provisionerd.UpdateJobResponse.parameter_values:type_name -> provisioner.ParameterValue 5, // 8: provisionerd.UpdateJobRequest.logs:type_name -> provisionerd.Log
14, // 9: provisionerd.AcquiredJob.WorkspaceProvision.parameter_values:type_name -> provisioner.ParameterValue 15, // 9: provisionerd.UpdateJobRequest.parameter_schemas:type_name -> provisioner.ParameterSchema
15, // 10: provisionerd.AcquiredJob.WorkspaceProvision.metadata:type_name -> provisioner.Provision.Metadata 16, // 10: provisionerd.UpdateJobResponse.parameter_values:type_name -> provisioner.ParameterValue
15, // 11: provisionerd.AcquiredJob.ProjectImport.metadata:type_name -> provisioner.Provision.Metadata 16, // 11: provisionerd.AcquiredJob.WorkspaceProvision.parameter_values:type_name -> provisioner.ParameterValue
16, // 12: provisionerd.CompletedJob.WorkspaceProvision.resources:type_name -> provisioner.Resource 17, // 12: provisionerd.AcquiredJob.WorkspaceProvision.metadata:type_name -> provisioner.Provision.Metadata
16, // 13: provisionerd.CompletedJob.ProjectImport.start_resources:type_name -> provisioner.Resource 17, // 13: provisionerd.AcquiredJob.ProjectImport.metadata:type_name -> provisioner.Provision.Metadata
16, // 14: provisionerd.CompletedJob.ProjectImport.stop_resources:type_name -> provisioner.Resource 18, // 14: provisionerd.CompletedJob.WorkspaceProvision.resources:type_name -> provisioner.Resource
1, // 15: provisionerd.ProvisionerDaemon.AcquireJob:input_type -> provisionerd.Empty 18, // 15: provisionerd.CompletedJob.ProjectImport.start_resources:type_name -> provisioner.Resource
6, // 16: provisionerd.ProvisionerDaemon.UpdateJob:input_type -> provisionerd.UpdateJobRequest 18, // 16: provisionerd.CompletedJob.ProjectImport.stop_resources:type_name -> provisioner.Resource
3, // 17: provisionerd.ProvisionerDaemon.CancelJob:input_type -> provisionerd.CancelledJob 1, // 17: provisionerd.ProvisionerDaemon.AcquireJob:input_type -> provisionerd.Empty
4, // 18: provisionerd.ProvisionerDaemon.CompleteJob:input_type -> provisionerd.CompletedJob 6, // 18: provisionerd.ProvisionerDaemon.UpdateJob:input_type -> provisionerd.UpdateJobRequest
2, // 19: provisionerd.ProvisionerDaemon.AcquireJob:output_type -> provisionerd.AcquiredJob 3, // 19: provisionerd.ProvisionerDaemon.FailJob:input_type -> provisionerd.FailedJob
7, // 20: provisionerd.ProvisionerDaemon.UpdateJob:output_type -> provisionerd.UpdateJobResponse 4, // 20: provisionerd.ProvisionerDaemon.CompleteJob:input_type -> provisionerd.CompletedJob
1, // 21: provisionerd.ProvisionerDaemon.CancelJob:output_type -> provisionerd.Empty 2, // 21: provisionerd.ProvisionerDaemon.AcquireJob:output_type -> provisionerd.AcquiredJob
1, // 22: provisionerd.ProvisionerDaemon.CompleteJob:output_type -> provisionerd.Empty 7, // 22: provisionerd.ProvisionerDaemon.UpdateJob:output_type -> provisionerd.UpdateJobResponse
19, // [19:23] is the sub-list for method output_type 1, // 23: provisionerd.ProvisionerDaemon.FailJob:output_type -> provisionerd.Empty
15, // [15:19] is the sub-list for method input_type 1, // 24: provisionerd.ProvisionerDaemon.CompleteJob:output_type -> provisionerd.Empty
15, // [15:15] is the sub-list for extension type_name 21, // [21:25] is the sub-list for method output_type
15, // [15:15] is the sub-list for extension extendee 17, // [17:21] is the sub-list for method input_type
0, // [0:15] is the sub-list for field type_name 17, // [17:17] is the sub-list for extension type_name
17, // [17:17] is the sub-list for extension extendee
0, // [0:17] is the sub-list for field type_name
} }
func init() { file_provisionerd_proto_provisionerd_proto_init() } func init() { file_provisionerd_proto_provisionerd_proto_init() }
@ -1028,7 +1173,7 @@ func file_provisionerd_proto_provisionerd_proto_init() {
} }
} }
file_provisionerd_proto_provisionerd_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { file_provisionerd_proto_provisionerd_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CancelledJob); i { switch v := v.(*FailedJob); i {
case 0: case 0:
return &v.state return &v.state
case 1: case 1:
@ -1112,7 +1257,7 @@ func file_provisionerd_proto_provisionerd_proto_init() {
} }
} }
file_provisionerd_proto_provisionerd_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { file_provisionerd_proto_provisionerd_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CompletedJob_WorkspaceProvision); i { switch v := v.(*FailedJob_WorkspaceProvision); i {
case 0: case 0:
return &v.state return &v.state
case 1: case 1:
@ -1124,6 +1269,30 @@ func file_provisionerd_proto_provisionerd_proto_init() {
} }
} }
file_provisionerd_proto_provisionerd_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { file_provisionerd_proto_provisionerd_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FailedJob_ProjectImport); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_provisionerd_proto_provisionerd_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CompletedJob_WorkspaceProvision); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_provisionerd_proto_provisionerd_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CompletedJob_ProjectImport); i { switch v := v.(*CompletedJob_ProjectImport); i {
case 0: case 0:
return &v.state return &v.state
@ -1140,6 +1309,10 @@ func file_provisionerd_proto_provisionerd_proto_init() {
(*AcquiredJob_WorkspaceProvision_)(nil), (*AcquiredJob_WorkspaceProvision_)(nil),
(*AcquiredJob_ProjectImport_)(nil), (*AcquiredJob_ProjectImport_)(nil),
} }
file_provisionerd_proto_provisionerd_proto_msgTypes[2].OneofWrappers = []interface{}{
(*FailedJob_WorkspaceProvision_)(nil),
(*FailedJob_ProjectImport_)(nil),
}
file_provisionerd_proto_provisionerd_proto_msgTypes[3].OneofWrappers = []interface{}{ file_provisionerd_proto_provisionerd_proto_msgTypes[3].OneofWrappers = []interface{}{
(*CompletedJob_WorkspaceProvision_)(nil), (*CompletedJob_WorkspaceProvision_)(nil),
(*CompletedJob_ProjectImport_)(nil), (*CompletedJob_ProjectImport_)(nil),
@ -1150,7 +1323,7 @@ func file_provisionerd_proto_provisionerd_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_provisionerd_proto_provisionerd_proto_rawDesc, RawDescriptor: file_provisionerd_proto_provisionerd_proto_rawDesc,
NumEnums: 1, NumEnums: 1,
NumMessages: 11, NumMessages: 13,
NumExtensions: 0, NumExtensions: 0,
NumServices: 1, NumServices: 1,
}, },

View File

@ -32,9 +32,18 @@ message AcquiredJob {
} }
} }
message CancelledJob { message FailedJob {
message WorkspaceProvision {
bytes state = 1;
}
message ProjectImport{
}
string job_id = 1; string job_id = 1;
string error = 2; string error = 2;
oneof type {
WorkspaceProvision workspace_provision = 3;
ProjectImport project_import = 4;
}
} }
// CompletedJob is sent when the provisioner daemon completes a job. // CompletedJob is sent when the provisioner daemon completes a job.
@ -92,9 +101,8 @@ service ProvisionerDaemon {
// is non-blocking. // is non-blocking.
rpc UpdateJob(UpdateJobRequest) returns (UpdateJobResponse); rpc UpdateJob(UpdateJobRequest) returns (UpdateJobResponse);
// CancelJob indicates a job has been cancelled with // FailJob indicates a job has failed.
// an error message. rpc FailJob(FailedJob) returns (Empty);
rpc CancelJob(CancelledJob) returns (Empty);
// CompleteJob indicates a job has been completed. // CompleteJob indicates a job has been completed.
rpc CompleteJob(CompletedJob) returns (Empty); rpc CompleteJob(CompletedJob) returns (Empty);

View File

@ -40,7 +40,7 @@ type DRPCProvisionerDaemonClient interface {
AcquireJob(ctx context.Context, in *Empty) (*AcquiredJob, error) AcquireJob(ctx context.Context, in *Empty) (*AcquiredJob, error)
UpdateJob(ctx context.Context, in *UpdateJobRequest) (*UpdateJobResponse, error) UpdateJob(ctx context.Context, in *UpdateJobRequest) (*UpdateJobResponse, error)
CancelJob(ctx context.Context, in *CancelledJob) (*Empty, error) FailJob(ctx context.Context, in *FailedJob) (*Empty, error)
CompleteJob(ctx context.Context, in *CompletedJob) (*Empty, error) CompleteJob(ctx context.Context, in *CompletedJob) (*Empty, error)
} }
@ -72,9 +72,9 @@ func (c *drpcProvisionerDaemonClient) UpdateJob(ctx context.Context, in *UpdateJ
return out, nil return out, nil
} }
func (c *drpcProvisionerDaemonClient) CancelJob(ctx context.Context, in *CancelledJob) (*Empty, error) { func (c *drpcProvisionerDaemonClient) FailJob(ctx context.Context, in *FailedJob) (*Empty, error) {
out := new(Empty) out := new(Empty)
err := c.cc.Invoke(ctx, "/provisionerd.ProvisionerDaemon/CancelJob", drpcEncoding_File_provisionerd_proto_provisionerd_proto{}, in, out) err := c.cc.Invoke(ctx, "/provisionerd.ProvisionerDaemon/FailJob", drpcEncoding_File_provisionerd_proto_provisionerd_proto{}, in, out)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -93,7 +93,7 @@ func (c *drpcProvisionerDaemonClient) CompleteJob(ctx context.Context, in *Compl
type DRPCProvisionerDaemonServer interface { type DRPCProvisionerDaemonServer interface {
AcquireJob(context.Context, *Empty) (*AcquiredJob, error) AcquireJob(context.Context, *Empty) (*AcquiredJob, error)
UpdateJob(context.Context, *UpdateJobRequest) (*UpdateJobResponse, error) UpdateJob(context.Context, *UpdateJobRequest) (*UpdateJobResponse, error)
CancelJob(context.Context, *CancelledJob) (*Empty, error) FailJob(context.Context, *FailedJob) (*Empty, error)
CompleteJob(context.Context, *CompletedJob) (*Empty, error) CompleteJob(context.Context, *CompletedJob) (*Empty, error)
} }
@ -107,7 +107,7 @@ func (s *DRPCProvisionerDaemonUnimplementedServer) UpdateJob(context.Context, *U
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
} }
func (s *DRPCProvisionerDaemonUnimplementedServer) CancelJob(context.Context, *CancelledJob) (*Empty, error) { func (s *DRPCProvisionerDaemonUnimplementedServer) FailJob(context.Context, *FailedJob) (*Empty, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
} }
@ -140,14 +140,14 @@ func (DRPCProvisionerDaemonDescription) Method(n int) (string, drpc.Encoding, dr
) )
}, DRPCProvisionerDaemonServer.UpdateJob, true }, DRPCProvisionerDaemonServer.UpdateJob, true
case 2: case 2:
return "/provisionerd.ProvisionerDaemon/CancelJob", drpcEncoding_File_provisionerd_proto_provisionerd_proto{}, return "/provisionerd.ProvisionerDaemon/FailJob", drpcEncoding_File_provisionerd_proto_provisionerd_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCProvisionerDaemonServer). return srv.(DRPCProvisionerDaemonServer).
CancelJob( FailJob(
ctx, ctx,
in1.(*CancelledJob), in1.(*FailedJob),
) )
}, DRPCProvisionerDaemonServer.CancelJob, true }, DRPCProvisionerDaemonServer.FailJob, true
case 3: case 3:
return "/provisionerd.ProvisionerDaemon/CompleteJob", drpcEncoding_File_provisionerd_proto_provisionerd_proto{}, return "/provisionerd.ProvisionerDaemon/CompleteJob", drpcEncoding_File_provisionerd_proto_provisionerd_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
@ -198,16 +198,16 @@ func (x *drpcProvisionerDaemon_UpdateJobStream) SendAndClose(m *UpdateJobRespons
return x.CloseSend() return x.CloseSend()
} }
type DRPCProvisionerDaemon_CancelJobStream interface { type DRPCProvisionerDaemon_FailJobStream interface {
drpc.Stream drpc.Stream
SendAndClose(*Empty) error SendAndClose(*Empty) error
} }
type drpcProvisionerDaemon_CancelJobStream struct { type drpcProvisionerDaemon_FailJobStream struct {
drpc.Stream drpc.Stream
} }
func (x *drpcProvisionerDaemon_CancelJobStream) SendAndClose(m *Empty) error { func (x *drpcProvisionerDaemon_FailJobStream) SendAndClose(m *Empty) error {
if err := x.MsgSend(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{}); err != nil { if err := x.MsgSend(m, drpcEncoding_File_provisionerd_proto_provisionerd_proto{}); err != nil {
return err return err
} }

View File

@ -52,7 +52,7 @@ type Options struct {
} }
// New creates and starts a provisioner daemon. // New creates and starts a provisioner daemon.
func New(clientDialer Dialer, opts *Options) io.Closer { func New(clientDialer Dialer, opts *Options) *Server {
if opts.PollInterval == 0 { if opts.PollInterval == 0 {
opts.PollInterval = 5 * time.Second opts.PollInterval = 5 * time.Second
} }
@ -60,15 +60,17 @@ func New(clientDialer Dialer, opts *Options) io.Closer {
opts.UpdateInterval = 5 * time.Second opts.UpdateInterval = 5 * time.Second
} }
ctx, ctxCancel := context.WithCancel(context.Background()) ctx, ctxCancel := context.WithCancel(context.Background())
daemon := &provisionerDaemon{ daemon := &Server{
clientDialer: clientDialer, clientDialer: clientDialer,
opts: opts, opts: opts,
closeCancel: ctxCancel, closeCancel: ctxCancel,
closed: make(chan struct{}), closed: make(chan struct{}),
jobRunning: make(chan struct{}), shutdown: make(chan struct{}),
jobCancelled: *atomic.NewBool(true),
jobRunning: make(chan struct{}),
jobFailed: *atomic.NewBool(true),
} }
// Start off with a closed channel so // Start off with a closed channel so
// isRunningJob() returns properly. // isRunningJob() returns properly.
@ -77,7 +79,7 @@ func New(clientDialer Dialer, opts *Options) io.Closer {
return daemon return daemon
} }
type provisionerDaemon struct { type Server struct {
opts *Options opts *Options
clientDialer Dialer clientDialer Dialer
@ -89,16 +91,19 @@ type provisionerDaemon struct {
closed chan struct{} closed chan struct{}
closeError error closeError error
// Locked when acquiring or canceling a job. shutdownMutex sync.Mutex
jobMutex sync.Mutex shutdown chan struct{}
jobID string
jobRunning chan struct{} // Locked when acquiring or failing a job.
jobCancelled atomic.Bool jobMutex sync.Mutex
jobCancel context.CancelFunc jobID string
jobRunning chan struct{}
jobFailed atomic.Bool
jobCancel context.CancelFunc
} }
// Connect establishes a connection to coderd. // Connect establishes a connection to coderd.
func (p *provisionerDaemon) connect(ctx context.Context) { func (p *Server) connect(ctx context.Context) {
var err error var err error
// An exponential back-off occurs when the connection is failing to dial. // An exponential back-off occurs when the connection is failing to dial.
// This is to prevent server spam in case of a coderd outage. // This is to prevent server spam in case of a coderd outage.
@ -161,7 +166,7 @@ func (p *provisionerDaemon) connect(ctx context.Context) {
}() }()
} }
func (p *provisionerDaemon) isRunningJob() bool { func (p *Server) isRunningJob() bool {
select { select {
case <-p.jobRunning: case <-p.jobRunning:
return false return false
@ -171,7 +176,7 @@ func (p *provisionerDaemon) isRunningJob() bool {
} }
// Locks a job in the database, and runs it! // Locks a job in the database, and runs it!
func (p *provisionerDaemon) acquireJob(ctx context.Context) { func (p *Server) acquireJob(ctx context.Context) {
p.jobMutex.Lock() p.jobMutex.Lock()
defer p.jobMutex.Unlock() defer p.jobMutex.Unlock()
if p.isClosed() { if p.isClosed() {
@ -181,6 +186,10 @@ func (p *provisionerDaemon) acquireJob(ctx context.Context) {
p.opts.Logger.Debug(context.Background(), "skipping acquire; job is already running") p.opts.Logger.Debug(context.Background(), "skipping acquire; job is already running")
return return
} }
if p.isShutdown() {
p.opts.Logger.Debug(context.Background(), "skipping acquire; provisionerd is shutting down...")
return
}
var err error var err error
job, err := p.client.AcquireJob(ctx, &proto.Empty{}) job, err := p.client.AcquireJob(ctx, &proto.Empty{})
if err != nil { if err != nil {
@ -199,7 +208,7 @@ func (p *provisionerDaemon) acquireJob(ctx context.Context) {
} }
ctx, p.jobCancel = context.WithCancel(ctx) ctx, p.jobCancel = context.WithCancel(ctx)
p.jobRunning = make(chan struct{}) p.jobRunning = make(chan struct{})
p.jobCancelled.Store(false) p.jobFailed.Store(false)
p.jobID = job.JobId p.jobID = job.JobId
p.opts.Logger.Info(context.Background(), "acquired job", p.opts.Logger.Info(context.Background(), "acquired job",
@ -211,7 +220,7 @@ func (p *provisionerDaemon) acquireJob(ctx context.Context) {
go p.runJob(ctx, job) go p.runJob(ctx, job)
} }
func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob) { func (p *Server) runJob(ctx context.Context, job *proto.AcquiredJob) {
go func() { go func() {
ticker := time.NewTicker(p.opts.UpdateInterval) ticker := time.NewTicker(p.opts.UpdateInterval)
defer ticker.Stop() defer ticker.Stop()
@ -225,7 +234,7 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob)
JobId: job.JobId, JobId: job.JobId,
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("send periodic update: %s", err) p.failActiveJobf("send periodic update: %s", err)
return return
} }
} }
@ -252,13 +261,13 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob)
// It's safe to cast this ProvisionerType. This data is coming directly from coderd. // It's safe to cast this ProvisionerType. This data is coming directly from coderd.
provisioner, hasProvisioner := p.opts.Provisioners[job.Provisioner] provisioner, hasProvisioner := p.opts.Provisioners[job.Provisioner]
if !hasProvisioner { if !hasProvisioner {
p.cancelActiveJobf("provisioner %q not registered", job.Provisioner) p.failActiveJobf("provisioner %q not registered", job.Provisioner)
return return
} }
err := os.MkdirAll(p.opts.WorkDirectory, 0700) err := os.MkdirAll(p.opts.WorkDirectory, 0700)
if err != nil { if err != nil {
p.cancelActiveJobf("create work directory %q: %s", p.opts.WorkDirectory, err) p.failActiveJobf("create work directory %q: %s", p.opts.WorkDirectory, err)
return return
} }
@ -270,13 +279,13 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob)
break break
} }
if err != nil { if err != nil {
p.cancelActiveJobf("read project source archive: %s", err) p.failActiveJobf("read project source archive: %s", err)
return return
} }
// #nosec // #nosec
path := filepath.Join(p.opts.WorkDirectory, header.Name) path := filepath.Join(p.opts.WorkDirectory, header.Name)
if !strings.HasPrefix(path, filepath.Clean(p.opts.WorkDirectory)) { if !strings.HasPrefix(path, filepath.Clean(p.opts.WorkDirectory)) {
p.cancelActiveJobf("tar attempts to target relative upper directory") p.failActiveJobf("tar attempts to target relative upper directory")
return return
} }
mode := header.FileInfo().Mode() mode := header.FileInfo().Mode()
@ -287,14 +296,14 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob)
case tar.TypeDir: case tar.TypeDir:
err = os.MkdirAll(path, mode) err = os.MkdirAll(path, mode)
if err != nil { if err != nil {
p.cancelActiveJobf("mkdir %q: %s", path, err) p.failActiveJobf("mkdir %q: %s", path, err)
return return
} }
p.opts.Logger.Debug(context.Background(), "extracted directory", slog.F("path", path)) p.opts.Logger.Debug(context.Background(), "extracted directory", slog.F("path", path))
case tar.TypeReg: case tar.TypeReg:
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, mode) file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, mode)
if err != nil { if err != nil {
p.cancelActiveJobf("create file %q (mode %s): %s", path, mode, err) p.failActiveJobf("create file %q (mode %s): %s", path, mode, err)
return return
} }
// Max file size of 10MB. // Max file size of 10MB.
@ -304,12 +313,12 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob)
} }
if err != nil { if err != nil {
_ = file.Close() _ = file.Close()
p.cancelActiveJobf("copy file %q: %s", path, err) p.failActiveJobf("copy file %q: %s", path, err)
return return
} }
err = file.Close() err = file.Close()
if err != nil { if err != nil {
p.cancelActiveJobf("close file %q: %s", path, err) p.failActiveJobf("close file %q: %s", path, err)
return return
} }
p.opts.Logger.Debug(context.Background(), "extracted file", p.opts.Logger.Debug(context.Background(), "extracted file",
@ -334,21 +343,21 @@ func (p *provisionerDaemon) runJob(ctx context.Context, job *proto.AcquiredJob)
p.runWorkspaceProvision(ctx, provisioner, job) p.runWorkspaceProvision(ctx, provisioner, job)
default: default:
p.cancelActiveJobf("unknown job type %q; ensure your provisioner daemon is up-to-date", reflect.TypeOf(job.Type).String()) p.failActiveJobf("unknown job type %q; ensure your provisioner daemon is up-to-date", reflect.TypeOf(job.Type).String())
return return
} }
// Ensure the job is still running to output. // Ensure the job is still running to output.
// It's possible the job was canceled. // It's possible the job has failed.
if p.isRunningJob() { if p.isRunningJob() {
p.opts.Logger.Info(context.Background(), "completed job", slog.F("id", job.JobId)) p.opts.Logger.Info(context.Background(), "completed job", slog.F("id", job.JobId))
} }
} }
func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob) { func (p *Server) runProjectImport(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob) {
parameterSchemas, err := p.runProjectImportParse(ctx, provisioner, job) parameterSchemas, err := p.runProjectImportParse(ctx, provisioner, job)
if err != nil { if err != nil {
p.cancelActiveJobf("run parse: %s", err) p.failActiveJobf("run parse: %s", err)
return return
} }
@ -357,7 +366,7 @@ func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sd
ParameterSchemas: parameterSchemas, ParameterSchemas: parameterSchemas,
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("update job: %s", err) p.failActiveJobf("update job: %s", err)
return return
} }
@ -368,7 +377,7 @@ func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sd
for _, parameterSchema := range parameterSchemas { for _, parameterSchema := range parameterSchemas {
_, ok := valueByName[parameterSchema.Name] _, ok := valueByName[parameterSchema.Name]
if !ok { if !ok {
p.cancelActiveJobf("%s: %s", missingParameterErrorText, parameterSchema.Name) p.failActiveJobf("%s: %s", missingParameterErrorText, parameterSchema.Name)
return return
} }
} }
@ -378,7 +387,7 @@ func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sd
WorkspaceTransition: sdkproto.WorkspaceTransition_START, WorkspaceTransition: sdkproto.WorkspaceTransition_START,
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("project import provision for start: %s", err) p.failActiveJobf("project import provision for start: %s", err)
return return
} }
stopResources, err := p.runProjectImportProvision(ctx, provisioner, job, updateResponse.ParameterValues, &sdkproto.Provision_Metadata{ stopResources, err := p.runProjectImportProvision(ctx, provisioner, job, updateResponse.ParameterValues, &sdkproto.Provision_Metadata{
@ -386,7 +395,7 @@ func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sd
WorkspaceTransition: sdkproto.WorkspaceTransition_STOP, WorkspaceTransition: sdkproto.WorkspaceTransition_STOP,
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("project import provision for start: %s", err) p.failActiveJobf("project import provision for start: %s", err)
return return
} }
@ -400,13 +409,13 @@ func (p *provisionerDaemon) runProjectImport(ctx context.Context, provisioner sd
}, },
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("complete job: %s", err) p.failActiveJobf("complete job: %s", err)
return return
} }
} }
// Parses parameter schemas from source. // Parses parameter schemas from source.
func (p *provisionerDaemon) runProjectImportParse(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob) ([]*sdkproto.ParameterSchema, error) { func (p *Server) runProjectImportParse(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob) ([]*sdkproto.ParameterSchema, error) {
stream, err := provisioner.Parse(ctx, &sdkproto.Parse_Request{ stream, err := provisioner.Parse(ctx, &sdkproto.Parse_Request{
Directory: p.opts.WorkDirectory, Directory: p.opts.WorkDirectory,
}) })
@ -453,7 +462,7 @@ func (p *provisionerDaemon) runProjectImportParse(ctx context.Context, provision
// Performs a dry-run provision when importing a project. // Performs a dry-run provision when importing a project.
// This is used to detect resources that would be provisioned // This is used to detect resources that would be provisioned
// for a workspace in various states. // for a workspace in various states.
func (p *provisionerDaemon) runProjectImportProvision(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob, values []*sdkproto.ParameterValue, metadata *sdkproto.Provision_Metadata) ([]*sdkproto.Resource, error) { func (p *Server) runProjectImportProvision(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob, values []*sdkproto.ParameterValue, metadata *sdkproto.Provision_Metadata) ([]*sdkproto.Resource, error) {
stream, err := provisioner.Provision(ctx, &sdkproto.Provision_Request{ stream, err := provisioner.Provision(ctx, &sdkproto.Provision_Request{
Directory: p.opts.WorkDirectory, Directory: p.opts.WorkDirectory,
ParameterValues: values, ParameterValues: values,
@ -504,7 +513,7 @@ func (p *provisionerDaemon) runProjectImportProvision(ctx context.Context, provi
} }
} }
func (p *provisionerDaemon) runWorkspaceProvision(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob) { func (p *Server) runWorkspaceProvision(ctx context.Context, provisioner sdkproto.DRPCProvisionerClient, job *proto.AcquiredJob) {
stream, err := provisioner.Provision(ctx, &sdkproto.Provision_Request{ stream, err := provisioner.Provision(ctx, &sdkproto.Provision_Request{
Directory: p.opts.WorkDirectory, Directory: p.opts.WorkDirectory,
ParameterValues: job.GetWorkspaceProvision().ParameterValues, ParameterValues: job.GetWorkspaceProvision().ParameterValues,
@ -512,7 +521,7 @@ func (p *provisionerDaemon) runWorkspaceProvision(ctx context.Context, provision
State: job.GetWorkspaceProvision().State, State: job.GetWorkspaceProvision().State,
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("provision: %s", err) p.failActiveJobf("provision: %s", err)
return return
} }
defer stream.Close() defer stream.Close()
@ -520,7 +529,7 @@ func (p *provisionerDaemon) runWorkspaceProvision(ctx context.Context, provision
for { for {
msg, err := stream.Recv() msg, err := stream.Recv()
if err != nil { if err != nil {
p.cancelActiveJobf("recv workspace provision: %s", err) p.failActiveJobf("recv workspace provision: %s", err)
return return
} }
switch msgType := msg.Type.(type) { switch msgType := msg.Type.(type) {
@ -541,10 +550,26 @@ func (p *provisionerDaemon) runWorkspaceProvision(ctx context.Context, provision
}}, }},
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("send job update: %s", err) p.failActiveJobf("send job update: %s", err)
return return
} }
case *sdkproto.Provision_Response_Complete: case *sdkproto.Provision_Response_Complete:
if msgType.Complete.Error != "" {
p.opts.Logger.Info(context.Background(), "provision failed; updating state",
slog.F("state_length", len(msgType.Complete.State)),
)
p.failActiveJob(&proto.FailedJob{
Error: msgType.Complete.Error,
Type: &proto.FailedJob_WorkspaceProvision_{
WorkspaceProvision: &proto.FailedJob_WorkspaceProvision{
State: msgType.Complete.State,
},
},
})
return
}
p.opts.Logger.Info(context.Background(), "provision successful; marking job as complete", p.opts.Logger.Info(context.Background(), "provision successful; marking job as complete",
slog.F("resource_count", len(msgType.Complete.Resources)), slog.F("resource_count", len(msgType.Complete.Resources)),
slog.F("resources", msgType.Complete.Resources), slog.F("resources", msgType.Complete.Resources),
@ -563,53 +588,56 @@ func (p *provisionerDaemon) runWorkspaceProvision(ctx context.Context, provision
}, },
}) })
if err != nil { if err != nil {
p.cancelActiveJobf("complete job: %s", err) p.failActiveJobf("complete job: %s", err)
return return
} }
// Return so we stop looping! // Return so we stop looping!
return return
default: default:
p.cancelActiveJobf("invalid message type %q received from provisioner", p.failActiveJobf("invalid message type %q received from provisioner",
reflect.TypeOf(msg.Type).String()) reflect.TypeOf(msg.Type).String())
return return
} }
} }
} }
func (p *provisionerDaemon) cancelActiveJobf(format string, args ...interface{}) { func (p *Server) failActiveJobf(format string, args ...interface{}) {
p.failActiveJob(&proto.FailedJob{
Error: fmt.Sprintf(format, args...),
})
}
func (p *Server) failActiveJob(failedJob *proto.FailedJob) {
p.jobMutex.Lock() p.jobMutex.Lock()
defer p.jobMutex.Unlock() defer p.jobMutex.Unlock()
errMsg := fmt.Sprintf(format, args...)
if !p.isRunningJob() { if !p.isRunningJob() {
if p.isClosed() { if p.isClosed() {
return return
} }
p.opts.Logger.Info(context.Background(), "skipping job cancel; none running", slog.F("error_message", errMsg)) p.opts.Logger.Info(context.Background(), "skipping job fail; none running", slog.F("error_message", failedJob.Error))
return return
} }
if p.jobCancelled.Load() { if p.jobFailed.Load() {
p.opts.Logger.Warn(context.Background(), "job has already been canceled", slog.F("error_messsage", errMsg)) p.opts.Logger.Warn(context.Background(), "job has already been marked as failed", slog.F("error_messsage", failedJob.Error))
return return
} }
p.jobCancelled.Store(true) p.jobFailed.Store(true)
p.jobCancel() p.jobCancel()
p.opts.Logger.Info(context.Background(), "canceling running job", p.opts.Logger.Info(context.Background(), "failing running job",
slog.F("error_message", errMsg), slog.F("error_message", failedJob.Error),
slog.F("job_id", p.jobID), slog.F("job_id", p.jobID),
) )
_, err := p.client.CancelJob(context.Background(), &proto.CancelledJob{ failedJob.JobId = p.jobID
JobId: p.jobID, _, err := p.client.FailJob(context.Background(), failedJob)
Error: fmt.Sprintf("provisioner daemon: %s", errMsg),
})
if err != nil { if err != nil {
p.opts.Logger.Warn(context.Background(), "failed to notify of cancel; job is no longer running", slog.Error(err)) p.opts.Logger.Warn(context.Background(), "failed to notify of error; job is no longer running", slog.Error(err))
return return
} }
p.opts.Logger.Debug(context.Background(), "canceled running job") p.opts.Logger.Debug(context.Background(), "marked running job as failed")
} }
// isClosed returns whether the API is closed or not. // isClosed returns whether the API is closed or not.
func (p *provisionerDaemon) isClosed() bool { func (p *Server) isClosed() bool {
select { select {
case <-p.closed: case <-p.closed:
return true return true
@ -618,13 +646,49 @@ func (p *provisionerDaemon) isClosed() bool {
} }
} }
// Close ends the provisioner. It will mark any running jobs as canceled. // isShutdown returns whether the API is shutdown or not.
func (p *provisionerDaemon) Close() error { func (p *Server) isShutdown() bool {
select {
case <-p.shutdown:
return true
default:
return false
}
}
// Shutdown triggers a graceful exit of each registered provisioner.
// It exits when an active job stops.
func (p *Server) Shutdown(ctx context.Context) error {
p.shutdownMutex.Lock()
defer p.shutdownMutex.Unlock()
if !p.isRunningJob() {
return nil
}
p.opts.Logger.Info(ctx, "attempting graceful shutdown")
close(p.shutdown)
for id, provisioner := range p.opts.Provisioners {
_, err := provisioner.Shutdown(ctx, &sdkproto.Empty{})
if err != nil {
return xerrors.Errorf("shutdown %q: %w", id, err)
}
}
select {
case <-ctx.Done():
p.opts.Logger.Warn(ctx, "graceful shutdown failed", slog.Error(ctx.Err()))
return ctx.Err()
case <-p.jobRunning:
p.opts.Logger.Info(ctx, "gracefully shutdown")
return nil
}
}
// Close ends the provisioner. It will mark any running jobs as failed.
func (p *Server) Close() error {
return p.closeWithError(nil) return p.closeWithError(nil)
} }
// closeWithError closes the provisioner; subsequent reads/writes will return the error err. // closeWithError closes the provisioner; subsequent reads/writes will return the error err.
func (p *provisionerDaemon) closeWithError(err error) error { func (p *Server) closeWithError(err error) error {
p.closeMutex.Lock() p.closeMutex.Lock()
defer p.closeMutex.Unlock() defer p.closeMutex.Unlock()
if p.isClosed() { if p.isClosed() {
@ -637,7 +701,7 @@ func (p *provisionerDaemon) closeWithError(err error) error {
if err != nil { if err != nil {
errMsg = err.Error() errMsg = err.Error()
} }
p.cancelActiveJobf(errMsg) p.failActiveJobf(errMsg)
<-p.jobRunning <-p.jobRunning
p.closeCancel() p.closeCancel()

View File

@ -103,7 +103,7 @@ func TestProvisionerd(t *testing.T) {
}, nil }, nil
}, },
updateJob: noopUpdateJob, updateJob: noopUpdateJob,
cancelJob: func(ctx context.Context, job *proto.CancelledJob) (*proto.Empty, error) { failJob: func(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error) {
close(completeChan) close(completeChan)
return &proto.Empty{}, nil return &proto.Empty{}, nil
}, },
@ -144,7 +144,7 @@ func TestProvisionerd(t *testing.T) {
}, nil }, nil
}, },
updateJob: noopUpdateJob, updateJob: noopUpdateJob,
cancelJob: func(ctx context.Context, job *proto.CancelledJob) (*proto.Empty, error) { failJob: func(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error) {
close(completeChan) close(completeChan)
return &proto.Empty{}, nil return &proto.Empty{}, nil
}, },
@ -179,7 +179,7 @@ func TestProvisionerd(t *testing.T) {
close(completeChan) close(completeChan)
return &proto.UpdateJobResponse{}, nil return &proto.UpdateJobResponse{}, nil
}, },
cancelJob: func(ctx context.Context, job *proto.CancelledJob) (*proto.Empty, error) { failJob: func(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error) {
return &proto.Empty{}, nil return &proto.Empty{}, nil
}, },
}), nil }), nil
@ -362,6 +362,127 @@ func TestProvisionerd(t *testing.T) {
require.True(t, didComplete.Load()) require.True(t, didComplete.Load())
require.NoError(t, closer.Close()) require.NoError(t, closer.Close())
}) })
t.Run("WorkspaceProvisionFailComplete", func(t *testing.T) {
t.Parallel()
var (
didFail atomic.Bool
didAcquireJob atomic.Bool
)
completeChan := make(chan struct{})
closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if didAcquireJob.Load() {
close(completeChan)
return &proto.AcquiredJob{}, nil
}
didAcquireJob.Store(true)
return &proto.AcquiredJob{
JobId: "test",
Provisioner: "someprovisioner",
ProjectSourceArchive: createTar(t, map[string]string{
"test.txt": "content",
}),
Type: &proto.AcquiredJob_WorkspaceProvision_{
WorkspaceProvision: &proto.AcquiredJob_WorkspaceProvision{
Metadata: &sdkproto.Provision_Metadata{},
},
},
}, nil
},
failJob: func(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error) {
didFail.Store(true)
return &proto.Empty{}, nil
},
}), nil
}, provisionerd.Provisioners{
"someprovisioner": createProvisionerClient(t, provisionerTestServer{
provision: func(request *sdkproto.Provision_Request, stream sdkproto.DRPCProvisioner_ProvisionStream) error {
return stream.Send(&sdkproto.Provision_Response{
Type: &sdkproto.Provision_Response_Complete{
Complete: &sdkproto.Provision_Complete{
Error: "some error",
},
},
})
},
}),
})
<-completeChan
require.True(t, didFail.Load())
require.NoError(t, closer.Close())
})
t.Run("Shutdown", func(t *testing.T) {
t.Parallel()
updateChan := make(chan struct{})
completeChan := make(chan struct{})
shutdownCtx, shutdownCtxCancel := context.WithCancel(context.Background())
defer shutdownCtxCancel()
server := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
return &proto.AcquiredJob{
JobId: "test",
Provisioner: "someprovisioner",
ProjectSourceArchive: createTar(t, map[string]string{
"test.txt": "content",
}),
Type: &proto.AcquiredJob_WorkspaceProvision_{
WorkspaceProvision: &proto.AcquiredJob_WorkspaceProvision{
Metadata: &sdkproto.Provision_Metadata{},
},
},
}, nil
},
updateJob: func(ctx context.Context, update *proto.UpdateJobRequest) (*proto.UpdateJobResponse, error) {
if len(update.Logs) > 0 {
// Close on a log so we know when the job is in progress!
close(updateChan)
}
return &proto.UpdateJobResponse{}, nil
},
failJob: func(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error) {
close(completeChan)
return &proto.Empty{}, nil
},
}), nil
}, provisionerd.Provisioners{
"someprovisioner": createProvisionerClient(t, provisionerTestServer{
shutdown: func(_ context.Context, _ *sdkproto.Empty) (*sdkproto.Empty, error) {
shutdownCtxCancel()
return &sdkproto.Empty{}, nil
},
provision: func(request *sdkproto.Provision_Request, stream sdkproto.DRPCProvisioner_ProvisionStream) error {
err := stream.Send(&sdkproto.Provision_Response{
Type: &sdkproto.Provision_Response_Log{
Log: &sdkproto.Log{
Level: sdkproto.LogLevel_DEBUG,
Output: "in progress",
},
},
})
require.NoError(t, err)
<-shutdownCtx.Done()
err = stream.Send(&sdkproto.Provision_Response{
Type: &sdkproto.Provision_Response_Complete{
Complete: &sdkproto.Provision_Complete{
Error: "some error",
},
},
})
require.NoError(t, err)
return nil
},
}),
})
<-updateChan
err := server.Shutdown(context.Background())
require.NoError(t, err)
<-completeChan
require.NoError(t, server.Close())
})
} }
// Creates an in-memory tar of the files provided. // Creates an in-memory tar of the files provided.
@ -385,8 +506,8 @@ func createTar(t *testing.T, files map[string]string) []byte {
} }
// Creates a provisionerd implementation with the provided dialer and provisioners. // Creates a provisionerd implementation with the provided dialer and provisioners.
func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, provisioners provisionerd.Provisioners) io.Closer { func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, provisioners provisionerd.Provisioners) *provisionerd.Server {
closer := provisionerd.New(dialer, &provisionerd.Options{ server := provisionerd.New(dialer, &provisionerd.Options{
Logger: slogtest.Make(t, nil).Named("provisionerd").Leveled(slog.LevelDebug), Logger: slogtest.Make(t, nil).Named("provisionerd").Leveled(slog.LevelDebug),
PollInterval: 50 * time.Millisecond, PollInterval: 50 * time.Millisecond,
UpdateInterval: 50 * time.Millisecond, UpdateInterval: 50 * time.Millisecond,
@ -394,9 +515,9 @@ func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, provisioners p
WorkDirectory: t.TempDir(), WorkDirectory: t.TempDir(),
}) })
t.Cleanup(func() { t.Cleanup(func() {
_ = closer.Close() _ = server.Close()
}) })
return closer return server
} }
// Creates a provisionerd protobuf client that's connected // Creates a provisionerd protobuf client that's connected
@ -440,10 +561,15 @@ func createProvisionerClient(t *testing.T, server provisionerTestServer) sdkprot
} }
type provisionerTestServer struct { type provisionerTestServer struct {
shutdown func(_ context.Context, _ *sdkproto.Empty) (*sdkproto.Empty, error)
parse func(request *sdkproto.Parse_Request, stream sdkproto.DRPCProvisioner_ParseStream) error parse func(request *sdkproto.Parse_Request, stream sdkproto.DRPCProvisioner_ParseStream) error
provision func(request *sdkproto.Provision_Request, stream sdkproto.DRPCProvisioner_ProvisionStream) error provision func(request *sdkproto.Provision_Request, stream sdkproto.DRPCProvisioner_ProvisionStream) error
} }
func (p *provisionerTestServer) Shutdown(ctx context.Context, empty *sdkproto.Empty) (*sdkproto.Empty, error) {
return p.shutdown(ctx, empty)
}
func (p *provisionerTestServer) Parse(request *sdkproto.Parse_Request, stream sdkproto.DRPCProvisioner_ParseStream) error { func (p *provisionerTestServer) Parse(request *sdkproto.Parse_Request, stream sdkproto.DRPCProvisioner_ParseStream) error {
return p.parse(request, stream) return p.parse(request, stream)
} }
@ -457,7 +583,7 @@ func (p *provisionerTestServer) Provision(request *sdkproto.Provision_Request, s
type provisionerDaemonTestServer struct { type provisionerDaemonTestServer struct {
acquireJob func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) acquireJob func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error)
updateJob func(ctx context.Context, update *proto.UpdateJobRequest) (*proto.UpdateJobResponse, error) updateJob func(ctx context.Context, update *proto.UpdateJobRequest) (*proto.UpdateJobResponse, error)
cancelJob func(ctx context.Context, job *proto.CancelledJob) (*proto.Empty, error) failJob func(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error)
completeJob func(ctx context.Context, job *proto.CompletedJob) (*proto.Empty, error) completeJob func(ctx context.Context, job *proto.CompletedJob) (*proto.Empty, error)
} }
@ -469,8 +595,8 @@ func (p *provisionerDaemonTestServer) UpdateJob(ctx context.Context, update *pro
return p.updateJob(ctx, update) return p.updateJob(ctx, update)
} }
func (p *provisionerDaemonTestServer) CancelJob(ctx context.Context, job *proto.CancelledJob) (*proto.Empty, error) { func (p *provisionerDaemonTestServer) FailJob(ctx context.Context, job *proto.FailedJob) (*proto.Empty, error) {
return p.cancelJob(ctx, job) return p.failJob(ctx, job)
} }
func (p *provisionerDaemonTestServer) CompleteJob(ctx context.Context, job *proto.CompletedJob) (*proto.Empty, error) { func (p *provisionerDaemonTestServer) CompleteJob(ctx context.Context, job *proto.CompletedJob) (*proto.Empty, error) {

File diff suppressed because it is too large Load Diff

View File

@ -4,6 +4,9 @@ option go_package = "github.com/coder/coder/provisionersdk/proto";
package provisioner; package provisioner;
// Empty indicates a successful request/response.
message Empty {}
// ParameterSource represents the source location for a parameter to get it's value from. // ParameterSource represents the source location for a parameter to get it's value from.
message ParameterSource { message ParameterSource {
enum Scheme { enum Scheme {
@ -123,7 +126,8 @@ message Provision {
} }
message Complete { message Complete {
bytes state = 1; bytes state = 1;
repeated Resource resources = 2; string error = 2;
repeated Resource resources = 3;
} }
message Response { message Response {
oneof type { oneof type {
@ -134,6 +138,7 @@ message Provision {
} }
service Provisioner { service Provisioner {
rpc Shutdown(Empty) returns (Empty);
rpc Parse(Parse.Request) returns (stream Parse.Response); rpc Parse(Parse.Request) returns (stream Parse.Response);
rpc Provision(Provision.Request) returns (stream Provision.Response); rpc Provision(Provision.Request) returns (stream Provision.Response);
} }

View File

@ -38,6 +38,7 @@ func (drpcEncoding_File_provisionersdk_proto_provisioner_proto) JSONUnmarshal(bu
type DRPCProvisionerClient interface { type DRPCProvisionerClient interface {
DRPCConn() drpc.Conn DRPCConn() drpc.Conn
Shutdown(ctx context.Context, in *Empty) (*Empty, error)
Parse(ctx context.Context, in *Parse_Request) (DRPCProvisioner_ParseClient, error) Parse(ctx context.Context, in *Parse_Request) (DRPCProvisioner_ParseClient, error)
Provision(ctx context.Context, in *Provision_Request) (DRPCProvisioner_ProvisionClient, error) Provision(ctx context.Context, in *Provision_Request) (DRPCProvisioner_ProvisionClient, error)
} }
@ -52,6 +53,15 @@ func NewDRPCProvisionerClient(cc drpc.Conn) DRPCProvisionerClient {
func (c *drpcProvisionerClient) DRPCConn() drpc.Conn { return c.cc } func (c *drpcProvisionerClient) DRPCConn() drpc.Conn { return c.cc }
func (c *drpcProvisionerClient) Shutdown(ctx context.Context, in *Empty) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/provisioner.Provisioner/Shutdown", drpcEncoding_File_provisionersdk_proto_provisioner_proto{}, in, out)
if err != nil {
return nil, err
}
return out, nil
}
func (c *drpcProvisionerClient) Parse(ctx context.Context, in *Parse_Request) (DRPCProvisioner_ParseClient, error) { func (c *drpcProvisionerClient) Parse(ctx context.Context, in *Parse_Request) (DRPCProvisioner_ParseClient, error) {
stream, err := c.cc.NewStream(ctx, "/provisioner.Provisioner/Parse", drpcEncoding_File_provisionersdk_proto_provisioner_proto{}) stream, err := c.cc.NewStream(ctx, "/provisioner.Provisioner/Parse", drpcEncoding_File_provisionersdk_proto_provisioner_proto{})
if err != nil { if err != nil {
@ -125,12 +135,17 @@ func (x *drpcProvisioner_ProvisionClient) RecvMsg(m *Provision_Response) error {
} }
type DRPCProvisionerServer interface { type DRPCProvisionerServer interface {
Shutdown(context.Context, *Empty) (*Empty, error)
Parse(*Parse_Request, DRPCProvisioner_ParseStream) error Parse(*Parse_Request, DRPCProvisioner_ParseStream) error
Provision(*Provision_Request, DRPCProvisioner_ProvisionStream) error Provision(*Provision_Request, DRPCProvisioner_ProvisionStream) error
} }
type DRPCProvisionerUnimplementedServer struct{} type DRPCProvisionerUnimplementedServer struct{}
func (s *DRPCProvisionerUnimplementedServer) Shutdown(context.Context, *Empty) (*Empty, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCProvisionerUnimplementedServer) Parse(*Parse_Request, DRPCProvisioner_ParseStream) error { func (s *DRPCProvisionerUnimplementedServer) Parse(*Parse_Request, DRPCProvisioner_ParseStream) error {
return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
} }
@ -141,11 +156,20 @@ func (s *DRPCProvisionerUnimplementedServer) Provision(*Provision_Request, DRPCP
type DRPCProvisionerDescription struct{} type DRPCProvisionerDescription struct{}
func (DRPCProvisionerDescription) NumMethods() int { return 2 } func (DRPCProvisionerDescription) NumMethods() int { return 3 }
func (DRPCProvisionerDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { func (DRPCProvisionerDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
switch n { switch n {
case 0: case 0:
return "/provisioner.Provisioner/Shutdown", drpcEncoding_File_provisionersdk_proto_provisioner_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCProvisionerServer).
Shutdown(
ctx,
in1.(*Empty),
)
}, DRPCProvisionerServer.Shutdown, true
case 1:
return "/provisioner.Provisioner/Parse", drpcEncoding_File_provisionersdk_proto_provisioner_proto{}, return "/provisioner.Provisioner/Parse", drpcEncoding_File_provisionersdk_proto_provisioner_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return nil, srv.(DRPCProvisionerServer). return nil, srv.(DRPCProvisionerServer).
@ -154,7 +178,7 @@ func (DRPCProvisionerDescription) Method(n int) (string, drpc.Encoding, drpc.Rec
&drpcProvisioner_ParseStream{in2.(drpc.Stream)}, &drpcProvisioner_ParseStream{in2.(drpc.Stream)},
) )
}, DRPCProvisionerServer.Parse, true }, DRPCProvisionerServer.Parse, true
case 1: case 2:
return "/provisioner.Provisioner/Provision", drpcEncoding_File_provisionersdk_proto_provisioner_proto{}, return "/provisioner.Provisioner/Provision", drpcEncoding_File_provisionersdk_proto_provisioner_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return nil, srv.(DRPCProvisionerServer). return nil, srv.(DRPCProvisionerServer).
@ -172,6 +196,22 @@ func DRPCRegisterProvisioner(mux drpc.Mux, impl DRPCProvisionerServer) error {
return mux.Register(impl, DRPCProvisionerDescription{}) return mux.Register(impl, DRPCProvisionerDescription{})
} }
type DRPCProvisioner_ShutdownStream interface {
drpc.Stream
SendAndClose(*Empty) error
}
type drpcProvisioner_ShutdownStream struct {
drpc.Stream
}
func (x *drpcProvisioner_ShutdownStream) SendAndClose(m *Empty) error {
if err := x.MsgSend(m, drpcEncoding_File_provisionersdk_proto_provisioner_proto{}); err != nil {
return err
}
return x.CloseSend()
}
type DRPCProvisioner_ParseStream interface { type DRPCProvisioner_ParseStream interface {
drpc.Stream drpc.Stream
Send(*Parse_Response) error Send(*Parse_Response) error