synced/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2025-07-06 15:41:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Convert all jobs to use a common resource and agent type (#369)

Browse Source 
* ci: Update DataDog GitHub branch to fallback to GITHUB_REF

This was detecting branches, but not our "main" branch before.
Hopefully this fixes it!

* Add basic Terraform Provider

* Rename post files to upload

* Add tests for resources

* Skip instance identity test

* Add tests for ensuring agent get's passed through properly

* Fix linting errors

* Add echo path

* Fix agent authentication

* fix: Convert all jobs to use a common resource and agent type

This enables a consistent API for project import and provisioned resources.
This commit is contained in:
Kyle Carberry
2022-02-28 12:00:52 -06:00
committed by GitHub
parent 2d6804c746
commit bd0293aff9
18 changed files with 632 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
coderd/projectimport.go
View File

@ -21,9 +21,6 @@ type ParameterSchema database.ParameterSchema
// ComputedParameterValue represents a computed parameter value.
type ComputedParameterValue parameter.ComputedValue
// ProjectImportJobResource is a resource created by a project import job.
type ProjectImportJobResource database.ProjectImportJobResource
// CreateProjectImportJobRequest provides options to create a project import job.
type CreateProjectImportJobRequest struct {
StorageMethod database.ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"`
@ -167,7 +164,7 @@ func (api *api) projectImportJobResourcesByID(rw http.ResponseWriter, r *http.Re
})
return
}
resources, err := api.Database.GetProjectImportJobResourcesByJobID(r.Context(), job.ID)
resources, err := api.Database.GetProvisionerJobResourcesByJobID(r.Context(), job.ID)
if errors.Is(err, sql.ErrNoRows) {
err = nil
}
@ -178,7 +175,7 @@ func (api *api) projectImportJobResourcesByID(rw http.ResponseWriter, r *http.Re
return
}
if resources == nil {
resources = []database.ProjectImportJobResource{}
resources = []database.ProvisionerJobResource{}
}
render.Status(r, http.StatusOK)
render.JSON(rw, r, resources)

87
coderd/provisionerdaemons.go
View File

@ -15,6 +15,7 @@ import (
"github.com/google/uuid"
"github.com/hashicorp/yamux"
"github.com/moby/moby/pkg/namesgenerator"
"github.com/tabbed/pqtype"
"golang.org/x/xerrors"
"nhooyr.io/websocket"
"storj.io/drpc/drpcmux"
@ -453,14 +454,8 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
slog.F("resource_name", resource.Name),
slog.F("resource_type", resource.Type),
slog.F("transition", transition))
_, err = server.Database.InsertProjectImportJobResource(ctx, database.InsertProjectImportJobResourceParams{
ID: uuid.New(),
CreatedAt: database.Now(),
JobID: jobID,
Transition: transition,
Type: resource.Type,
Name: resource.Name,
})
err = insertProvisionerJobResource(ctx, server.Database, jobID, transition, resource)
if err != nil {
return nil, xerrors.Errorf("insert resource: %w", err)
}
@ -516,26 +511,9 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
}
// This could be a bulk insert to improve performance.
for _, protoResource := range jobType.WorkspaceProvision.Resources {
var instanceID sql.NullString
if protoResource.Agent != nil && protoResource.Agent.GetGoogleInstanceIdentity() != nil {
instanceID = sql.NullString{
String: protoResource.Agent.GetGoogleInstanceIdentity().InstanceId,
Valid: true,
}
}
_, err = db.InsertWorkspaceResource(ctx, database.InsertWorkspaceResourceParams{
ID: uuid.New(),
CreatedAt: database.Now(),
WorkspaceHistoryID: input.WorkspaceHistoryID,
Type: protoResource.Type,
Name: protoResource.Name,
InstanceID: instanceID,
// TODO: Generate this at the variable validation phase.
// Set the value in `default_source`, and disallow overwrite.
WorkspaceAgentToken: uuid.NewString(),
})
err = insertProvisionerJobResource(ctx, db, job.ID, workspaceHistory.Transition, protoResource)
if err != nil {
return xerrors.Errorf("insert workspace resource %q: %w", protoResource.Name, err)
return xerrors.Errorf("insert provisioner job: %w", err)
}
}
return nil
@ -551,6 +529,61 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
return &proto.Empty{}, nil
}
func insertProvisionerJobResource(ctx context.Context, db database.Store, jobID uuid.UUID, transition database.WorkspaceTransition, protoResource *sdkproto.Resource) error {
resource, err := db.InsertProvisionerJobResource(ctx, database.InsertProvisionerJobResourceParams{
ID: uuid.New(),
CreatedAt: database.Now(),
JobID: jobID,
Transition: transition,
Type: protoResource.Type,
Name: protoResource.Name,
AgentID: uuid.NullUUID{
UUID: uuid.New(),
Valid: protoResource.Agent != nil,
},
})
if err != nil {
return xerrors.Errorf("insert provisioner job resource %q: %w", protoResource.Name, err)
}
if resource.AgentID.Valid {
var instanceID sql.NullString
if protoResource.Agent.GetGoogleInstanceIdentity() != nil {
instanceID = sql.NullString{
String: protoResource.Agent.GetGoogleInstanceIdentity().InstanceId,
Valid: true,
}
}
var env pqtype.NullRawMessage
if protoResource.Agent.Env != nil {
data, err := json.Marshal(protoResource.Agent.Env)
if err != nil {
return xerrors.Errorf("marshal env: %w", err)
}
env = pqtype.NullRawMessage{
RawMessage: data,
Valid: true,
}
}
_, err := db.InsertProvisionerJobAgent(ctx, database.InsertProvisionerJobAgentParams{
ID: resource.AgentID.UUID,
CreatedAt: database.Now(),
ResourceID: resource.ID,
AuthToken: uuid.New(),
AuthInstanceID: instanceID,
EnvironmentVariables: env,
StartupScript: sql.NullString{
String: protoResource.Agent.StartupScript,
Valid: protoResource.Agent.StartupScript != "",
},
})
if err != nil {
return xerrors.Errorf("insert agent: %w", err)
}
}
return nil
}
func convertValidationTypeSystem(typeSystem sdkproto.ParameterSchema_TypeSystem) (database.ParameterTypeSystem, error) {
switch typeSystem {
case sdkproto.ParameterSchema_None:

21
coderd/provisionerjobs.go
View File

@ -50,13 +50,32 @@ type ProvisionerJob struct {
// ProvisionerJobLog represents a single log from a provisioner job.
type ProvisionerJobLog struct {
ID uuid.UUID
ID uuid.UUID `json:"id"`
CreatedAt time.Time `json:"created_at"`
Source database.LogSource `json:"log_source"`
Level database.LogLevel `json:"log_level"`
Output string `json:"output"`
}
type ProvisionerJobResource struct {
ID uuid.UUID `json:"id"`
CreatedAt time.Time `json:"created_at"`
JobID uuid.UUID `json:"job_id"`
Transition database.WorkspaceTransition `json:"workspace_transition"`
Type string `json:"type"`
Name string `json:"name"`
}
type ProvisionerJobAgent struct {
ID uuid.UUID `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ResourceID uuid.UUID `json:"resource_id"`
InstanceID string `json:"instance_id,omitempty"`
EnvironmentVariables map[string]string `json:"environment_variables"`
StartupScript string `json:"startup_script,omitempty"`
}
func (*api) provisionerJobByID(rw http.ResponseWriter, r *http.Request) {
job := httpmw.ProvisionerJobParam(r)
render.Status(r, http.StatusOK)

42
coderd/workspaceagent.go
View File

@ -2,12 +2,14 @@ package coderd
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/go-chi/render"
"github.com/coder/coder/database"
"github.com/coder/coder/httpapi"
"github.com/mitchellh/mapstructure"
@ -54,14 +56,48 @@ func (api *api) postAuthenticateWorkspaceAgentUsingGoogleInstanceIdentity(rw htt
})
return
}
resource, err := api.Database.GetWorkspaceResourceByInstanceID(r.Context(), claims.Google.ComputeEngine.InstanceID)
agent, err := api.Database.GetProvisionerJobAgentByInstanceID(r.Context(), claims.Google.ComputeEngine.InstanceID)
if errors.Is(err, sql.ErrNoRows) {
httpapi.Write(rw, http.StatusNotFound, httpapi.Response{
Message: fmt.Sprintf("instance with id %q not found", claims.Google.ComputeEngine.InstanceID),
})
return
}
resourceHistory, err := api.Database.GetWorkspaceHistoryByID(r.Context(), resource.WorkspaceHistoryID)
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get provisioner job agent: %s", err),
})
return
}
resource, err := api.Database.GetProvisionerJobResourceByID(r.Context(), agent.ResourceID)
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get provisioner job resource: %s", err),
})
return
}
job, err := api.Database.GetProvisionerJobByID(r.Context(), resource.JobID)
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get provisioner job: %s", err),
})
return
}
if job.Type != database.ProvisionerJobTypeWorkspaceProvision {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("%q jobs cannot be authenticated", job.Type),
})
return
}
var jobData workspaceProvisionJob
err = json.Unmarshal(job.Input, &jobData)
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("extract job data: %s", err),
})
return
}
resourceHistory, err := api.Database.GetWorkspaceHistoryByID(r.Context(), jobData.WorkspaceHistoryID)
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get workspace history: %s", err),
@ -86,6 +122,6 @@ func (api *api) postAuthenticateWorkspaceAgentUsingGoogleInstanceIdentity(rw htt
}
render.Status(r, http.StatusOK)
render.JSON(rw, r, WorkspaceAgentAuthenticateResponse{
SessionToken: resource.WorkspaceAgentToken,
SessionToken: agent.AuthToken.String(),
})
}