feat: add provisioner job hang detector (#7927)

This commit is contained in:
Dean Sheather
2023-06-25 23:17:00 +10:00
committed by GitHub
parent 3671846b1b
commit 98a5ae7f48
28 changed files with 1414 additions and 54 deletions

View File

@ -1527,8 +1527,6 @@ SELECT pg_advisory_xact_lock($1)
//
// This must be called from within a transaction. The lock will be automatically
// released when the transaction ends.
//
// Use database.LockID() to generate a unique lock ID from a string.
func (q *sqlQuerier) AcquireLock(ctx context.Context, pgAdvisoryXactLock int64) error {
_, err := q.db.ExecContext(ctx, acquireLock, pgAdvisoryXactLock)
return err
@ -1542,8 +1540,6 @@ SELECT pg_try_advisory_xact_lock($1)
//
// This must be called from within a transaction. The lock will be automatically
// released when the transaction ends.
//
// Use database.LockID() to generate a unique lock ID from a string.
func (q *sqlQuerier) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) {
row := q.db.QueryRowContext(ctx, tryAcquireLock, pgTryAdvisoryXactLock)
var pg_try_advisory_xact_lock bool
@ -2201,6 +2197,59 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi
return i, err
}
const getHungProvisionerJobs = `-- name: GetHungProvisionerJobs :many
SELECT
id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata
FROM
provisioner_jobs
WHERE
updated_at < $1
AND started_at IS NOT NULL
AND completed_at IS NULL
`
func (q *sqlQuerier) GetHungProvisionerJobs(ctx context.Context, updatedAt time.Time) ([]ProvisionerJob, error) {
rows, err := q.db.QueryContext(ctx, getHungProvisionerJobs, updatedAt)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ProvisionerJob
for rows.Next() {
var i ProvisionerJob
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.StartedAt,
&i.CanceledAt,
&i.CompletedAt,
&i.Error,
&i.OrganizationID,
&i.InitiatorID,
&i.Provisioner,
&i.StorageMethod,
&i.Type,
&i.Input,
&i.WorkerID,
&i.FileID,
&i.Tags,
&i.ErrorCode,
&i.TraceMetadata,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one
SELECT
id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata