mirror of
https://github.com/coder/coder.git
synced 2025-07-03 16:13:58 +00:00
feat: add startup script logs to the ui (#6558)
* Add startup script logs to the database * Add coderd endpoints for startup script logs * Push startup script logs from agent * Pull startup script logs on frontend * Rename queries * Add constraint * Start creating log sending loop * Add log sending to the agent * Add tests for streaming logs * Shorten notify channel name * Add FE * Improve bulk log performance * Finish UI display * Fix startup log visibility * Add warning for overflow * Fix agent queue logs overflow * Display staartup logs in a virtual DOM for performance * Fix agent queue with loads of logs * Fix authorize test * Remove faulty test * Fix startup and shutdown reporting error * Fix gen * Fix comments * Periodically purge old database entries * Add test fixture for migration * Add Storybook * Check if there are logs when displaying features * Fix startup component overflow gap * Fix startup log wrapping --------- Co-authored-by: Asher <ash@coder.com>
This commit is contained in:
Kyle Carberry
GitHub
@ -2286,7 +2286,7 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getProvisionerLogsByIDBetween = `-- name: GetProvisionerLogsByIDBetween :many
|
||||
const getProvisionerLogsAfterID = `-- name: GetProvisionerLogsAfterID :many
|
||||
SELECT
|
||||
job_id, created_at, source, level, stage, output, id
|
||||
FROM
|
||||
@ -2295,18 +2295,16 @@ WHERE
|
||||
job_id = $1
|
||||
AND (
|
||||
id > $2
|
||||
OR id < $3
|
||||
) ORDER BY id ASC
|
||||
`
|
||||
|
||||
type GetProvisionerLogsByIDBetweenParams struct {
|
||||
JobID uuid.UUID `db:"job_id" json:"job_id"`
|
||||
CreatedAfter int64 `db:"created_after" json:"created_after"`
|
||||
CreatedBefore int64 `db:"created_before" json:"created_before"`
|
||||
type GetProvisionerLogsAfterIDParams struct {
|
||||
JobID uuid.UUID `db:"job_id" json:"job_id"`
|
||||
CreatedAfter int64 `db:"created_after" json:"created_after"`
|
||||
}
|
||||
|
||||
func (q *sqlQuerier) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getProvisionerLogsByIDBetween, arg.JobID, arg.CreatedAfter, arg.CreatedBefore)
|
||||
func (q *sqlQuerier) GetProvisionerLogsAfterID(ctx context.Context, arg GetProvisionerLogsAfterIDParams) ([]ProvisionerJobLog, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getProvisionerLogsAfterID, arg.JobID, arg.CreatedAfter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -5072,9 +5070,22 @@ func (q *sqlQuerier) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusP
|
||||
return i, err
|
||||
}
|
||||
|
||||
const deleteOldWorkspaceAgentStartupLogs = `-- name: DeleteOldWorkspaceAgentStartupLogs :exec
|
||||
DELETE FROM workspace_agent_startup_logs WHERE agent_id IN
|
||||
(SELECT id FROM workspace_agents WHERE last_connected_at IS NOT NULL
|
||||
AND last_connected_at < NOW() - INTERVAL '7 day')
|
||||
`
|
||||
|
||||
// If an agent hasn't connected in the last 7 days, we purge it's logs.
|
||||
// Logs can take up a lot of space, so it's important we clean up frequently.
|
||||
func (q *sqlQuerier) DeleteOldWorkspaceAgentStartupLogs(ctx context.Context) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteOldWorkspaceAgentStartupLogs)
|
||||
return err
|
||||
}
|
||||
|
||||
const getWorkspaceAgentByAuthToken = `-- name: GetWorkspaceAgentByAuthToken :one
|
||||
SELECT
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds, startup_logs_length, startup_logs_overflowed
|
||||
FROM
|
||||
workspace_agents
|
||||
WHERE
|
||||
@ -5115,13 +5126,15 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken
|
||||
&i.ExpandedDirectory,
|
||||
&i.ShutdownScript,
|
||||
&i.ShutdownScriptTimeoutSeconds,
|
||||
&i.StartupLogsLength,
|
||||
&i.StartupLogsOverflowed,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getWorkspaceAgentByID = `-- name: GetWorkspaceAgentByID :one
|
||||
SELECT
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds, startup_logs_length, startup_logs_overflowed
|
||||
FROM
|
||||
workspace_agents
|
||||
WHERE
|
||||
@ -5160,13 +5173,15 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W
|
||||
&i.ExpandedDirectory,
|
||||
&i.ShutdownScript,
|
||||
&i.ShutdownScriptTimeoutSeconds,
|
||||
&i.StartupLogsLength,
|
||||
&i.StartupLogsOverflowed,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getWorkspaceAgentByInstanceID = `-- name: GetWorkspaceAgentByInstanceID :one
|
||||
SELECT
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds, startup_logs_length, startup_logs_overflowed
|
||||
FROM
|
||||
workspace_agents
|
||||
WHERE
|
||||
@ -5207,13 +5222,60 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst
|
||||
&i.ExpandedDirectory,
|
||||
&i.ShutdownScript,
|
||||
&i.ShutdownScriptTimeoutSeconds,
|
||||
&i.StartupLogsLength,
|
||||
&i.StartupLogsOverflowed,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getWorkspaceAgentStartupLogsAfter = `-- name: GetWorkspaceAgentStartupLogsAfter :many
|
||||
SELECT
|
||||
agent_id, created_at, output, id
|
||||
FROM
|
||||
workspace_agent_startup_logs
|
||||
WHERE
|
||||
agent_id = $1
|
||||
AND (
|
||||
id > $2
|
||||
) ORDER BY id ASC
|
||||
`
|
||||
|
||||
type GetWorkspaceAgentStartupLogsAfterParams struct {
|
||||
AgentID uuid.UUID `db:"agent_id" json:"agent_id"`
|
||||
CreatedAfter int64 `db:"created_after" json:"created_after"`
|
||||
}
|
||||
|
||||
func (q *sqlQuerier) GetWorkspaceAgentStartupLogsAfter(ctx context.Context, arg GetWorkspaceAgentStartupLogsAfterParams) ([]WorkspaceAgentStartupLog, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getWorkspaceAgentStartupLogsAfter, arg.AgentID, arg.CreatedAfter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []WorkspaceAgentStartupLog
|
||||
for rows.Next() {
|
||||
var i WorkspaceAgentStartupLog
|
||||
if err := rows.Scan(
|
||||
&i.AgentID,
|
||||
&i.CreatedAt,
|
||||
&i.Output,
|
||||
&i.ID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getWorkspaceAgentsByResourceIDs = `-- name: GetWorkspaceAgentsByResourceIDs :many
|
||||
SELECT
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds
|
||||
id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds, startup_logs_length, startup_logs_overflowed
|
||||
FROM
|
||||
workspace_agents
|
||||
WHERE
|
||||
@ -5258,6 +5320,8 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids []
|
||||
&i.ExpandedDirectory,
|
||||
&i.ShutdownScript,
|
||||
&i.ShutdownScriptTimeoutSeconds,
|
||||
&i.StartupLogsLength,
|
||||
&i.StartupLogsOverflowed,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -5273,7 +5337,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids []
|
||||
}
|
||||
|
||||
const getWorkspaceAgentsCreatedAfter = `-- name: GetWorkspaceAgentsCreatedAfter :many
|
||||
SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds FROM workspace_agents WHERE created_at > $1
|
||||
SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds, startup_logs_length, startup_logs_overflowed FROM workspace_agents WHERE created_at > $1
|
||||
`
|
||||
|
||||
func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) {
|
||||
@ -5314,6 +5378,8 @@ func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, created
|
||||
&i.ExpandedDirectory,
|
||||
&i.ShutdownScript,
|
||||
&i.ShutdownScriptTimeoutSeconds,
|
||||
&i.StartupLogsLength,
|
||||
&i.StartupLogsOverflowed,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -5354,7 +5420,7 @@ INSERT INTO
|
||||
shutdown_script_timeout_seconds
|
||||
)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds
|
||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, version, last_connected_replica_id, connection_timeout_seconds, troubleshooting_url, motd_file, lifecycle_state, login_before_ready, startup_script_timeout_seconds, expanded_directory, shutdown_script, shutdown_script_timeout_seconds, startup_logs_length, startup_logs_overflowed
|
||||
`
|
||||
|
||||
type InsertWorkspaceAgentParams struct {
|
||||
@ -5435,10 +5501,66 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa
|
||||
&i.ExpandedDirectory,
|
||||
&i.ShutdownScript,
|
||||
&i.ShutdownScriptTimeoutSeconds,
|
||||
&i.StartupLogsLength,
|
||||
&i.StartupLogsOverflowed,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const insertWorkspaceAgentStartupLogs = `-- name: InsertWorkspaceAgentStartupLogs :many
|
||||
WITH new_length AS (
|
||||
UPDATE workspace_agents SET
|
||||
startup_logs_length = startup_logs_length + $4 WHERE workspace_agents.id = $1
|
||||
)
|
||||
INSERT INTO
|
||||
workspace_agent_startup_logs
|
||||
SELECT
|
||||
$1 :: uuid AS agent_id,
|
||||
unnest($2 :: timestamptz [ ]) AS created_at,
|
||||
unnest($3 :: VARCHAR(1024) [ ]) AS output
|
||||
RETURNING workspace_agent_startup_logs.agent_id, workspace_agent_startup_logs.created_at, workspace_agent_startup_logs.output, workspace_agent_startup_logs.id
|
||||
`
|
||||
|
||||
type InsertWorkspaceAgentStartupLogsParams struct {
|
||||
AgentID uuid.UUID `db:"agent_id" json:"agent_id"`
|
||||
CreatedAt []time.Time `db:"created_at" json:"created_at"`
|
||||
Output []string `db:"output" json:"output"`
|
||||
OutputLength int32 `db:"output_length" json:"output_length"`
|
||||
}
|
||||
|
||||
func (q *sqlQuerier) InsertWorkspaceAgentStartupLogs(ctx context.Context, arg InsertWorkspaceAgentStartupLogsParams) ([]WorkspaceAgentStartupLog, error) {
|
||||
rows, err := q.db.QueryContext(ctx, insertWorkspaceAgentStartupLogs,
|
||||
arg.AgentID,
|
||||
pq.Array(arg.CreatedAt),
|
||||
pq.Array(arg.Output),
|
||||
arg.OutputLength,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []WorkspaceAgentStartupLog
|
||||
for rows.Next() {
|
||||
var i WorkspaceAgentStartupLog
|
||||
if err := rows.Scan(
|
||||
&i.AgentID,
|
||||
&i.CreatedAt,
|
||||
&i.Output,
|
||||
&i.ID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const updateWorkspaceAgentConnectionByID = `-- name: UpdateWorkspaceAgentConnectionByID :exec
|
||||
UPDATE
|
||||
workspace_agents
|
||||
@ -5513,6 +5635,25 @@ func (q *sqlQuerier) UpdateWorkspaceAgentStartupByID(ctx context.Context, arg Up
|
||||
return err
|
||||
}
|
||||
|
||||
const updateWorkspaceAgentStartupLogOverflowByID = `-- name: UpdateWorkspaceAgentStartupLogOverflowByID :exec
|
||||
UPDATE
|
||||
workspace_agents
|
||||
SET
|
||||
startup_logs_overflowed = $2
|
||||
WHERE
|
||||
id = $1
|
||||
`
|
||||
|
||||
type UpdateWorkspaceAgentStartupLogOverflowByIDParams struct {
|
||||
ID uuid.UUID `db:"id" json:"id"`
|
||||
StartupLogsOverflowed bool `db:"startup_logs_overflowed" json:"startup_logs_overflowed"`
|
||||
}
|
||||
|
||||
func (q *sqlQuerier) UpdateWorkspaceAgentStartupLogOverflowByID(ctx context.Context, arg UpdateWorkspaceAgentStartupLogOverflowByIDParams) error {
|
||||
_, err := q.db.ExecContext(ctx, updateWorkspaceAgentStartupLogOverflowByID, arg.ID, arg.StartupLogsOverflowed)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteOldWorkspaceAgentStats = `-- name: DeleteOldWorkspaceAgentStats :exec
|
||||
DELETE FROM workspace_agent_stats WHERE created_at < NOW() - INTERVAL '30 days'
|
||||
`
|
||||
|
||||
Reference in New Issue
Block a user