feat(agent/agentcontainers): add ContainerEnvInfoer (#16623)

This PR adds an alternative implementation of EnvInfo
(https://github.com/coder/coder/pull/16603) that reads information from
a running container.

---------

Co-authored-by: Mathias Fredriksson <mafredri@gmail.com>
This commit is contained in:
Cian Johnston
2025-02-24 15:05:15 +00:00
committed by GitHub
parent ac88c9ba17
commit 304007b5ea
3 changed files with 462 additions and 27 deletions

View File

@ -144,6 +144,8 @@ type Lister interface {
// NoopLister is a Lister interface that never returns any containers.
type NoopLister struct{}
var _ Lister = NoopLister{}
func (NoopLister) List(_ context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
return codersdk.WorkspaceAgentListContainersResponse{}, nil
}

View File

@ -6,6 +6,9 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"os/user"
"slices"
"sort"
"strconv"
"strings"
@ -31,6 +34,210 @@ func NewDocker(execer agentexec.Execer) Lister {
}
}
// DockerEnvInfoer is an implementation of agentssh.EnvInfoer that returns
// information about a container.
type DockerEnvInfoer struct {
container string
user *user.User
userShell string
env []string
}
// EnvInfo returns information about the environment of a container.
func EnvInfo(ctx context.Context, execer agentexec.Execer, container, containerUser string) (*DockerEnvInfoer, error) {
var dei DockerEnvInfoer
dei.container = container
if containerUser == "" {
// Get the "default" user of the container if no user is specified.
// TODO: handle different container runtimes.
cmd, args := wrapDockerExec(container, "", "whoami")
stdout, stderr, err := run(ctx, execer, cmd, args...)
if err != nil {
return nil, xerrors.Errorf("get container user: run whoami: %w: %s", err, stderr)
}
if len(stdout) == 0 {
return nil, xerrors.Errorf("get container user: run whoami: empty output")
}
containerUser = stdout
}
// Now that we know the username, get the required info from the container.
// We can't assume the presence of `getent` so we'll just have to sniff /etc/passwd.
cmd, args := wrapDockerExec(container, containerUser, "cat", "/etc/passwd")
stdout, stderr, err := run(ctx, execer, cmd, args...)
if err != nil {
return nil, xerrors.Errorf("get container user: read /etc/passwd: %w: %q", err, stderr)
}
scanner := bufio.NewScanner(strings.NewReader(stdout))
var foundLine string
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if !strings.HasPrefix(line, containerUser+":") {
continue
}
foundLine = line
break
}
if err := scanner.Err(); err != nil {
return nil, xerrors.Errorf("get container user: scan /etc/passwd: %w", err)
}
if foundLine == "" {
return nil, xerrors.Errorf("get container user: no matching entry for %q found in /etc/passwd", containerUser)
}
// Parse the output of /etc/passwd. It looks like this:
// postgres:x:999:999::/var/lib/postgresql:/bin/bash
passwdFields := strings.Split(foundLine, ":")
if len(passwdFields) != 7 {
return nil, xerrors.Errorf("get container user: invalid line in /etc/passwd: %q", foundLine)
}
// The fifth entry in /etc/passwd contains GECOS information, which is a
// comma-separated list of fields. The first field is the user's full name.
gecos := strings.Split(passwdFields[4], ",")
fullName := ""
if len(gecos) > 1 {
fullName = gecos[0]
}
dei.user = &user.User{
Gid: passwdFields[3],
HomeDir: passwdFields[5],
Name: fullName,
Uid: passwdFields[2],
Username: containerUser,
}
dei.userShell = passwdFields[6]
// We need to inspect the container labels for remoteEnv and append these to
// the resulting docker exec command.
// ref: https://code.visualstudio.com/docs/devcontainers/attach-container
env, err := devcontainerEnv(ctx, execer, container)
if err != nil { // best effort.
return nil, xerrors.Errorf("read devcontainer remoteEnv: %w", err)
}
dei.env = env
return &dei, nil
}
func (dei *DockerEnvInfoer) CurrentUser() (*user.User, error) {
// Clone the user so that the caller can't modify it
u := *dei.user
return &u, nil
}
func (*DockerEnvInfoer) Environ() []string {
// Return a clone of the environment so that the caller can't modify it
return os.Environ()
}
func (*DockerEnvInfoer) UserHomeDir() (string, error) {
// We default the working directory of the command to the user's home
// directory. Since this came from inside the container, we cannot guarantee
// that this exists on the host. Return the "real" home directory of the user
// instead.
return os.UserHomeDir()
}
func (dei *DockerEnvInfoer) UserShell(string) (string, error) {
return dei.userShell, nil
}
func (dei *DockerEnvInfoer) ModifyCommand(cmd string, args ...string) (string, []string) {
// Wrap the command with `docker exec` and run it as the container user.
// There is some additional munging here regarding the container user and environment.
dockerArgs := []string{
"exec",
// The assumption is that this command will be a shell command, so allocate a PTY.
"--interactive",
"--tty",
// Run the command as the user in the container.
"--user",
dei.user.Username,
// Set the working directory to the user's home directory as a sane default.
"--workdir",
dei.user.HomeDir,
}
// Append the environment variables from the container.
for _, e := range dei.env {
dockerArgs = append(dockerArgs, "--env", e)
}
// Append the container name and the command.
dockerArgs = append(dockerArgs, dei.container, cmd)
return "docker", append(dockerArgs, args...)
}
// devcontainerEnv is a helper function that inspects the container labels to
// find the required environment variables for running a command in the container.
func devcontainerEnv(ctx context.Context, execer agentexec.Execer, container string) ([]string, error) {
ins, stderr, err := runDockerInspect(ctx, execer, container)
if err != nil {
return nil, xerrors.Errorf("inspect container: %w: %q", err, stderr)
}
if len(ins) != 1 {
return nil, xerrors.Errorf("inspect container: expected 1 container, got %d", len(ins))
}
in := ins[0]
if in.Config.Labels == nil {
return nil, nil
}
// We want to look for the devcontainer metadata, which is in the
// value of the label `devcontainer.metadata`.
rawMeta, ok := in.Config.Labels["devcontainer.metadata"]
if !ok {
return nil, nil
}
meta := struct {
RemoteEnv map[string]string `json:"remoteEnv"`
}{}
if err := json.Unmarshal([]byte(rawMeta), &meta); err != nil {
return nil, xerrors.Errorf("unmarshal devcontainer.metadata: %w", err)
}
// The environment variables are stored in the `remoteEnv` key.
env := make([]string, 0, len(meta.RemoteEnv))
for k, v := range meta.RemoteEnv {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
slices.Sort(env)
return env, nil
}
// wrapDockerExec is a helper function that wraps the given command and arguments
// with a docker exec command that runs as the given user in the given
// container. This is used to fetch information about a container prior to
// running the actual command.
func wrapDockerExec(containerName, userName, cmd string, args ...string) (string, []string) {
dockerArgs := []string{"exec", "--interactive"}
if userName != "" {
dockerArgs = append(dockerArgs, "--user", userName)
}
dockerArgs = append(dockerArgs, containerName, cmd)
return "docker", append(dockerArgs, args...)
}
// Helper function to run a command and return its stdout and stderr.
// We want to differentiate stdout and stderr instead of using CombinedOutput.
// We also want to differentiate between a command running successfully with
// output to stderr and a non-zero exit code.
func run(ctx context.Context, execer agentexec.Execer, cmd string, args ...string) (stdout, stderr string, err error) {
var stdoutBuf, stderrBuf strings.Builder
execCmd := execer.CommandContext(ctx, cmd, args...)
execCmd.Stdout = &stdoutBuf
execCmd.Stderr = &stderrBuf
err = execCmd.Run()
stdout = strings.TrimSpace(stdoutBuf.String())
stderr = strings.TrimSpace(stderrBuf.String())
return stdout, stderr, err
}
func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) {
var stdoutBuf, stderrBuf bytes.Buffer
// List all container IDs, one per line, with no truncation
@ -66,30 +273,16 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
}
// now we can get the detailed information for each container
// Run `docker inspect` on each container ID
stdoutBuf.Reset()
stderrBuf.Reset()
// nolint: gosec // We are not executing user input, these IDs come from
// `docker ps`.
cmd = dcl.execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...)
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
if err := cmd.Run(); err != nil {
return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w: %s", err, strings.TrimSpace(stderrBuf.String()))
}
dockerInspectStderr := strings.TrimSpace(stderrBuf.String())
// Run `docker inspect` on each container ID.
// NOTE: There is an unavoidable potential race condition where a
// container is removed between `docker ps` and `docker inspect`.
// In this case, stderr will contain an error message but stdout
// will still contain valid JSON. We will just end up missing
// information about the removed container. We could potentially
// log this error, but I'm not sure it's worth it.
ins := make([]dockerInspect, 0, len(ids))
if err := json.NewDecoder(&stdoutBuf).Decode(&ins); err != nil {
// However, if we just get invalid JSON, we should absolutely return an error.
return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("decode docker inspect output: %w", err)
ins, dockerInspectStderr, err := runDockerInspect(ctx, dcl.execer, ids...)
if err != nil {
return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w", err)
}
res := codersdk.WorkspaceAgentListContainersResponse{
@ -111,6 +304,28 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi
return res, nil
}
// runDockerInspect is a helper function that runs `docker inspect` on the given
// container IDs and returns the parsed output.
// The stderr output is also returned for logging purposes.
func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) ([]dockerInspect, string, error) {
var stdoutBuf, stderrBuf bytes.Buffer
cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...)
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
err := cmd.Run()
stderr := strings.TrimSpace(stderrBuf.String())
if err != nil {
return nil, stderr, err
}
var ins []dockerInspect
if err := json.NewDecoder(&stdoutBuf).Decode(&ins); err != nil {
return nil, stderr, xerrors.Errorf("decode docker inspect output: %w", err)
}
return ins, stderr, nil
}
// To avoid a direct dependency on the Docker API, we use the docker CLI
// to fetch information about containers.
type dockerInspect struct {

View File

@ -3,34 +3,38 @@ package agentcontainers
import (
"fmt"
"os"
"slices"
"strconv"
"strings"
"testing"
"time"
"go.uber.org/mock/gomock"
"github.com/google/uuid"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/coder/coder/v2/agent/agentcontainers/acmock"
"github.com/coder/coder/v2/agent/agentexec"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/pty"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
// TestDockerCLIContainerLister tests the happy path of the
// dockerCLIContainerLister.List method. It starts a container with a known
// TestIntegrationDocker tests agentcontainers functionality using a real
// Docker container. It starts a container with a known
// label, lists the containers, and verifies that the expected container is
// returned. The container is deleted after the test is complete.
// returned. It also executes a sample command inside the container.
// The container is deleted after the test is complete.
// As this test creates containers, it is skipped by default.
// It can be run manually as follows:
//
// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerCLIContainerLister
func TestDockerCLIContainerLister(t *testing.T) {
func TestIntegrationDocker(t *testing.T) {
t.Parallel()
if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
@ -47,7 +51,10 @@ func TestDockerCLIContainerLister(t *testing.T) {
Repository: "busybox",
Tag: "latest",
Cmd: []string{"sleep", "infnity"},
Labels: map[string]string{"com.coder.test": testLabelValue},
Labels: map[string]string{
"com.coder.test": testLabelValue,
"devcontainer.metadata": `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`,
},
Mounts: []string{testTempDir + ":" + testTempDir},
ExposedPorts: []string{fmt.Sprintf("%d/tcp", testRandPort)},
PortBindings: map[docker.Port][]docker.PortBinding{
@ -68,6 +75,11 @@ func TestDockerCLIContainerLister(t *testing.T) {
assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
t.Logf("Purged container %q", ct.Container.Name)
})
// Wait for container to start
require.Eventually(t, func() bool {
ct, ok := pool.ContainerByName(ct.Container.Name)
return ok && ct.Container.State.Running
}, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time")
dcl := NewDocker(agentexec.DefaultExecer)
ctx := testutil.Context(t, testutil.WaitShort)
@ -93,12 +105,93 @@ func TestDockerCLIContainerLister(t *testing.T) {
if assert.Len(t, foundContainer.Volumes, 1) {
assert.Equal(t, testTempDir, foundContainer.Volumes[testTempDir])
}
// Test that EnvInfo is able to correctly modify a command to be
// executed inside the container.
dei, err := EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, "")
require.NoError(t, err, "Expected no error from DockerEnvInfo()")
ptyWrappedCmd, ptyWrappedArgs := dei.ModifyCommand("/bin/sh", "--norc")
ptyCmd, ptyPs, err := pty.Start(agentexec.DefaultExecer.PTYCommandContext(ctx, ptyWrappedCmd, ptyWrappedArgs...))
require.NoError(t, err, "failed to start pty command")
t.Cleanup(func() {
_ = ptyPs.Kill()
_ = ptyCmd.Close()
})
tr := testutil.NewTerminalReader(t, ptyCmd.OutputReader())
matchPrompt := func(line string) bool {
return strings.Contains(line, "#")
}
matchHostnameCmd := func(line string) bool {
return strings.Contains(strings.TrimSpace(line), "hostname")
}
matchHostnameOuput := func(line string) bool {
return strings.Contains(strings.TrimSpace(line), ct.Container.Config.Hostname)
}
matchEnvCmd := func(line string) bool {
return strings.Contains(strings.TrimSpace(line), "env")
}
matchEnvOutput := func(line string) bool {
return strings.Contains(line, "FOO=bar") || strings.Contains(line, "MULTILINE=foo")
}
require.NoError(t, tr.ReadUntil(ctx, matchPrompt), "failed to match prompt")
t.Logf("Matched prompt")
_, err = ptyCmd.InputWriter().Write([]byte("hostname\r\n"))
require.NoError(t, err, "failed to write to pty")
t.Logf("Wrote hostname command")
require.NoError(t, tr.ReadUntil(ctx, matchHostnameCmd), "failed to match hostname command")
t.Logf("Matched hostname command")
require.NoError(t, tr.ReadUntil(ctx, matchHostnameOuput), "failed to match hostname output")
t.Logf("Matched hostname output")
_, err = ptyCmd.InputWriter().Write([]byte("env\r\n"))
require.NoError(t, err, "failed to write to pty")
t.Logf("Wrote env command")
require.NoError(t, tr.ReadUntil(ctx, matchEnvCmd), "failed to match env command")
t.Logf("Matched env command")
require.NoError(t, tr.ReadUntil(ctx, matchEnvOutput), "failed to match env output")
t.Logf("Matched env output")
break
}
}
assert.True(t, found, "Expected to find container with label 'com.coder.test=%s'", testLabelValue)
}
func TestWrapDockerExec(t *testing.T) {
t.Parallel()
tests := []struct {
name string
containerUser string
cmdArgs []string
wantCmd []string
}{
{
name: "cmd with no args",
containerUser: "my-user",
cmdArgs: []string{"my-cmd"},
wantCmd: []string{"docker", "exec", "--interactive", "--user", "my-user", "my-container", "my-cmd"},
},
{
name: "cmd with args",
containerUser: "my-user",
cmdArgs: []string{"my-cmd", "arg1", "--arg2", "arg3", "--arg4"},
wantCmd: []string{"docker", "exec", "--interactive", "--user", "my-user", "my-container", "my-cmd", "arg1", "--arg2", "arg3", "--arg4"},
},
{
name: "no user specified",
containerUser: "",
cmdArgs: []string{"my-cmd"},
wantCmd: []string{"docker", "exec", "--interactive", "my-container", "my-cmd"},
},
}
for _, tt := range tests {
tt := tt // appease the linter even though this isn't needed anymore
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
actualCmd, actualArgs := wrapDockerExec("my-container", tt.containerUser, tt.cmdArgs[0], tt.cmdArgs[1:]...)
assert.Equal(t, tt.wantCmd[0], actualCmd)
assert.Equal(t, tt.wantCmd[1:], actualArgs)
})
}
}
// TestContainersHandler tests the containersHandler.getContainers method using
// a mock implementation. It specifically tests caching behavior.
func TestContainersHandler(t *testing.T) {
@ -319,6 +412,131 @@ func TestConvertDockerVolume(t *testing.T) {
}
}
// TestDockerEnvInfoer tests the ability of EnvInfo to extract information from
// running containers. Containers are deleted after the test is complete.
// As this test creates containers, it is skipped by default.
// It can be run manually as follows:
//
// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerEnvInfoer
func TestDockerEnvInfoer(t *testing.T) {
t.Parallel()
if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" {
t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
}
pool, err := dockertest.NewPool("")
require.NoError(t, err, "Could not connect to docker")
// nolint:paralleltest // variable recapture no longer required
for idx, tt := range []struct {
image string
labels map[string]string
expectedEnv []string
containerUser string
expectedUsername string
expectedUserShell string
}{
{
image: "busybox:latest",
labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
expectedUsername: "root",
expectedUserShell: "/bin/sh",
},
{
image: "busybox:latest",
labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
containerUser: "root",
expectedUsername: "root",
expectedUserShell: "/bin/sh",
},
{
image: "codercom/enterprise-minimal:ubuntu",
labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
expectedUsername: "coder",
expectedUserShell: "/bin/bash",
},
{
image: "codercom/enterprise-minimal:ubuntu",
labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
containerUser: "coder",
expectedUsername: "coder",
expectedUserShell: "/bin/bash",
},
{
image: "codercom/enterprise-minimal:ubuntu",
labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`},
expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"},
containerUser: "root",
expectedUsername: "root",
expectedUserShell: "/bin/bash",
},
} {
t.Run(fmt.Sprintf("#%d", idx), func(t *testing.T) {
t.Parallel()
// Start a container with the given image
// and environment variables
image := strings.Split(tt.image, ":")[0]
tag := strings.Split(tt.image, ":")[1]
ct, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: image,
Tag: tag,
Cmd: []string{"sleep", "infinity"},
Labels: tt.labels,
}, func(config *docker.HostConfig) {
config.AutoRemove = true
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
})
require.NoError(t, err, "Could not start test docker container")
t.Logf("Created container %q", ct.Container.Name)
t.Cleanup(func() {
assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name)
t.Logf("Purged container %q", ct.Container.Name)
})
ctx := testutil.Context(t, testutil.WaitShort)
dei, err := EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, tt.containerUser)
require.NoError(t, err, "Expected no error from DockerEnvInfo()")
u, err := dei.CurrentUser()
require.NoError(t, err, "Expected no error from CurrentUser()")
require.Equal(t, tt.expectedUsername, u.Username, "Expected username to match")
hd, err := dei.UserHomeDir()
require.NoError(t, err, "Expected no error from UserHomeDir()")
require.NotEmpty(t, hd, "Expected user homedir to be non-empty")
sh, err := dei.UserShell(tt.containerUser)
require.NoError(t, err, "Expected no error from UserShell()")
require.Equal(t, tt.expectedUserShell, sh, "Expected user shell to match")
// We don't need to test the actual environment variables here.
environ := dei.Environ()
require.NotEmpty(t, environ, "Expected environ to be non-empty")
// Test that the environment variables are present in modified command
// output.
envCmd, envArgs := dei.ModifyCommand("env")
for _, env := range tt.expectedEnv {
require.Subset(t, envArgs, []string{"--env", env})
}
// Run the command in the container and check the output
// HACK: we remove the --tty argument because we're not running in a tty
envArgs = slices.DeleteFunc(envArgs, func(s string) bool { return s == "--tty" })
stdout, stderr, err := run(ctx, agentexec.DefaultExecer, envCmd, envArgs...)
require.Empty(t, stderr, "Expected no stderr output")
require.NoError(t, err, "Expected no error from running command")
for _, env := range tt.expectedEnv {
require.Contains(t, stdout, env)
}
})
}
}
func fakeContainer(t *testing.T, mut ...func(*codersdk.WorkspaceAgentDevcontainer)) codersdk.WorkspaceAgentDevcontainer {
t.Helper()
ct := codersdk.WorkspaceAgentDevcontainer{