example: Add Kubernetes multi-service (#1092)

* example: Add Kubernetes multi-service

* fix: change to CODER_AGENT_TOKEN

* example: use ServiceAccount for cluster authentication (#1096)

Co-authored-by: Ben <ben@coder.com>

examples/kubernetes-multi-service/README.md

@@ -0,0 +1,77 @@
+---
+name: Develop multiple services in Kubernetes
+description: Get started with Kubernetes development.
+tags: [cloud, kubernetes]
+---
+
+# Authentication
+
+This template has several ways to authenticate to a Kubernetes cluster.
+
+## kubeconfig (Coder host)
+
+If the Coder host has a local `~/.kube/config`, this can be used to authenticate with Coder. Make sure this is on the same user running the `coder` service.
+
+## ServiceAccount
+
+Create a ServiceAccount and role on your cluster to authenticate your template with Coder.
+
+1. Run the following command on a device with Kubernetes context:
+
+    ```sh
+    CODER_NAMESPACE=default
+    kubectl apply -n $CODER_NAMESPACE -f - <<EOF
+    apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+    name: coder
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: Role
+    metadata:
+    name: coder
+    rules:
+    - apiGroups: ["", "apps", "networking.k8s.io"] # "" indicates the core API group
+        resources: ["persistentvolumeclaims", "pods", "deployments", "services", "secrets", "pods/exec","pods/log", "events", "networkpolicies", "serviceaccounts"]
+        verbs: ["create", "get", "list", "watch", "update", "patch", "delete", "deletecollection"]
+    - apiGroups: ["metrics.k8s.io", "storage.k8s.io"]
+        resources: ["pods", "storageclasses"]
+        verbs: ["get", "list", "watch"]
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata:
+    name: coder
+    subjects:
+    - kind: ServiceAccount
+        name: coder
+    roleRef:
+    kind: Role
+    name: coder
+    apiGroup: rbac.authorization.k8s.io
+    EOF
+    ```
+
+   1. Use the following commands to fetch the values:
+
+        **Cluster IP:**
+
+        ```sh
+        kubectl cluster-info | grep "control plane"
+        ```
+
+        **CA certificate**
+
+        ```sh
+        kubectl get secrets -n $CODER_NAMESPACE -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='coder')].data['ca\.crt']}{'\n'}"
+        ```
+
+        **Token**
+
+        ```sh
+        kubectl get secrets -n $CODER_NAMESPACE -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='coder')].data['token']}{'\n'}"
+        ```
+
+        **Namespace**
+
+        This should be the same as `$CODER_NAMESPACE`, set in step 1.

examples/kubernetes-multi-service/main.tf

@@ -0,0 +1,132 @@
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "~> 0.3.1"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.10"
+    }
+  }
+}
+
+variable "step1_use_kubeconfig" {
+  type        = bool
+  sensitive   = true
+  description = "Use local ~/.kube/config? (true/false)"
+}
+
+variable "step2_cluster_host" {
+  type        = string
+  sensitive   = true
+  description = <<-EOF
+  Hint: You can use:
+  $ kubectl cluster-info | grep "control plane"
+
+
+  Leave blank if using ~/.kube/config (from step 1)
+  EOF
+}
+
+variable "step3_certificate" {
+  type        = string
+  sensitive   = true
+  description = <<-EOF
+  Use docs at https://github.com/coder/coder/tree/main/examples/kubernetes-multi-service#serviceaccount to create a ServiceAccount for Coder and grab values.
+
+  Enter CA certificate
+
+  Leave blank if using ~/.kube/config (from step 1)
+  EOF
+}
+
+variable "step4_token" {
+  type        = string
+  sensitive   = true
+  description = <<-EOF
+  Enter token (refer to docs at https://github.com/coder/coder/tree/main/examples/kubernetes-multi-service#serviceaccount)
+
+  Leave blank if using ~/.kube/config (from step 1)
+  EOF
+}
+
+variable "step5_coder_namespace" {
+  type        = string
+  sensitive   = true
+  description = <<-EOF
+  Enter namespace (refer to docs at https://github.com/coder/coder/tree/main/examples/kubernetes-multi-service#serviceaccount)
+
+  Leave blank if using ~/.kube/config (from step 1)
+  EOF
+}
+
+provider "kubernetes" {
+  # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences
+  config_path            = var.step1_use_kubeconfig == true ? "~/.kube/config" : null
+  host                   = var.step1_use_kubeconfig == false ? var.step2_cluster_host : null
+  cluster_ca_certificate = var.step1_use_kubeconfig == false ? base64decode(var.step3_certificate) : null
+  token                  = var.step1_use_kubeconfig == false ? base64decode(var.step4_token) : null
+}
+
+data "coder_workspace" "me" {}
+
+resource "coder_agent" "go" {
+  os   = "linux"
+  arch = "amd64"
+}
+
+resource "coder_agent" "java" {
+  os   = "linux"
+  arch = "amd64"
+}
+
+resource "coder_agent" "ubuntu" {
+  os   = "linux"
+  arch = "amd64"
+}
+
+resource "kubernetes_pod" "main" {
+  count = data.coder_workspace.me.start_count
+  metadata {
+    name = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
+  }
+  spec {
+    container {
+      name    = "go"
+      image   = "mcr.microsoft.com/vscode/devcontainers/go:1"
+      command = ["sh", "-c", coder_agent.go.init_script]
+      security_context {
+        run_as_user = "1000"
+      }
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.go.token
+      }
+    }
+    container {
+      name    = "java"
+      image   = "mcr.microsoft.com/vscode/devcontainers/java"
+      command = ["sh", "-c", coder_agent.java.init_script]
+      security_context {
+        run_as_user = "1000"
+      }
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.java.token
+      }
+    }
+    container {
+      name    = "ubuntu"
+      image   = "mcr.microsoft.com/vscode/devcontainers/base:ubuntu"
+      command = ["sh", "-c", coder_agent.ubuntu.init_script]
+      security_context {
+        run_as_user = "1000"
+      }
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.ubuntu.token
+      }
+    }
+  }
+}