chore: update scaletest terraform with latest findings (#8249)

Updates scaletest terraform with learnings from #8213:

- Increase max pods per node to 256
- Decrease CPU requests for test workspace to allow maxing out workspaces per node
- Explicitly set CODER_ACCESS_URL for ssh to work
- Explicitly disable rate limits in coderd
- Increase DB size for medium and large scenarios
- Mount cache volume directly under /tmp/coder instead of /tmp.
- Plumb through requests and limits for workspaces
- Plumb through requests for coderd

scaletest/terraform/coder.tf

@@ -83,6 +83,8 @@ coder:
               operator: "In"
               values:   ["${local.coder_release_name}"]
   env:
+    - name: "CODER_ACCESS_URL"
+      value: "${local.coder_url}"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
     - name: "CODER_ENABLE_TELEMETRY"
@@ -108,17 +110,19 @@ coder:
       value: "true"
     - name: "CODER_EXPERIMENTS"
       value: "${var.coder_experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
   image:
     repo: ${var.coder_image_repo}
     tag: ${var.coder_image_tag}
   replicaCount: "${var.coder_replicas}"
   resources:
     requests:
-      cpu: "${var.coder_cpu}"
-      memory: "${var.coder_mem}"
+      cpu: "${var.coder_cpu_request}"
+      memory: "${var.coder_mem_request}"
     limits:
-      cpu: "${var.coder_cpu}"
-      memory: "${var.coder_mem}"
+      cpu: "${var.coder_cpu_limit}"
+      memory: "${var.coder_mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   service:
@@ -126,7 +130,7 @@ coder:
     sessionAffinity: None
     loadBalancerIP: "${local.coder_address}"
   volumeMounts:
-  - mountPath: "/tmp"
+  - mountPath: "/tmp/coder"
     name: cache
     readOnly: false
   volumes:
@@ -197,12 +201,12 @@ resource "local_file" "kubernetes_template" {
           }
           resources {
             requests = {
-              "cpu"    = "0.1"
-              "memory" = "128Mi"
+              "cpu"    = "${var.workspace_cpu_request}"
+              "memory" = "${var.workspace_mem_request}"
             }
             limits = {
-              "cpu"    = "1"
-              "memory" = "1Gi"
+              "cpu"    = "${var.workspace_cpu_limit}"
+              "memory" = "${var.workspace_mem_limit}"
             }
           }
         }