ci: improve caching (#7954)

.github/actions/setup-go/action.yaml

@@ -1,36 +1,65 @@
 name: "Setup Go"
 description: |
   Sets up the Go environment for tests, builds, etc.
+inputs:
+  version:
+    description: "The Go version to use."
+    default: "1.20.5"
 runs:
   using: "composite"
   steps:
-    - uses: buildjet/setup-go@v4
+    - name: Cache go toolchain
-      with:
-        cache: true
-        go-version: "1.20.5"
-
-    - name: Cache go
       uses: buildjet/cache@v3
       with:
-        # ~/go/pkg is the same across operating systems.
         path: |
-          ~/go/pkg
+          ${{ runner.tool_cache }}/go/${{ inputs.version }}
-          ~/.cache/go-build
+        key: gotoolchain-${{ runner.os }}-${{ inputs.version }}
-          ~/AppData/Local/go-build
+        restore-keys: |
-          ~/Library/Caches/go-build
+          gotoolchain-${{ runner.os }}-
+
+    - uses: buildjet/setup-go@v4
+      with:
+        # We do our own caching for implementation clarity.
+        cache: false
+        go-version: ${{ inputs.version }}
+
+    - name: Get cache dirs
+      shell: bash
+      run: |
+        set -x
+        echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV
+        echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_ENV
+
+    # We split up GOMODCACHE from GOCACHE because the latter must be invalidated
+    # on code change, but the former can be kept.
+    - name: Cache $GOMODCACHE
+      uses: buildjet/cache@v3
+      with:
+        path: |
+          ${{ env.GOMODCACHE }}
+        key: gomodcache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}
+        restore-keys: |
+          gomodcache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}-
+          gomodcache-${{ runner.os }}-
+
+    - name: Cache $GOCACHE
+      uses: buildjet/cache@v3
+      with:
+        path: |
+          ${{ env.GOCACHE }}
         # Job name must be included in the key for effective
         # test cache reuse.
-        key: go-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/*.go', 'go.**') }}
+        # The key format is intentionally different than GOMODCACHE, because any
+        # time a Go file changes we invalidate this cache, whereas GOMODCACHE
+        # is only invalidated when go.sum changes.
+        key: gocache-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/*.go', 'go.**') }}
         restore-keys: |
-          go-${{ runner.os }}-${{ github.job }}-
+          gocache-${{ runner.os }}-${{ github.job }}-
-          go-${{ runner.os }}-
+          gocache-${{ runner.os }}-
-          go-
 
     - name: Install gotestsum
-      uses: jaxxstorm/action-install-gh-release@v1.10.0
+      shell: bash
-      with:
+      run: go install gotest.tools/gotestsum@latest
-        repo: gotestyourself/gotestsum
-        tag: v1.9.0
 
     # It isn't necessary that we ever do this, but it helps
     # separate the "setup" from the "run" times.

.github/workflows/ci.yaml

@@ -100,11 +100,29 @@ jobs:
 
       - uses: ./.github/actions/setup-go
 
-      # Check for any typos!
+      - uses: ./.github/actions/setup-node
+
+      - name: Get golangci-lint cache dir
+        run: |
+          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
+          dir=$(golangci-lint cache status | awk '/Dir/ { print $2 }')
+          echo "LINT_CACHE_DIR=$dir" >> $GITHUB_ENV
+
+      - name: golangci-lint cache
+        uses: buildjet/cache@v3
+        with:
+          path: |
+            ${{ env.LINT_CACHE_DIR }}
+          key: golangci-lint-${{ runner.os }}-${{ hashFiles('**/*.go') }}
+          restore-keys: |
+            golangci-lint-${{ runner.os }}-
+
+      # Check for any typos
       - name: Check for typos
         uses: crate-ci/typos@v1.14.12
         with:
           config: .github/workflows/typos.toml
+
       - name: Fix the typos
         if: ${{ failure() }}
         run: |
@@ -112,37 +130,15 @@ jobs:
           cargo install typos-cli
           typos -c .github/workflows/typos.toml -w"
 
-      # Check for Go linting errors!
+      # Needed for helm chart linting
-      - name: Lint Go
-        uses: golangci/golangci-lint-action@v3.5.0
-        with:
-          version: v1.52.2
-
-      - name: Lint shell scripts
-        uses: ludeeus/action-shellcheck@2.0.0
-        env:
-          SHELLCHECK_OPTS: --external-sources
-        with:
-          ignore: node_modules
-
-      - uses: ./.github/actions/setup-node
-      - name: Lint TypeScript
-        run: yarn lint
-        working-directory: site
-
-      # Make sure the Helm chart is linted!
       - name: Install helm
         uses: azure/setup-helm@v3
         with:
           version: v3.9.2
-      - name: Lint Helm chart
-        run: |
-          cd helm
-          make lint
 
-      # Ensure AGPL and Enterprise are separated!
+      - name: make lint
-      - name: Check for AGPL code importing Enterprise...
+        run: |
-        run: ./scripts/check_enterprise_imports.sh
+          make --output-sync=line -j lint
 
   gen:
     timeout-minutes: 8
@@ -158,16 +154,14 @@ jobs:
       - name: Install sqlc
         run: |
           curl -sSL https://github.com/kyleconroy/sqlc/releases/download/v1.17.2/sqlc_1.17.2_linux_amd64.tar.gz | sudo tar -C /usr/bin -xz sqlc
-      - name: Install protoc-gen-go
+
-        run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
+      - name: go install tools
-      - name: Install protoc-gen-go-drpc
+        run: |
-        run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
-      - name: Install goimports
+          go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33
-        run: go install golang.org/x/tools/cmd/goimports@latest
+          go install golang.org/x/tools/cmd/goimports@latest
-      - name: Install yq
+          go install github.com/mikefarah/yq/v4@v4.30.6
-        run: go run github.com/mikefarah/yq/v4@v4.30.6
+          go install github.com/golang/mock/mockgen@v1.6.0
-      - name: Install mockgen
-        run: go install github.com/golang/mock/mockgen@v1.6.0
 
       - name: Install Protoc
         run: |
@@ -189,7 +183,7 @@ jobs:
         run: ./scripts/check_unstaged.sh
 
   fmt:
-    runs-on: ubuntu-latest
+    runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
     timeout-minutes: 5
     steps:
       - name: Checkout
@@ -268,7 +262,7 @@ jobs:
           go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
 
       - uses: ./.github/actions/upload-datadog
-        if: always()
+        if: success() || failure()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
@@ -315,15 +309,8 @@ jobs:
           # so we need to print the test stats to the log.
           go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
 
-      - uses: actions/upload-artifact@v3
-        if: success() || failure()
-        with:
-          name: gotests-postgres.xml
-          path: ./gotests.xml
-          retention-days: 30
-
       - uses: ./.github/actions/upload-datadog
-        if: always()
+        if: success() || failure()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
@@ -349,11 +336,6 @@ jobs:
 
       - uses: ./.github/actions/setup-go
 
-      - uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_version: 1.1.9
-          terraform_wrapper: false
-
       - name: Run Tests
         run: |
           gotestsum --junitfile="gotests.xml" -- -race ./...

.golangci.yaml

@@ -200,12 +200,11 @@ issues:
   max-same-issues: 0
 
 run:
-  concurrency: 4
   skip-dirs:
     - node_modules
   skip-files:
     - scripts/rules.go
-  timeout: 5m
+  timeout: 10m
 
 # Over time, add more and more linters from
 # https://golangci-lint.run/usage/linters/ as the code improves.

Makefile

@@ -402,11 +402,17 @@ else
 endif
 .PHONY: fmt/shfmt
 
-lint: lint/shellcheck lint/go
+lint: lint/shellcheck lint/go lint/ts lint/helm
 .PHONY: lint
 
+lint/ts:
+	cd site
+	yarn && yarn lint
+.PHONY: lint/ts
+
 lint/go:
 	./scripts/check_enterprise_imports.sh
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
 	golangci-lint run
 .PHONY: lint/go
 
@@ -416,6 +422,11 @@ lint/shellcheck: $(SHELL_SRC_FILES)
 	shellcheck --external-sources $(SHELL_SRC_FILES)
 .PHONY: lint/shellcheck
 
+lint/helm:
+	cd helm
+	make lint
+.PHONY: lint/helm
+
 # all gen targets should be added here and to gen/mark-fresh
 gen: \
 	coderd/database/dump.sql \

enterprise/cli/root_internal_test.go

@@ -3,11 +3,10 @@ package cli
 import (
 	"testing"
 
-	"github.com/coder/coder/cli"
 	"github.com/stretchr/testify/require"
 
+	"github.com/coder/coder/cli"
 	"github.com/coder/coder/cli/clibase"
-
 	"github.com/coder/coder/cli/clitest"
 )