chore: run test-go-pg on macOS and Windows in regular CI (#17853)

This PR starts running test-go-pg on macOS and Windows in regular CI.
Previously this suite was only run in the nightly gauntlet for 2
reasons:

- it was flaky
- it was slow (took 17 minutes)

We've since stabilized the flakiness by switching to depot runners,
using ram disks, optimizing the number of tests run in parallel, and
automatically re-running failing tests. We've also [brought
down](https://github.com/coder/coder/pull/17756) the time to run the
suite to 9 minutes. Additionally, this PR allows test-go-pg to use cache
from previous runs, which speeds it up further. The cache is only used
on PRs, `main` will still run tests without it.

This PR also:

- removes the nightly gauntlet since all tests now run in regular CI
- removes the `test-cli` job for the same reason
- removes the `setup-imdisk` action which is now fully replaced by
[coder/setup-ramdisk-action](https://github.com/coder/setup-ramdisk-action)
- makes 2 minor changes which could be separate PRs, but I rolled them
into this because they were helpful when iterating on it:
- replace the `if: always()` condition on the `gen` job with a `if: ${{
!cancelled() }}` to allow the job to be cancelled. Previously the job
would run to completion even if the entire workflow was cancelled. See
[the GitHub
docs](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#always)
for more details.
- disable the recently added `TestReinitializeAgent` since it does not
pass on Windows with Postgres. There's an open issue to fix it:
https://github.com/coder/internal/issues/642

This PR will:

- unblock https://github.com/coder/coder/issues/15109
- alleviate https://github.com/coder/internal/issues/647

I tested caching by temporarily enabling cache upload on this PR: here's
[a
run](https://github.com/coder/coder/actions/runs/15119046903/job/42496939341?pr=17853#step:13:1296)
showing cache being used.

.github/actions/setup-go-paths/action.yml

@@ -0,0 +1,57 @@
+name: "Setup Go Paths"
+description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
+outputs:
+  gocache:
+    description: "Value of GOCACHE"
+    value: ${{ steps.paths.outputs.gocache }}
+  gomodcache:
+    description: "Value of GOMODCACHE"
+    value: ${{ steps.paths.outputs.gomodcache }}
+  gopath:
+    description: "Value of GOPATH"
+    value: ${{ steps.paths.outputs.gopath }}
+  gotmp:
+    description: "Value of GOTMPDIR"
+    value: ${{ steps.paths.outputs.gotmp }}
+  cached-dirs:
+    description: "Go directories that should be cached between CI runs"
+    value: ${{ steps.paths.outputs.cached-dirs }}
+runs:
+  using: "composite"
+  steps:
+    - name: Override Go paths
+      id: paths
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+      with:
+        script: |
+          const path = require('path');
+
+          // RUNNER_TEMP should be backed by a RAM disk on Windows if
+          // coder/setup-ramdisk-action was used
+          const runnerTemp = process.env.RUNNER_TEMP;
+          const gocacheDir = path.join(runnerTemp, 'go-cache');
+          const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
+          const gopathDir = path.join(runnerTemp, 'go-path');
+          const gotmpDir = path.join(runnerTemp, 'go-tmp');
+
+          core.exportVariable('GOCACHE', gocacheDir);
+          core.exportVariable('GOMODCACHE', gomodcacheDir);
+          core.exportVariable('GOPATH', gopathDir);
+          core.exportVariable('GOTMPDIR', gotmpDir);
+
+          core.setOutput('gocache', gocacheDir);
+          core.setOutput('gomodcache', gomodcacheDir);
+          core.setOutput('gopath', gopathDir);
+          core.setOutput('gotmp', gotmpDir);
+
+          const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
+          core.setOutput('cached-dirs', cachedDirs);
+
+    - name: Create directories
+      shell: bash
+      run: |
+        set -e
+        mkdir -p "$GOCACHE"
+        mkdir -p "$GOMODCACHE"
+        mkdir -p "$GOPATH"
+        mkdir -p "$GOTMPDIR"

.github/actions/setup-go/action.yaml

@@ -8,42 +8,26 @@ inputs:
   use-preinstalled-go:
     description: "Whether to use preinstalled Go."
     default: "false"
-  use-temp-cache-dirs:
+  use-cache:
-    description: "Whether to use temporary GOCACHE and GOMODCACHE directories."
+    description: "Whether to use the cache."
-    default: "false"
+    default: "true"
 runs:
   using: "composite"
   steps:
-    - name: Override GOCACHE and GOMODCACHE
-      shell: bash
-      if: inputs.use-temp-cache-dirs == 'true'
-      run: |
-        # cd to another directory to ensure we're not inside a Go project.
-        # That'd trigger Go to download the toolchain for that project.
-        cd "$RUNNER_TEMP"
-        # RUNNER_TEMP should be backed by a RAM disk on Windows if
-        # coder/setup-ramdisk-action was used
-        export GOCACHE_DIR="$RUNNER_TEMP""\go-cache"
-        export GOMODCACHE_DIR="$RUNNER_TEMP""\go-mod-cache"
-        export GOPATH_DIR="$RUNNER_TEMP""\go-path"
-        export GOTMP_DIR="$RUNNER_TEMP""\go-tmp"
-        mkdir -p "$GOCACHE_DIR"
-        mkdir -p "$GOMODCACHE_DIR"
-        mkdir -p "$GOPATH_DIR"
-        mkdir -p "$GOTMP_DIR"
-        go env -w GOCACHE="$GOCACHE_DIR"
-        go env -w GOMODCACHE="$GOMODCACHE_DIR"
-        go env -w GOPATH="$GOPATH_DIR"
-        go env -w GOTMPDIR="$GOTMP_DIR"
     - name: Setup Go
       uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
       with:
         go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
+        cache: ${{ inputs.use-cache }}
 
     - name: Install gotestsum
       shell: bash
       run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
 
+    - name: Install mtimehash
+      shell: bash
+      run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
+
     # It isn't necessary that we ever do this, but it helps
     # separate the "setup" from the "run" times.
     - name: go mod download

.github/actions/setup-imdisk/action.yaml

@@ -1,27 +0,0 @@
-name: "Setup ImDisk"
-if: runner.os == 'Windows'
-description: |
-  Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-runs:
-  using: "composite"
-  steps:
-    - name: Download ImDisk
-      if: runner.os == 'Windows'
-      shell: bash
-      run: |
-        mkdir imdisk
-        cd imdisk
-        curl -L -o files.cab https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/files.cab
-        curl -L -o install.bat https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/install.bat
-        cd ..
-
-    - name: Install ImDisk
-      shell: cmd
-      run: |
-        cd imdisk
-        install.bat /silent
-
-    - name: Create RAM Disk
-      shell: cmd
-      run: |
-        imdisk -a -s 4096M -m R: -p "/fs:ntfs /q /y"

.github/workflows/ci.yaml

@@ -224,7 +224,7 @@ jobs:
   gen:
     timeout-minutes: 8
     runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
-    if: always()
+    if: ${{ !cancelled() }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
@@ -336,13 +336,16 @@ jobs:
       # a separate repository to allow its use before actions/checkout.
       - name: Setup RAM Disks
         if: runner.os == 'Windows'
-        uses: coder/setup-ramdisk-action@81c5c441bda00c6c3d6bcee2e5a33ed4aadbbcc1
+        uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
 
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
 
+      - name: Setup Go Paths
+        uses: ./.github/actions/setup-go-paths
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
         with:
@@ -350,7 +353,6 @@ jobs:
           # download the toolchain configured in go.mod, so we don't
           # need to reinstall it. It's faster on Windows runners.
           use-preinstalled-go: ${{ runner.os == 'Windows' }}
-          use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
@@ -398,63 +400,10 @@ jobs:
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
-  # We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR.
-  # We run the test suite in test-go-pg, including CLI.
-  test-cli:
-    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
-    needs: changes
-    if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
-    strategy:
-      matrix:
-        os:
-          - macos-latest
-          - windows-2022
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
-        with:
-          egress-policy: audit
-
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 1
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-
-      - name: Setup Terraform
-        uses: ./.github/actions/setup-tf
-
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
-      - name: Test CLI
-        env:
-          TS_DEBUG_DISCO: "true"
-          LC_CTYPE: "en_US.UTF-8"
-          LC_ALL: "en_US.UTF-8"
-          TEST_RETRIES: 2
-        shell: bash
-        run: |
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
-
-          make test-cli
-
-      - name: Upload test stats to Datadog
-        timeout-minutes: 1
-        continue-on-error: true
-        uses: ./.github/actions/upload-datadog
-        if: success() || failure()
-        with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
-
   test-go-pg:
-    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os }}
+    # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
+    # when changing runner sizes
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os && matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
     needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     # This timeout must be greater than the timeout set by `go test` in
@@ -466,48 +415,157 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+          - macos-latest
+          - windows-2022
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
         with:
           egress-policy: audit
 
+      # macOS indexes all new files in the background. Our Postgres tests
+      # create and destroy thousands of databases on disk, and Spotlight
+      # tries to index all of them, seriously slowing down the tests.
+      - name: Disable Spotlight Indexing
+        if: runner.os == 'macOS'
+        run: |
+          sudo mdutil -a -i off
+          sudo mdutil -X /
+          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
+
+      # Set up RAM disks to speed up the rest of the job. This action is in
+      # a separate repository to allow its use before actions/checkout.
+      - name: Setup RAM Disks
+        if: runner.os == 'Windows'
+        uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 1
 
+      - name: Setup Go Paths
+        id: go-paths
+        uses: ./.github/actions/setup-go-paths
+
+      - name: Download Go Build Cache
+        id: download-go-build-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }}
+          cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
+
       - name: Setup Go
         uses: ./.github/actions/setup-go
+        with:
+          # Runners have Go baked-in and Go will automatically
+          # download the toolchain configured in go.mod, so we don't
+          # need to reinstall it. It's faster on Windows runners.
+          use-preinstalled-go: ${{ runner.os == 'Windows' }}
+          # Cache is already downloaded above
+          use-cache: false
 
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
-      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
-      - name: Setup ImDisk
-        if: runner.os == 'Windows'
-        uses: ./.github/actions/setup-imdisk
-
       - name: Download Test Cache
         id: download-cache
         uses: ./.github/actions/test-cache/download
         with:
           key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
 
+      - name: Normalize File and Directory Timestamps
+        shell: bash
+        # Normalize file modification timestamps so that go test can use the
+        # cache from the previous CI run. See https://github.com/golang/go/issues/58571
+        # for more details.
+        run: |
+          find . -type f ! -path ./.git/\*\* | mtimehash
+          find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
+
       - name: Test with PostgreSQL Database
         env:
           POSTGRES_VERSION: "13"
           TS_DEBUG_DISCO: "true"
           LC_CTYPE: "en_US.UTF-8"
           LC_ALL: "en_US.UTF-8"
-          TEST_RETRIES: 2
         shell: bash
         run: |
-          # By default Go will use the number of logical CPUs, which
+          set -o errexit
-          # is a fine default.
+          set -o pipefail
-          PARALLEL_FLAG=""
 
-          make test-postgres
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Create a temp dir on the R: ramdisk drive for Windows. The default
+            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
+            mkdir -p "R:/temp/embedded-pg"
+            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Postgres runs faster on a ramdisk on macOS too
+            mkdir -p /tmp/tmpfs
+            sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
+            go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            make test-postgres-docker
+          fi
+
+          # if macOS, install google-chrome for scaletests
+          # As another concern, should we really have this kind of external dependency
+          # requirement on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
+
+          if [ "${{ runner.os }}" == "Windows" ]; then
+            # Our Windows runners have 16 cores.
+            # On Windows Postgres chokes up when we have 16x16=256 tests
+            # running in parallel, and dbtestutil.NewDB starts to take more than
+            # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
+            # Postgres tends not to choke.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "macOS" ]; then
+            # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
+            # because the tests complete faster and Postgres doesn't choke. It seems
+            # that macOS's tmpfs is faster than the one on Windows.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=16
+          elif [ "${{ runner.os }}" == "Linux" ]; then
+            # Our Linux runners have 8 cores.
+            NUM_PARALLEL_PACKAGES=8
+            NUM_PARALLEL_TESTS=8
+          fi
+
+          # by default, run tests with cache
+          TESTCOUNT=""
+          if [ "${{ github.ref }}" == "refs/heads/main" ]; then
+            # on main, run tests without cache
+            TESTCOUNT="-count=1"
+          fi
+
+          mkdir -p "$RUNNER_TEMP/sym"
+          source scripts/normalize_path.sh
+          # terraform gets installed in a random directory, so we need to normalize
+          # the path to the terraform binary or a bunch of cached tests will be
+          # invalidated. See scripts/normalize_path.sh for more details.
+          normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))"
+
+          # We rerun failing tests to counteract flakiness coming from Postgres
+          # choking on macOS and Windows sometimes.
+          DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
+            --format standard-quiet --packages "./..." \
+            -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
+
+      - name: Upload Go Build Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
+          cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
 
       - name: Upload Test Cache
         uses: ./.github/actions/test-cache/upload

.github/workflows/nightly-gauntlet.yaml

@@ -1,183 +0,0 @@
-# The nightly-gauntlet runs tests that are either too flaky or too slow to block
-# every PR.
-name: nightly-gauntlet
-on:
-  schedule:
-    # Every day at 4AM
-    - cron: "0 4 * * 1-5"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  test-go-pg:
-    # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
-    # when changing runner sizes
-    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
-    # This timeout must be greater than the timeout set by `go test` in
-    # `make test-postgres` to ensure we receive a trace of running
-    # goroutines. Setting this to the timeout +5m should work quite well
-    # even if some of the preceding steps are slow.
-    timeout-minutes: 25
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - macos-latest
-          - windows-2022
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
-        with:
-          egress-policy: audit
-
-      # macOS indexes all new files in the background. Our Postgres tests
-      # create and destroy thousands of databases on disk, and Spotlight
-      # tries to index all of them, seriously slowing down the tests.
-      - name: Disable Spotlight Indexing
-        if: runner.os == 'macOS'
-        run: |
-          sudo mdutil -a -i off
-          sudo mdutil -X /
-          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
-
-      # Set up RAM disks to speed up the rest of the job. This action is in
-      # a separate repository to allow its use before actions/checkout.
-      - name: Setup RAM Disks
-        if: runner.os == 'Windows'
-        uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439
-
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 1
-
-      - name: Setup Go
-        uses: ./.github/actions/setup-go
-        with:
-          # Runners have Go baked-in and Go will automatically
-          # download the toolchain configured in go.mod, so we don't
-          # need to reinstall it. It's faster on Windows runners.
-          use-preinstalled-go: ${{ runner.os == 'Windows' }}
-          use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
-
-      - name: Setup Terraform
-        uses: ./.github/actions/setup-tf
-
-      - name: Test with PostgreSQL Database
-        env:
-          POSTGRES_VERSION: "13"
-          TS_DEBUG_DISCO: "true"
-          LC_CTYPE: "en_US.UTF-8"
-          LC_ALL: "en_US.UTF-8"
-        shell: bash
-        run: |
-          if [ "${{ runner.os }}" == "Windows" ]; then
-            # Create a temp dir on the R: ramdisk drive for Windows. The default
-            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
-            mkdir -p "R:/temp/embedded-pg"
-            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
-          fi
-          if [ "${{ runner.os }}" == "macOS" ]; then
-            # Postgres runs faster on a ramdisk on macOS too
-            mkdir -p /tmp/tmpfs
-            sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
-            go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
-          fi
-
-          # if macOS, install google-chrome for scaletests
-          # As another concern, should we really have this kind of external dependency
-          # requirement on standard CI?
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            brew install google-chrome
-          fi
-
-          # By default Go will use the number of logical CPUs, which
-          # is a fine default.
-          PARALLEL_FLAG=""
-
-          # macOS will output "The default interactive shell is now zsh"
-          # intermittently in CI...
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
-          fi
-
-          # Golang's default for these 2 variables is the number of logical CPUs.
-          # Our Windows and Linux runners have 16 cores, so they match up there.
-          NUM_PARALLEL_PACKAGES=16
-          NUM_PARALLEL_TESTS=16
-          if [ "${{ runner.os }}" == "Windows" ]; then
-            # On Windows Postgres chokes up when we have 16x16=256 tests
-            # running in parallel, and dbtestutil.NewDB starts to take more than
-            # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
-            # Postgres tends not to choke.
-            NUM_PARALLEL_PACKAGES=8
-          fi
-          if [ "${{ runner.os }}" == "macOS" ]; then
-            # Our macOS runners have 8 cores. We leave NUM_PARALLEL_TESTS at 16
-            # because the tests complete faster and Postgres doesn't choke. It seems
-            # that macOS's tmpfs is faster than the one on Windows.
-            NUM_PARALLEL_PACKAGES=8
-          fi
-
-          # We rerun failing tests to counteract flakiness coming from Postgres
-          # choking on macOS and Windows sometimes.
-          DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=1000 \
-            --format standard-quiet --packages "./..." \
-            -- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1
-
-      - name: Upload test stats to Datadog
-        timeout-minutes: 1
-        continue-on-error: true
-        uses: ./.github/actions/upload-datadog
-        if: success() || failure()
-        with:
-          api-key: ${{ secrets.DATADOG_API_KEY }}
-
-  notify-slack-on-failure:
-    needs:
-      - test-go-pg
-    runs-on: ubuntu-latest
-    if: failure() && github.ref == 'refs/heads/main'
-
-    steps:
-      - name: Send Slack notification
-        run: |
-          curl -X POST -H 'Content-type: application/json' \
-          --data '{
-            "blocks": [
-              {
-                "type": "header",
-                "text": {
-                  "type": "plain_text",
-                  "text": "❌ Nightly gauntlet failed",
-                  "emoji": true
-                }
-              },
-              {
-                "type": "section",
-                "fields": [
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Workflow:*\n${{ github.workflow }}"
-                  },
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Committer:*\n${{ github.actor }}"
-                  },
-                  {
-                    "type": "mrkdwn",
-                    "text": "*Commit:*\n${{ github.sha }}"
-                  }
-                ]
-              },
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
-                }
-              }
-            ]
-          }' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}

scripts/normalize_path.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Call: normalize_path_with_symlinks [target_dir] [dir_prefix]
+#
+# Normalizes the PATH environment variable by replacing each directory that
+# begins with dir_prefix with a symbolic link in target_dir. For example, if
+# PATH is "/usr/bin:/bin", target_dir is /tmp, and dir_prefix is /usr, then
+# PATH will become "/tmp/0:/bin", where /tmp/0 links to /usr/bin.
+#
+# This is useful for ensuring that PATH is consistent across CI runs and helps
+# with reusing the same cache across them. Many of our go tests read the PATH
+# variable, and if it changes between runs, the cache gets invalidated.
+normalize_path_with_symlinks() {
+	local target_dir="${1:-}"
+	local dir_prefix="${2:-}"
+
+	if [[ -z "$target_dir" || -z "$dir_prefix" ]]; then
+		echo "Usage: normalize_path_with_symlinks <target_dir> <dir_prefix>"
+		return 1
+	fi
+
+	local old_path="$PATH"
+	local -a new_parts=()
+	local i=0
+
+	IFS=':' read -ra _parts <<<"$old_path"
+	for dir in "${_parts[@]}"; do
+		# Skip empty components that can arise from "::"
+		[[ -z $dir ]] && continue
+
+		# Skip directories that don't start with $dir_prefix
+		if [[ "$dir" != "$dir_prefix"* ]]; then
+			new_parts+=("$dir")
+			continue
+		fi
+
+		local link="$target_dir/$i"
+
+		# Replace any pre-existing file or link at $target_dir/$i
+		if [[ -e $link || -L $link ]]; then
+			rm -rf -- "$link"
+		fi
+
+		# without MSYS ln will deepcopy the directory on Windows
+		MSYS=winsymlinks:nativestrict ln -s -- "$dir" "$link"
+		new_parts+=("$link")
+		i=$((i + 1))
+	done
+
+	export PATH
+	PATH="$(
+		IFS=':'
+		echo "${new_parts[*]}"
+	)"
+}