feat: add provisioner job hang detector (#7927)

provisioner/terraform/provision_test.go

@@ -129,8 +129,7 @@ func TestProvision_Cancel(t *testing.T) {
 			require.NoError(t, err)
 
 			ctx, api := setupProvisioner(t, &provisionerServeOptions{
-				binaryPath:  binPath,
-				exitTimeout: time.Nanosecond,
+				binaryPath: binPath,
 			})
 
 			response, err := api.Provision(ctx)
@@ -186,6 +185,75 @@ func TestProvision_Cancel(t *testing.T) {
 	}
 }
 
+func TestProvision_CancelTimeout(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		t.Skip("This test uses interrupts and is not supported on Windows")
+	}
+
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+	fakeBin := filepath.Join(cwd, "testdata", "fake_cancel_hang.sh")
+
+	dir := t.TempDir()
+	binPath := filepath.Join(dir, "terraform")
+
+	// Example: exec /path/to/terrafork_fake_cancel.sh 1.2.1 apply "$@"
+	content := fmt.Sprintf("#!/bin/sh\nexec %q %s \"$@\"\n", fakeBin, terraform.TerraformVersion.String())
+	err = os.WriteFile(binPath, []byte(content), 0o755) //#nosec
+	require.NoError(t, err)
+
+	ctx, api := setupProvisioner(t, &provisionerServeOptions{
+		binaryPath:  binPath,
+		exitTimeout: time.Second,
+	})
+
+	response, err := api.Provision(ctx)
+	require.NoError(t, err)
+	err = response.Send(&proto.Provision_Request{
+		Type: &proto.Provision_Request_Apply{
+			Apply: &proto.Provision_Apply{
+				Config: &proto.Provision_Config{
+					Directory: dir,
+					Metadata:  &proto.Provision_Metadata{},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	for _, line := range []string{"init", "apply_start"} {
+	LoopStart:
+		msg, err := response.Recv()
+		require.NoError(t, err)
+
+		t.Log(msg.Type)
+
+		log := msg.GetLog()
+		if log == nil {
+			goto LoopStart
+		}
+		require.Equal(t, line, log.Output)
+	}
+
+	err = response.Send(&proto.Provision_Request{
+		Type: &proto.Provision_Request_Cancel{
+			Cancel: &proto.Provision_Cancel{},
+		},
+	})
+	require.NoError(t, err)
+
+	for {
+		msg, err := response.Recv()
+		require.NoError(t, err)
+
+		if c := msg.GetComplete(); c != nil {
+			require.Contains(t, c.Error, "killed")
+			break
+		}
+	}
+}
+
 func TestProvision(t *testing.T) {
 	t.Parallel()