chore: do not refresh tokens that have already failed refreshing (#15608)

Once a token refresh fails, we remove the `oauth_refresh_token` from the
database. This will prevent the token from hitting the IDP for
subsequent refresh attempts.

Without this change, a bad script can cause a failing token to hit a
remote IDP repeatedly with each `git` operation. With this change, after
the first hit, subsequent hits will fail locally, and never contact the
IDP.

The solution in both cases is to authenticate the external auth link. So
the resolution is the same as before.

coderd/externalauth/externalauth.go

@@ -118,7 +118,7 @@ func (c *Config) RefreshToken(ctx context.Context, db database.Store, externalAu
 		// This is true for github, which has no expiry.
 		!externalAuthLink.OAuthExpiry.IsZero() &&
 		externalAuthLink.OAuthExpiry.Before(dbtime.Now()) {
-		return externalAuthLink, InvalidTokenError("token expired, refreshing is disabled")
+		return externalAuthLink, InvalidTokenError("token expired, refreshing is either disabled or refreshing failed and will not be retried")
 	}
 
 	// This is additional defensive programming. Because TokenSource is an interface,
@@ -130,16 +130,41 @@ func (c *Config) RefreshToken(ctx context.Context, db database.Store, externalAu
 		refreshToken = ""
 	}
 
-	token, err := c.TokenSource(ctx, &oauth2.Token{
+	existingToken := &oauth2.Token{
 		AccessToken:  externalAuthLink.OAuthAccessToken,
 		RefreshToken: refreshToken,
 		Expiry:       externalAuthLink.OAuthExpiry,
-	}).Token()
+	}
+
+	token, err := c.TokenSource(ctx, existingToken).Token()
 	if err != nil {
-		// Even if the token fails to be obtained, do not return the error as an error.
+		// TokenSource can fail for numerous reasons. If it fails because of
+		// a bad refresh token, then the refresh token is invalid, and we should
+		// get rid of it. Keeping it around will cause additional refresh
+		// attempts that will fail and cost us api rate limits.
+		if isFailedRefresh(existingToken, err) {
+			dbExecErr := db.RemoveRefreshToken(ctx, database.RemoveRefreshTokenParams{
+				UpdatedAt:  dbtime.Now(),
+				ProviderID: externalAuthLink.ProviderID,
+				UserID:     externalAuthLink.UserID,
+			})
+			if dbExecErr != nil {
+				// This error should be rare.
+				return externalAuthLink, InvalidTokenError(fmt.Sprintf("refresh token failed: %q, then removing refresh token failed: %q", err.Error(), dbExecErr.Error()))
+			}
+			// The refresh token was cleared
+			externalAuthLink.OAuthRefreshToken = ""
+		}
+
+		// Unfortunately have to match exactly on the error message string.
+		// Improve the error message to account refresh tokens are deleted if
+		// invalid on our end.
+		if err.Error() == "oauth2: token expired and refresh token is not set" {
+			return externalAuthLink, InvalidTokenError("token expired, refreshing is either disabled or refreshing failed and will not be retried")
+		}
+
 		// TokenSource(...).Token() will always return the current token if the token is not expired.
-		// If it is expired, it will attempt to refresh the token, and if it cannot, it will fail with
-		// an error. This error is a reason the token is invalid.
+		// So this error is only returned if a refresh of the token failed.
 		return externalAuthLink, InvalidTokenError(fmt.Sprintf("refresh token: %s", err.Error()))
 	}
 
@@ -973,3 +998,50 @@ func IsGithubDotComURL(str string) bool {
 	}
 	return ghURL.Host == "github.com"
 }
+
+// isFailedRefresh returns true if the error returned by the TokenSource.Token()
+// is due to a failed refresh. The failure being the refresh token itself.
+// If this returns true, no amount of retries will fix the issue.
+//
+// Notes: Provider responses are not uniform. Here are some examples:
+// Github
+//   - Returns a 200 with Code "bad_refresh_token" and Description "The refresh token passed is incorrect or expired."
+//
+// Gitea [TODO: get an expired refresh token]
+//   - [Bad JWT] Returns 400 with Code "unauthorized_client" and Description "unable to parse refresh token"
+//
+// Gitlab
+//   - Returns 400 with Code "invalid_grant" and Description "The provided authorization grant is invalid, expired, revoked, does not match the redirection URI used in the authorization request, or was issued to another client."
+func isFailedRefresh(existingToken *oauth2.Token, err error) bool {
+	if existingToken.RefreshToken == "" {
+		return false // No refresh token, so this cannot be refreshed
+	}
+
+	if existingToken.Valid() {
+		return false // Valid tokens are not refreshed
+	}
+
+	var oauthErr *oauth2.RetrieveError
+	if xerrors.As(err, &oauthErr) {
+		switch oauthErr.ErrorCode {
+		// Known error codes that indicate a failed refresh.
+		// 'Spec' means the code is defined in the spec.
+		case "bad_refresh_token", // Github
+			"invalid_grant",          // Gitlab & Spec
+			"unauthorized_client",    // Gitea & Spec
+			"unsupported_grant_type": // Spec, refresh not supported
+			return true
+		}
+
+		switch oauthErr.Response.StatusCode {
+		case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusOK:
+			// Status codes that indicate the request was processed, and rejected.
+			return true
+		case http.StatusInternalServerError, http.StatusTooManyRequests:
+			// These do not indicate a failed refresh, but could be a temporary issue.
+			return false
+		}
+	}
+
+	return false
+}