Merge pull request #3769 from Infisical/daniel/full-gateway-auth

feat(gateway): use gateway for full k8s request life-cycle
This commit is contained in:
Daniel Hougaard
2025-06-11 00:55:38 +04:00
committed by GitHub
13 changed files with 245 additions and 105 deletions

View File

@@ -0,0 +1,45 @@
import { Knex } from "knex";
import { selectAllTableCols } from "@app/lib/knex";
import { TableName } from "../schemas";
const BATCH_SIZE = 1000;
export async function up(knex: Knex): Promise<void> {
const hasKubernetesHostColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "kubernetesHost");
if (hasKubernetesHostColumn) {
await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => {
table.string("kubernetesHost").nullable().alter();
});
}
}
export async function down(knex: Knex): Promise<void> {
const hasKubernetesHostColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "kubernetesHost");
// find all rows where kubernetesHost is null
const rows = await knex(TableName.IdentityKubernetesAuth)
.whereNull("kubernetesHost")
.select(selectAllTableCols(TableName.IdentityKubernetesAuth));
if (rows.length > 0) {
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
// eslint-disable-next-line no-await-in-loop
await knex(TableName.IdentityKubernetesAuth)
.whereIn(
"id",
batch.map((row) => row.id)
)
.update({ kubernetesHost: "" });
}
}
if (hasKubernetesHostColumn) {
await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => {
table.string("kubernetesHost").notNullable().alter();
});
}
}

View File

@@ -18,7 +18,7 @@ export const IdentityKubernetesAuthsSchema = z.object({
createdAt: z.date(),
updatedAt: z.date(),
identityId: z.string().uuid(),
kubernetesHost: z.string(),
kubernetesHost: z.string().nullable().optional(),
encryptedCaCert: z.string().nullable().optional(),
caCertIV: z.string().nullable().optional(),
caCertTag: z.string().nullable().optional(),

View File

@@ -10,7 +10,8 @@ export enum GatewayProxyProtocol {
}
export enum GatewayHttpProxyActions {
InjectGatewayK8sServiceAccountToken = "inject-k8s-sa-auth-token"
InjectGatewayK8sServiceAccountToken = "inject-k8s-sa-auth-token",
UseGatewayK8sServiceAccount = "use-k8s-sa"
}
export interface IGatewayProxyOptions {

View File

@@ -108,17 +108,21 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
.string()
.trim()
.min(1)
.nullable()
.describe(KUBERNETES_AUTH.ATTACH.kubernetesHost)
.refine(
(val) =>
characterValidator([
(val) => {
if (val === null) return true;
return characterValidator([
CharacterType.Alphabets,
CharacterType.Numbers,
CharacterType.Colon,
CharacterType.Period,
CharacterType.ForwardSlash,
CharacterType.Hyphen
])(val),
])(val);
},
{
message:
"Kubernetes host must only contain alphabets, numbers, colons, periods, hyphen, and forward slashes."
@@ -164,6 +168,13 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
.describe(KUBERNETES_AUTH.ATTACH.accessTokenNumUsesLimit)
})
.superRefine((data, ctx) => {
if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && !data.kubernetesHost) {
ctx.addIssue({
path: ["kubernetesHost"],
code: z.ZodIssueCode.custom,
message: "When token review mode is set to API, a Kubernetes host must be provided"
});
}
if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && !data.gatewayId) {
ctx.addIssue({
path: ["gatewayId"],
@@ -171,6 +182,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
message: "When token review mode is set to Gateway, a gateway must be selected"
});
}
if (data.accessTokenTTL > data.accessTokenMaxTTL) {
ctx.addIssue({
path: ["accessTokenTTL"],
@@ -203,7 +215,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
type: EventType.ADD_IDENTITY_KUBERNETES_AUTH,
metadata: {
identityId: identityKubernetesAuth.identityId,
kubernetesHost: identityKubernetesAuth.kubernetesHost,
kubernetesHost: identityKubernetesAuth.kubernetesHost ?? "",
allowedNamespaces: identityKubernetesAuth.allowedNamespaces,
allowedNames: identityKubernetesAuth.allowedNames,
accessTokenTTL: identityKubernetesAuth.accessTokenTTL,
@@ -243,6 +255,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
.string()
.trim()
.min(1)
.nullable()
.optional()
.describe(KUBERNETES_AUTH.UPDATE.kubernetesHost)
.refine(
@@ -345,7 +358,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide
type: EventType.UPDATE_IDENTITY_KUBENETES_AUTH,
metadata: {
identityId: identityKubernetesAuth.identityId,
kubernetesHost: identityKubernetesAuth.kubernetesHost,
kubernetesHost: identityKubernetesAuth.kubernetesHost ?? "",
allowedNamespaces: identityKubernetesAuth.allowedNamespaces,
allowedNames: identityKubernetesAuth.allowedNames,
accessTokenTTL: identityKubernetesAuth.accessTokenTTL,

View File

@@ -104,11 +104,15 @@ export const identityKubernetesAuthServiceFactory = ({
cert: relayDetails.certificate,
key: relayDetails.privateKey.toString()
},
// we always pass this, because its needed for both tcp and http protocol
httpsAgent: new https.Agent({
ca: inputs.caCert,
rejectUnauthorized: Boolean(inputs.caCert)
})
// only needed for TCP protocol, because the gateway as reviewer will use the pod's CA cert for auth directly
...(!inputs.reviewTokenThroughGateway
? {
httpsAgent: new https.Agent({
ca: inputs.caCert,
rejectUnauthorized: Boolean(inputs.caCert)
})
}
: {})
}
);
@@ -142,8 +146,15 @@ export const identityKubernetesAuthServiceFactory = ({
caCert = decryptor({ cipherTextBlob: identityKubernetesAuth.encryptedKubernetesCaCertificate }).toString();
}
const tokenReviewCallbackRaw = async (host: string = identityKubernetesAuth.kubernetesHost, port?: number) => {
const tokenReviewCallbackRaw = async (host = identityKubernetesAuth.kubernetesHost, port?: number) => {
logger.info({ host, port }, "tokenReviewCallbackRaw: Processing kubernetes token review using raw API");
if (!host || !identityKubernetesAuth.kubernetesHost) {
throw new BadRequestError({
message: "Kubernetes host is required when token review mode is set to API"
});
}
let tokenReviewerJwt = "";
if (identityKubernetesAuth.encryptedKubernetesTokenReviewerJwt) {
tokenReviewerJwt = decryptor({
@@ -211,11 +222,7 @@ export const identityKubernetesAuthServiceFactory = ({
return res.data;
};
const tokenReviewCallbackThroughGateway = async (
host: string = identityKubernetesAuth.kubernetesHost,
port?: number,
httpsAgent?: https.Agent
) => {
const tokenReviewCallbackThroughGateway = async (host: string, port?: number) => {
logger.info(
{
host,
@@ -224,11 +231,9 @@ export const identityKubernetesAuthServiceFactory = ({
"tokenReviewCallbackThroughGateway: Processing kubernetes token review using gateway"
);
const baseUrl = port ? `${host}:${port}` : host;
const res = await axios
.post<TCreateTokenReviewResponse>(
`${baseUrl}/apis/authentication.k8s.io/v1/tokenreviews`,
`${host}:${port}/apis/authentication.k8s.io/v1/tokenreviews`,
{
apiVersion: "authentication.k8s.io/v1",
kind: "TokenReview",
@@ -240,11 +245,10 @@ export const identityKubernetesAuthServiceFactory = ({
{
headers: {
"Content-Type": "application/json",
"x-infisical-action": GatewayHttpProxyActions.InjectGatewayK8sServiceAccountToken
"x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount
},
signal: AbortSignal.timeout(10000),
timeout: 10000,
...(httpsAgent ? { httpsAgent } : {})
timeout: 10000
}
)
.catch((err) => {
@@ -273,29 +277,6 @@ export const identityKubernetesAuthServiceFactory = ({
let data: TCreateTokenReviewResponse | undefined;
if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway) {
const { kubernetesHost } = identityKubernetesAuth;
let urlString = kubernetesHost;
if (!kubernetesHost.startsWith("http://") && !kubernetesHost.startsWith("https://")) {
urlString = `https://${kubernetesHost}`;
}
const url = new URL(urlString);
let { port: k8sPort } = url;
const { protocol, hostname: k8sHost } = url;
const cleanedProtocol = new RE2(/[^a-zA-Z0-9]/g).replace(protocol, "").toLowerCase();
if (!["https", "http"].includes(cleanedProtocol)) {
throw new BadRequestError({
message: "Invalid Kubernetes host URL, must start with http:// or https://"
});
}
if (!k8sPort) {
k8sPort = cleanedProtocol === "https" ? "443" : "80";
}
if (!identityKubernetesAuth.gatewayId) {
throw new BadRequestError({
message: "Gateway ID is required when token review mode is set to Gateway"
@@ -305,14 +286,19 @@ export const identityKubernetesAuthServiceFactory = ({
data = await $gatewayProxyWrapper(
{
gatewayId: identityKubernetesAuth.gatewayId,
targetHost: `${cleanedProtocol}://${k8sHost}`, // note(daniel): must include the protocol (https|http)
targetPort: k8sPort ? Number(k8sPort) : 443,
caCert,
targetHost: `/`, // note(daniel): the targetURL will be constructed as `/:0`, which the gateway will handle as a special case, by replacing the /:0, with the internal kubernetes base URL (only when the action header is set to `GatewayHttpProxyActions.UseGatewayK8sServiceAccount`)
targetPort: 0,
reviewTokenThroughGateway: true
},
tokenReviewCallbackThroughGateway
);
} else if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api) {
if (!identityKubernetesAuth.kubernetesHost) {
throw new BadRequestError({
message: "Kubernetes host is required when token review mode is set to API"
});
}
let { kubernetesHost } = identityKubernetesAuth;
if (kubernetesHost.startsWith("https://") || kubernetesHost.startsWith("http://")) {
kubernetesHost = new RE2("^https?:\\/\\/").replace(kubernetesHost, "");

View File

@@ -12,7 +12,7 @@ export enum IdentityKubernetesAuthTokenReviewMode {
export type TAttachKubernetesAuthDTO = {
identityId: string;
kubernetesHost: string;
kubernetesHost: string | null;
caCert: string;
tokenReviewerJwt?: string;
tokenReviewMode: IdentityKubernetesAuthTokenReviewMode;
@@ -29,7 +29,7 @@ export type TAttachKubernetesAuthDTO = {
export type TUpdateKubernetesAuthDTO = {
identityId: string;
kubernetesHost?: string;
kubernetesHost?: string | null;
caCert?: string;
tokenReviewerJwt?: string | null;
tokenReviewMode?: IdentityKubernetesAuthTokenReviewMode;

View File

@@ -116,7 +116,9 @@ func handleStream(stream quic.Stream, quicConn quic.Connection) {
targetURL := string(argParts[0])
if !isValidURL(targetURL) {
// ? note(daniel): special case: if the target URL is "/:0", we don't validate it.
// ? the reason for this is because we want to be able to send requests to the gateway without knowing the actual target URL, and instead let the gateway construct the target URL.
if targetURL != "/:0" && !isValidURL(targetURL) {
log.Error().Msgf("Invalid target URL: %s", targetURL)
return
}
@@ -183,11 +185,6 @@ func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string,
transport.TLSClientConfig = tlsConfig
}
client := &http.Client{
Transport: transport,
Timeout: 30 * time.Second,
}
// Loop to handle multiple HTTP requests on the same stream
for {
req, err := http.ReadRequest(reader)
@@ -201,18 +198,56 @@ func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string,
}
log.Info().Msgf("Received HTTP request: %s", req.URL.Path)
actionHeader := req.Header.Get("x-infisical-action")
actionHeader := HttpProxyAction(req.Header.Get(INFISICAL_HTTP_PROXY_ACTION_HEADER))
if actionHeader != "" {
if actionHeader == "inject-k8s-sa-auth-token" {
token, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
if actionHeader == HttpProxyActionInjectGatewayK8sServiceAccountToken {
token, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)
if err != nil {
stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa auth token")))
continue // Continue to next request instead of returning
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(token)))
log.Info().Msgf("Injected gateway k8s SA auth token in request to %s", targetURL)
} else if actionHeader == HttpProxyActionUseGatewayK8sServiceAccount {
// set the ca cert to the pod's k8s service account ca cert:
caCert, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_CA_CERT_PATH)
if err != nil {
stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa ca cert")))
continue
}
caCertPool := x509.NewCertPool()
appendSuccess := caCertPool.AppendCertsFromPEM(caCert)
if !appendSuccess {
stream.Write([]byte(buildHttpInternalServerError("failed to parse k8s sa ca cert")))
continue
}
transport.TLSClientConfig = &tls.Config{
RootCAs: caCertPool,
}
// set authorization header to the pod's k8s service account token:
token, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)
if err != nil {
stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa auth token")))
continue
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(token)))
// update the target URL to point to the kubernetes API server:
kubernetesServiceHost := os.Getenv(KUBERNETES_SERVICE_HOST_ENV_NAME)
kubernetesServicePort := os.Getenv(KUBERNETES_SERVICE_PORT_HTTPS_ENV_NAME)
fullBaseUrl := fmt.Sprintf("https://%s:%s", kubernetesServiceHost, kubernetesServicePort)
targetURL = fullBaseUrl
log.Info().Msgf("Redirected request to Kubernetes API server: %s", targetURL)
}
req.Header.Del("x-infisical-action")
req.Header.Del(INFISICAL_HTTP_PROXY_ACTION_HEADER)
}
// Build full target URL
@@ -242,6 +277,11 @@ func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string,
log.Info().Msgf("Proxying %s %s to %s", req.Method, req.URL.Path, targetFullURL)
client := &http.Client{
Transport: transport,
Timeout: 30 * time.Second,
}
resp, err := client.Do(proxyReq)
if err != nil {
log.Error().Msgf("Failed to reach target: %v", err)

View File

@@ -0,0 +1,17 @@
package gateway
const (
KUBERNETES_SERVICE_HOST_ENV_NAME = "KUBERNETES_SERVICE_HOST"
KUBERNETES_SERVICE_PORT_HTTPS_ENV_NAME = "KUBERNETES_SERVICE_PORT_HTTPS"
KUBERNETES_SERVICE_ACCOUNT_CA_CERT_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token"
INFISICAL_HTTP_PROXY_ACTION_HEADER = "x-infisical-action"
)
type HttpProxyAction string
const (
HttpProxyActionInjectGatewayK8sServiceAccountToken HttpProxyAction = "inject-k8s-sa-auth-token"
HttpProxyActionUseGatewayK8sServiceAccount HttpProxyAction = "use-k8s-sa"
)

View File

@@ -403,7 +403,7 @@ export type IdentityKubernetesAuth = {
export type AddIdentityKubernetesAuthDTO = {
organizationId: string;
identityId: string;
kubernetesHost: string;
kubernetesHost: string | null;
tokenReviewerJwt?: string;
tokenReviewMode: IdentityKubernetesAuthTokenReviewMode;
allowedNamespaces: string;
@@ -422,7 +422,7 @@ export type AddIdentityKubernetesAuthDTO = {
export type UpdateIdentityKubernetesAuthDTO = {
organizationId: string;
identityId: string;
kubernetesHost?: string;
kubernetesHost?: string | null;
tokenReviewerJwt?: string | null;
tokenReviewMode?: IdentityKubernetesAuthTokenReviewMode;
allowedNamespaces?: string;

View File

@@ -46,13 +46,13 @@ const schema = z
tokenReviewMode: z
.nativeEnum(IdentityKubernetesAuthTokenReviewMode)
.default(IdentityKubernetesAuthTokenReviewMode.Api),
kubernetesHost: z.string().min(1),
kubernetesHost: z.string().optional().nullable(),
tokenReviewerJwt: z.string().optional(),
gatewayId: z.string().optional().nullable(),
allowedNames: z.string(),
allowedNamespaces: z.string(),
allowedAudience: z.string(),
caCert: z.string(),
caCert: z.string().optional(),
accessTokenTTL: z.string().refine((val) => Number(val) <= 315360000, {
message: "Access Token TTL cannot be greater than 315360000"
}),
@@ -69,6 +69,17 @@ const schema = z
.min(1)
})
.superRefine((data, ctx) => {
if (
data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api &&
!data.kubernetesHost?.length
) {
ctx.addIssue({
path: ["kubernetesHost"],
code: z.ZodIssueCode.custom,
message: "When token review mode is set to API, a Kubernetes host must be provided"
});
}
if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && !data.gatewayId) {
ctx.addIssue({
path: ["gatewayId"],
@@ -201,7 +212,13 @@ export const IdentityKubernetesAuthForm = ({
if (data) {
await updateMutateAsync({
organizationId: orgId,
kubernetesHost,
...(tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api
? {
kubernetesHost: kubernetesHost || ""
}
: {
kubernetesHost: null
}),
tokenReviewerJwt: tokenReviewerJwt || null,
allowedNames,
allowedNamespaces,
@@ -219,7 +236,13 @@ export const IdentityKubernetesAuthForm = ({
await addMutateAsync({
organizationId: orgId,
identityId,
kubernetesHost: kubernetesHost || "",
...(tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api
? {
kubernetesHost: kubernetesHost || ""
}
: {
kubernetesHost: null
}),
tokenReviewerJwt: tokenReviewerJwt || undefined,
allowedNames: allowedNames || "",
allowedNamespaces: allowedNamespaces || "",
@@ -278,23 +301,6 @@ export const IdentityKubernetesAuthForm = ({
<Tab value={IdentityFormTab.Advanced}>Advanced</Tab>
</TabList>
<TabPanel value={IdentityFormTab.Configuration}>
<Controller
control={control}
defaultValue="2592000"
name="kubernetesHost"
render={({ field, fieldState: { error } }) => (
<FormControl
label="Kubernetes Host / Base Kubernetes API URL "
isError={Boolean(error)}
errorText={error?.message}
tooltipText="The host string, host:port pair, or URL to the base of the Kubernetes API server. This can usually be obtained by running 'kubectl cluster-info'"
isRequired
>
<Input {...field} placeholder="https://my-example-k8s-api-host.com" type="text" />
</FormControl>
)}
/>
<div className="flex w-full items-center gap-2">
<div className="w-full flex-1">
<OrgPermissionCan
@@ -383,8 +389,31 @@ export const IdentityKubernetesAuthForm = ({
)}
/>
</div>
{tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && (
<Controller
control={control}
defaultValue="2592000"
name="kubernetesHost"
render={({ field, fieldState: { error } }) => (
<FormControl
label="Kubernetes Host / Base Kubernetes API URL "
isError={Boolean(error)}
errorText={error?.message}
tooltipText="The host string, host:port pair, or URL to the base of the Kubernetes API server. This can usually be obtained by running 'kubectl cluster-info'"
isRequired
>
<Input
{...field}
placeholder="https://my-example-k8s-api-host.com"
type="text"
value={field.value || ""}
/>
</FormControl>
)}
/>
)}
{tokenReviewMode === "api" && (
{tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && (
<Controller
control={control}
name="tokenReviewerJwt"
@@ -493,20 +522,22 @@ export const IdentityKubernetesAuthForm = ({
</FormControl>
)}
/>
<Controller
control={control}
name="caCert"
render={({ field, fieldState: { error } }) => (
<FormControl
label="CA Certificate"
errorText={error?.message}
isError={Boolean(error)}
tooltipText="An optional PEM-encoded CA cert for the Kubernetes API server. This is used by the TLS client for secure communication with the Kubernetes API server."
>
<TextArea {...field} placeholder="-----BEGIN CERTIFICATE----- ..." />
</FormControl>
)}
/>
{tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && (
<Controller
control={control}
name="caCert"
render={({ field, fieldState: { error } }) => (
<FormControl
label="CA Certificate"
errorText={error?.message}
isError={Boolean(error)}
tooltipText="An optional PEM-encoded CA cert for the Kubernetes API server. This is used by the TLS client for secure communication with the Kubernetes API server."
>
<TextArea {...field} placeholder="-----BEGIN CERTIFICATE----- ..." />
</FormControl>
)}
/>
)}
{accessTokenTrustedIpsFields.map(({ id }, index) => (
<div className="mb-3 flex items-end space-x-2" key={id}>
<Controller

View File

@@ -1,6 +1,13 @@
## 0.0.41 (June 10, 2025)
* Added new gateway action for fully off-loading CA certificate, cluster URL, and token management to the gateway.
* Structural improvements
## 0.0.4 (June 7th, 2025)
* Improvements to HTTP proxy error handling.
## 0.0.3 (June 6, 2025)
* Minor fix for handling malformed URLs for HTTP forwarding
* Minor fix for handling malformed URLs for HTTP forwarding.
## 0.0.2 (June 6, 2025)
@@ -9,4 +16,4 @@
## 0.0.1 (May 1, 2025)
* Initial helm release
* Initial helm release.

View File

@@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.0.4
version: 0.0.5
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "0.0.4"
appVersion: "0.0.5"

View File

@@ -1,6 +1,6 @@
image:
pullPolicy: IfNotPresent
tag: "0.41.83"
tag: "0.41.84"
secret:
# The secret that contains the environment variables to be used by the gateway, such as INFISICAL_API_URL and TOKEN