Merge pull request #2760 from Infisical/daniel/request-ids

feat: request ID support
This commit is contained in:
Daniel Hougaard
2024-11-25 16:13:19 +04:00
committed by GitHub
20 changed files with 210 additions and 86 deletions

View File

@ -24,6 +24,7 @@
"@fastify/multipart": "8.3.0",
"@fastify/passport": "^2.4.0",
"@fastify/rate-limit": "^9.0.0",
"@fastify/request-context": "^5.1.0",
"@fastify/session": "^10.7.0",
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
@ -5528,6 +5529,15 @@
"toad-cache": "^3.3.0"
}
},
"node_modules/@fastify/request-context": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@fastify/request-context/-/request-context-5.1.0.tgz",
"integrity": "sha512-PM7wrLJOEylVDpxabOFLaYsdAiaa0lpDUcP2HMFJ1JzgiWuC6k4r3duf6Pm9YLnzlGmT+Yp4tkQjqsu7V/pSOA==",
"license": "MIT",
"dependencies": {
"fastify-plugin": "^4.0.0"
}
},
"node_modules/@fastify/send": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@fastify/send/-/send-2.1.0.tgz",

View File

@ -132,6 +132,7 @@
"@fastify/multipart": "8.3.0",
"@fastify/passport": "^2.4.0",
"@fastify/rate-limit": "^9.0.0",
"@fastify/request-context": "^5.1.0",
"@fastify/session": "^10.7.0",
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",

View File

@ -0,0 +1,7 @@
import "@fastify/request-context";
declare module "@fastify/request-context" {
interface RequestContextData {
requestId: string;
}
}

View File

@ -1,6 +1,6 @@
import { FastifyInstance, RawReplyDefaultExpression, RawRequestDefaultExpression, RawServerDefault } from "fastify";
import { Logger } from "pino";
import { CustomLogger } from "@app/lib/logger/logger";
import { ZodTypeProvider } from "@app/server/plugins/fastify-zod";
declare global {
@ -8,7 +8,7 @@ declare global {
RawServerDefault,
RawRequestDefaultExpression<RawServerDefault>,
RawReplyDefaultExpression<RawServerDefault>,
Readonly<Logger>,
Readonly<CustomLogger>,
ZodTypeProvider
>;

View File

@ -27,7 +27,7 @@ export const initializeHsmModule = () => {
logger.info("PKCS#11 module initialized");
} catch (err) {
logger.error("Failed to initialize PKCS#11 module:", err);
logger.error(err, "Failed to initialize PKCS#11 module");
throw err;
}
};
@ -39,7 +39,7 @@ export const initializeHsmModule = () => {
isInitialized = false;
logger.info("PKCS#11 module finalized");
} catch (err) {
logger.error("Failed to finalize PKCS#11 module:", err);
logger.error(err, "Failed to finalize PKCS#11 module");
throw err;
}
}

View File

@ -36,8 +36,7 @@ export const testLDAPConfig = async (ldapConfig: TLDAPConfig): Promise<boolean>
});
ldapClient.on("error", (err) => {
logger.error("LDAP client error:", err);
logger.error(err);
logger.error(err, "LDAP client error");
resolve(false);
});

View File

@ -161,8 +161,8 @@ export const licenseServiceFactory = ({
}
} catch (error) {
logger.error(
`getPlan: encountered an error when fetching pan [orgId=${orgId}] [projectId=${projectId}] [error]`,
error
error,
`getPlan: encountered an error when fetching pan [orgId=${orgId}] [projectId=${projectId}] [error]`
);
await keyStore.setItemWithExpiry(
FEATURE_CACHE_KEY(orgId),

View File

@ -46,7 +46,7 @@ export const rateLimitServiceFactory = ({ rateLimitDAL, licenseService }: TRateL
}
return rateLimit;
} catch (err) {
logger.error("Error fetching rate limits %o", err);
logger.error(err, "Error fetching rate limits");
return undefined;
}
};
@ -69,12 +69,12 @@ export const rateLimitServiceFactory = ({ rateLimitDAL, licenseService }: TRateL
mfaRateLimit: rateLimit.mfaRateLimit
};
logger.info(`syncRateLimitConfiguration: rate limit configuration: %o`, newRateLimitMaxConfiguration);
logger.info(newRateLimitMaxConfiguration, "syncRateLimitConfiguration: rate limit configuration");
Object.freeze(newRateLimitMaxConfiguration);
rateLimitMaxConfiguration = newRateLimitMaxConfiguration;
}
} catch (error) {
logger.error(`Error syncing rate limit configurations: %o`, error);
logger.error(error, "Error syncing rate limit configurations");
}
};

View File

@ -238,11 +238,11 @@ export const secretScanningQueueFactory = ({
});
queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => {
logger.error("Failed to secret scan on push", job?.data, err);
logger.error(err, "Failed to secret scan on push", job?.data);
});
queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => {
logger.error("Failed to do full repo secret scan", job?.data, err);
logger.error(err, "Failed to do full repo secret scan", job?.data);
});
return { startFullRepoScan, startPushEventScan };

View File

@ -1,7 +1,7 @@
import { Logger } from "pino";
import { z } from "zod";
import { removeTrailingSlash } from "../fn";
import { CustomLogger } from "../logger/logger";
import { zpStr } from "../zod";
export const GITLAB_URL = "https://gitlab.com";
@ -212,7 +212,7 @@ let envCfg: Readonly<z.infer<typeof envSchema>>;
export const getConfig = () => envCfg;
// cannot import singleton logger directly as it needs config to load various transport
export const initEnvConfig = (logger?: Logger) => {
export const initEnvConfig = (logger?: CustomLogger) => {
const parsedEnv = envSchema.safeParse(process.env);
if (!parsedEnv.success) {
(logger ?? console).error("Invalid environment variables. Check the error below");

View File

@ -1,6 +1,8 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
// logger follows a singleton pattern
// easier to use it that's all.
import { requestContext } from "@fastify/request-context";
import pino, { Logger } from "pino";
import { z } from "zod";
@ -13,14 +15,37 @@ const logLevelToSeverityLookup: Record<string, string> = {
"60": "CRITICAL"
};
// eslint-disable-next-line import/no-mutable-exports
export let logger: Readonly<Logger>;
// akhilmhdh:
// The logger is not placed in the main app config to avoid a circular dependency.
// The config requires the logger to display errors when an invalid environment is supplied.
// On the other hand, the logger needs the config to obtain credentials for AWS or other transports.
// By keeping the logger separate, it becomes an independent package.
// We define our own custom logger interface to enforce structure to the logging methods.
export interface CustomLogger extends Omit<Logger, "info" | "error" | "warn" | "debug"> {
info: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(obj: unknown, msg?: string, ...args: any[]): void;
};
error: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(obj: unknown, msg?: string, ...args: any[]): void;
};
warn: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(obj: unknown, msg?: string, ...args: any[]): void;
};
debug: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(obj: unknown, msg?: string, ...args: any[]): void;
};
}
// eslint-disable-next-line import/no-mutable-exports
export let logger: Readonly<CustomLogger>;
const loggerConfig = z.object({
AWS_CLOUDWATCH_LOG_GROUP_NAME: z.string().default("infisical-log-stream"),
AWS_CLOUDWATCH_LOG_REGION: z.string().default("us-east-1"),
@ -62,6 +87,17 @@ const redactedKeys = [
"config"
];
const UNKNOWN_REQUEST_ID = "UNKNOWN_REQUEST_ID";
const extractRequestId = () => {
try {
return requestContext.get("requestId") || UNKNOWN_REQUEST_ID;
} catch (err) {
console.log("failed to get request context", err);
return UNKNOWN_REQUEST_ID;
}
};
export const initLogger = async () => {
const cfg = loggerConfig.parse(process.env);
const targets: pino.TransportMultiOptions["targets"][number][] = [
@ -94,6 +130,30 @@ export const initLogger = async () => {
targets
});
const wrapLogger = (originalLogger: Logger): CustomLogger => {
// eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any
originalLogger.info = (obj: unknown, msg?: string, ...args: any[]) => {
return originalLogger.child({ requestId: extractRequestId() }).info(obj, msg, ...args);
};
// eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any
originalLogger.error = (obj: unknown, msg?: string, ...args: any[]) => {
return originalLogger.child({ requestId: extractRequestId() }).error(obj, msg, ...args);
};
// eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any
originalLogger.warn = (obj: unknown, msg?: string, ...args: any[]) => {
return originalLogger.child({ requestId: extractRequestId() }).warn(obj, msg, ...args);
};
// eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any
originalLogger.debug = (obj: unknown, msg?: string, ...args: any[]) => {
return originalLogger.child({ requestId: extractRequestId() }).debug(obj, msg, ...args);
};
return originalLogger;
};
logger = pino(
{
mixin(_context, level) {
@ -113,5 +173,6 @@ export const initLogger = async () => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
transport
);
return logger;
return wrapLogger(logger);
};

View File

@ -10,13 +10,15 @@ import fastifyFormBody from "@fastify/formbody";
import helmet from "@fastify/helmet";
import type { FastifyRateLimitOptions } from "@fastify/rate-limit";
import ratelimiter from "@fastify/rate-limit";
import { fastifyRequestContext } from "@fastify/request-context";
import fastify from "fastify";
import { Knex } from "knex";
import { Logger } from "pino";
import { HsmModule } from "@app/ee/services/hsm/hsm-types";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig, IS_PACKAGED } from "@app/lib/config/env";
import { CustomLogger } from "@app/lib/logger/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TQueueServiceFactory } from "@app/queue";
import { TSmtpService } from "@app/services/smtp/smtp-service";
@ -35,7 +37,7 @@ type TMain = {
auditLogDb?: Knex;
db: Knex;
smtp: TSmtpService;
logger?: Logger;
logger?: CustomLogger;
queue: TQueueServiceFactory;
keyStore: TKeyStoreFactory;
hsmModule: HsmModule;
@ -47,7 +49,9 @@ export const main = async ({ db, hsmModule, auditLogDb, smtp, logger, queue, key
const server = fastify({
logger: appCfg.NODE_ENV === "test" ? false : logger,
genReqId: () => `req-${alphaNumericNanoId(14)}`,
trustProxy: true,
connectionTimeout: appCfg.isHsmConfigured ? 90_000 : 30_000,
ignoreTrailingSlash: true,
pluginTimeout: 40_000
@ -104,6 +108,13 @@ export const main = async ({ db, hsmModule, auditLogDb, smtp, logger, queue, key
await server.register(maintenanceMode);
await server.register(fastifyRequestContext, {
defaultStoreValues: (request) => ({
requestId: request.id,
log: request.log.child({ requestId: request.id })
})
});
await server.register(registerRoutes, { smtp, queue, db, auditLogDb, keyStore, hsmModule });
if (appCfg.isProductionMode) {

View File

@ -39,29 +39,42 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
if (error instanceof BadRequestError) {
void res
.status(HttpStatusCodes.BadRequest)
.send({ statusCode: HttpStatusCodes.BadRequest, message: error.message, error: error.name });
.send({ requestId: req.id, statusCode: HttpStatusCodes.BadRequest, message: error.message, error: error.name });
} else if (error instanceof NotFoundError) {
void res
.status(HttpStatusCodes.NotFound)
.send({ statusCode: HttpStatusCodes.NotFound, message: error.message, error: error.name });
.send({ requestId: req.id, statusCode: HttpStatusCodes.NotFound, message: error.message, error: error.name });
} else if (error instanceof UnauthorizedError) {
void res
.status(HttpStatusCodes.Unauthorized)
.send({ statusCode: HttpStatusCodes.Unauthorized, message: error.message, error: error.name });
void res.status(HttpStatusCodes.Unauthorized).send({
requestId: req.id,
statusCode: HttpStatusCodes.Unauthorized,
message: error.message,
error: error.name
});
} else if (error instanceof DatabaseError || error instanceof InternalServerError) {
void res
.status(HttpStatusCodes.InternalServerError)
.send({ statusCode: HttpStatusCodes.InternalServerError, message: "Something went wrong", error: error.name });
void res.status(HttpStatusCodes.InternalServerError).send({
requestId: req.id,
statusCode: HttpStatusCodes.InternalServerError,
message: "Something went wrong",
error: error.name
});
} else if (error instanceof GatewayTimeoutError) {
void res
.status(HttpStatusCodes.GatewayTimeout)
.send({ statusCode: HttpStatusCodes.GatewayTimeout, message: error.message, error: error.name });
void res.status(HttpStatusCodes.GatewayTimeout).send({
requestId: req.id,
statusCode: HttpStatusCodes.GatewayTimeout,
message: error.message,
error: error.name
});
} else if (error instanceof ZodError) {
void res
.status(HttpStatusCodes.Unauthorized)
.send({ statusCode: HttpStatusCodes.Unauthorized, error: "ValidationFailure", message: error.issues });
void res.status(HttpStatusCodes.Unauthorized).send({
requestId: req.id,
statusCode: HttpStatusCodes.Unauthorized,
error: "ValidationFailure",
message: error.issues
});
} else if (error instanceof ForbiddenError) {
void res.status(HttpStatusCodes.Forbidden).send({
requestId: req.id,
statusCode: HttpStatusCodes.Forbidden,
error: "PermissionDenied",
message: `You are not allowed to ${error.action} on ${error.subjectType}`,
@ -74,48 +87,54 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
});
} else if (error instanceof ForbiddenRequestError) {
void res.status(HttpStatusCodes.Forbidden).send({
requestId: req.id,
statusCode: HttpStatusCodes.Forbidden,
message: error.message,
error: error.name
});
} else if (error instanceof RateLimitError) {
void res.status(HttpStatusCodes.TooManyRequests).send({
requestId: req.id,
statusCode: HttpStatusCodes.TooManyRequests,
message: error.message,
error: error.name
});
} else if (error instanceof ScimRequestError) {
void res.status(error.status).send({
requestId: req.id,
schemas: error.schemas,
status: error.status,
detail: error.detail
});
} else if (error instanceof OidcAuthError) {
void res
.status(HttpStatusCodes.InternalServerError)
.send({ statusCode: HttpStatusCodes.InternalServerError, message: error.message, error: error.name });
void res.status(HttpStatusCodes.InternalServerError).send({
requestId: req.id,
statusCode: HttpStatusCodes.InternalServerError,
message: error.message,
error: error.name
});
} else if (error instanceof jwt.JsonWebTokenError) {
const message = (() => {
if (error.message === JWTErrors.JwtExpired) {
return "Your token has expired. Please re-authenticate.";
}
if (error.message === JWTErrors.JwtMalformed) {
return "The provided access token is malformed. Please use a valid token or generate a new one and try again.";
}
if (error.message === JWTErrors.InvalidAlgorithm) {
return "The access token is signed with an invalid algorithm. Please provide a valid token and try again.";
}
let errorMessage = error.message;
return error.message;
})();
if (error.message === JWTErrors.JwtExpired) {
errorMessage = "Your token has expired. Please re-authenticate.";
} else if (error.message === JWTErrors.JwtMalformed) {
errorMessage =
"The provided access token is malformed. Please use a valid token or generate a new one and try again.";
} else if (error.message === JWTErrors.InvalidAlgorithm) {
errorMessage =
"The access token is signed with an invalid algorithm. Please provide a valid token and try again.";
}
void res.status(HttpStatusCodes.Forbidden).send({
requestId: req.id,
statusCode: HttpStatusCodes.Forbidden,
error: "TokenError",
message
message: errorMessage
});
} else {
void res.status(HttpStatusCodes.InternalServerError).send({
requestId: req.id,
statusCode: HttpStatusCodes.InternalServerError,
error: "InternalServerError",
message: "Something went wrong"

View File

@ -19,7 +19,7 @@ export const registerSecretScannerGhApp = async (server: FastifyZodProvider) =>
app.on("installation", async (context) => {
const { payload } = context;
logger.info("Installed secret scanner to:", { repositories: payload.repositories });
logger.info({ repositories: payload.repositories }, "Installed secret scanner to");
});
app.on("push", async (context) => {

View File

@ -30,27 +30,32 @@ export const integrationAuthPubSchema = IntegrationAuthsSchema.pick({
export const DefaultResponseErrorsSchema = {
400: z.object({
requestId: z.string(),
statusCode: z.literal(400),
message: z.string(),
error: z.string()
}),
404: z.object({
requestId: z.string(),
statusCode: z.literal(404),
message: z.string(),
error: z.string()
}),
401: z.object({
requestId: z.string(),
statusCode: z.literal(401),
message: z.any(),
error: z.string()
}),
403: z.object({
requestId: z.string(),
statusCode: z.literal(403),
message: z.string(),
details: z.any().optional(),
error: z.string()
}),
500: z.object({
requestId: z.string(),
statusCode: z.literal(500),
message: z.string(),
error: z.string()

View File

@ -285,11 +285,14 @@ export const projectQueueFactory = ({
if (!orgMembership) {
// This can happen. Since we don't remove project memberships and project keys when a user is removed from an org, this is a valid case.
logger.info("User is not in organization", {
userId: key.receiverId,
orgId: project.orgId,
projectId: project.id
});
logger.info(
{
userId: key.receiverId,
orgId: project.orgId,
projectId: project.id
},
"User is not in organization"
);
// eslint-disable-next-line no-continue
continue;
}
@ -551,10 +554,10 @@ export const projectQueueFactory = ({
.catch(() => [null]);
if (!project) {
logger.error("Failed to upgrade project, because no project was found", data);
logger.error(data, "Failed to upgrade project, because no project was found");
} else {
await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.Failed);
logger.error("Failed to upgrade project", err, {
logger.error(err, "Failed to upgrade project", {
extra: {
project,
jobData: data

View File

@ -142,7 +142,7 @@ export const fnTriggerWebhook = async ({
!isDisabled && picomatch.isMatch(secretPath, hookSecretPath, { strictSlashes: false })
);
if (!toBeTriggeredHooks.length) return;
logger.info("Secret webhook job started", { environment, secretPath, projectId });
logger.info({ environment, secretPath, projectId }, "Secret webhook job started");
const project = await projectDAL.findById(projectId);
const webhooksTriggered = await Promise.allSettled(
toBeTriggeredHooks.map((hook) =>
@ -195,5 +195,5 @@ export const fnTriggerWebhook = async ({
);
}
});
logger.info("Secret webhook job ended", { environment, secretPath, projectId });
logger.info({ environment, secretPath, projectId }, "Secret webhook job ended");
};

View File

@ -51,17 +51,26 @@ export enum ApiErrorTypes {
export type TApiErrors =
| {
requestId: string;
error: ApiErrorTypes.ValidationError;
message: ZodIssue[];
statusCode: 401;
}
| {
requestId: string;
error: ApiErrorTypes.UnauthorizedError;
message: string;
statusCode: 401;
}
| {
requestId: string;
error: ApiErrorTypes.ForbiddenError;
message: string;
details: PureAbility["rules"];
statusCode: 403;
}
| {
requestId: string;
statusCode: 400;
message: string;
error: ApiErrorTypes.BadRequestError;

View File

@ -32,7 +32,12 @@ export const queryClient = new QueryClient({
{
title: "Validation Error",
type: "error",
text: "Please check the input and try again.",
text: (
<div>
<p>Please check the input and try again.</p>
<p className="mt-2 text-xs">Request ID: {serverResponse.requestId}</p>
</div>
),
children: (
<Modal>
<ModalTrigger>
@ -72,7 +77,8 @@ export const queryClient = new QueryClient({
{
title: "Forbidden Access",
type: "error",
text: serverResponse.message,
text: `${serverResponse.message} [requestId=${serverResponse.requestId}]`,
children: serverResponse?.details?.length ? (
<Modal>
<ModalTrigger>
@ -165,7 +171,11 @@ export const queryClient = new QueryClient({
);
return;
}
createNotification({ title: "Bad Request", type: "error", text: serverResponse.message });
createNotification({
title: "Bad Request",
type: "error",
text: `${serverResponse.message} [requestId=${serverResponse.requestId}]`
});
}
}
}),

View File

@ -63,32 +63,21 @@ export const IdentityClientSecretModal = ({ popUp, handlePopUpToggle }: Props) =
};
const onFormSubmit = async ({ description, ttl, numUsesLimit }: FormData) => {
try {
const { clientSecret } = await createClientSecret({
identityId: popUpData.identityId,
description,
ttl: Number(ttl),
numUsesLimit: Number(numUsesLimit)
});
const { clientSecret } = await createClientSecret({
identityId: popUpData.identityId,
description,
ttl: Number(ttl),
numUsesLimit: Number(numUsesLimit)
});
setToken(clientSecret);
setToken(clientSecret);
createNotification({
text: "Successfully created client secret",
type: "success"
});
createNotification({
text: "Successfully created client secret",
type: "success"
});
reset();
} catch (err) {
console.error(err);
const error = err as any;
const text = error?.response?.data?.message ?? "Failed to create client secret";
createNotification({
text,
type: "error"
});
}
reset();
};
return (