Compare commits

..

1 Commits

Author SHA1 Message Date
Daniel Hougaard
8f5f56a25e improvement: add composite index for identityId/authMethod on identity access tokens 2024-11-19 22:51:47 +04:00
27 changed files with 81 additions and 1880 deletions

View File

@@ -74,14 +74,6 @@ CAPTCHA_SECRET=
NEXT_PUBLIC_CAPTCHA_SITE_KEY=
OTEL_TELEMETRY_COLLECTION_ENABLED=
OTEL_EXPORT_TYPE=
OTEL_EXPORT_OTLP_ENDPOINT=
OTEL_OTLP_PUSH_INTERVAL=
OTEL_COLLECTOR_BASIC_AUTH_USERNAME=
OTEL_COLLECTOR_BASIC_AUTH_PASSWORD=
PLAIN_API_KEY=
PLAIN_WISH_LABEL_IDS=

View File

@@ -10,9 +10,6 @@ up-dev:
up-dev-ldap:
docker compose -f docker-compose.dev.yml --profile ldap up --build
up-dev-metrics:
docker compose -f docker-compose.dev.yml --profile metrics up --build
up-prod:
docker-compose -f docker-compose.prod.yml up --build
@@ -30,3 +27,4 @@ reviewable-api:
npm run type:check
reviewable: reviewable-ui reviewable-api

1606
backend/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -140,14 +140,6 @@
"@octokit/plugin-retry": "^5.0.5",
"@octokit/rest": "^20.0.2",
"@octokit/webhooks-types": "^7.3.1",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.53.0",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.55.0",
"@opentelemetry/exporter-prometheus": "^0.55.0",
"@opentelemetry/instrumentation": "^0.55.0",
"@opentelemetry/resources": "^1.28.0",
"@opentelemetry/sdk-metrics": "^1.28.0",
"@opentelemetry/semantic-conventions": "^1.27.0",
"@peculiar/asn1-schema": "^2.3.8",
"@peculiar/x509": "^1.12.1",
"@serdnam/pino-cloudwatch-transport": "^1.0.4",

View File

@@ -0,0 +1,25 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasIdentityIdColumn = await knex.schema.hasColumn(TableName.IdentityAccessToken, "identityId");
const hasAuthMethodColumn = await knex.schema.hasColumn(TableName.IdentityAccessToken, "authMethod");
await knex.schema.alterTable(TableName.IdentityAccessToken, (t) => {
if (hasIdentityIdColumn && hasAuthMethodColumn) {
t.index(["authMethod", "identityId"]);
}
});
}
export async function down(knex: Knex): Promise<void> {
const hasIdentityIdColumn = await knex.schema.hasColumn(TableName.IdentityAccessToken, "identityId");
const hasAuthMethodColumn = await knex.schema.hasColumn(TableName.IdentityAccessToken, "authMethod");
await knex.schema.alterTable(TableName.IdentityAccessToken, (t) => {
if (hasIdentityIdColumn && hasAuthMethodColumn) {
t.dropIndex(["authMethod", "identityId"]);
}
});
}

View File

@@ -157,15 +157,6 @@ const envSchema = z
INFISICAL_CLOUD: zodStrBool.default("false"),
MAINTENANCE_MODE: zodStrBool.default("false"),
CAPTCHA_SECRET: zpStr(z.string().optional()),
// TELEMETRY
OTEL_TELEMETRY_COLLECTION_ENABLED: zodStrBool.default("false"),
OTEL_EXPORT_OTLP_ENDPOINT: zpStr(z.string().optional()),
OTEL_OTLP_PUSH_INTERVAL: z.coerce.number().default(30000),
OTEL_COLLECTOR_BASIC_AUTH_USERNAME: zpStr(z.string().optional()),
OTEL_COLLECTOR_BASIC_AUTH_PASSWORD: zpStr(z.string().optional()),
OTEL_EXPORT_TYPE: z.enum(["prometheus", "otlp"]).optional(),
PLAIN_API_KEY: zpStr(z.string().optional()),
PLAIN_WISH_LABEL_IDS: zpStr(z.string().optional()),
DISABLE_AUDIT_LOG_GENERATION: zodStrBool.default("false"),
@@ -212,11 +203,11 @@ let envCfg: Readonly<z.infer<typeof envSchema>>;
export const getConfig = () => envCfg;
// cannot import singleton logger directly as it needs config to load various transport
export const initEnvConfig = (logger?: Logger) => {
export const initEnvConfig = (logger: Logger) => {
const parsedEnv = envSchema.safeParse(process.env);
if (!parsedEnv.success) {
(logger ?? console).error("Invalid environment variables. Check the error below");
(logger ?? console).error(parsedEnv.error.issues);
logger.error("Invalid environment variables. Check the error below");
logger.error(parsedEnv.error.issues);
process.exit(-1);
}

View File

@@ -1,91 +0,0 @@
import opentelemetry, { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
import { registerInstrumentations } from "@opentelemetry/instrumentation";
import { Resource } from "@opentelemetry/resources";
import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
import dotenv from "dotenv";
import { initEnvConfig } from "../config/env";
dotenv.config();
const initTelemetryInstrumentation = ({
exportType,
otlpURL,
otlpUser,
otlpPassword,
otlpPushInterval
}: {
exportType?: string;
otlpURL?: string;
otlpUser?: string;
otlpPassword?: string;
otlpPushInterval?: number;
}) => {
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);
const resource = Resource.default().merge(
new Resource({
[ATTR_SERVICE_NAME]: "infisical-core",
[ATTR_SERVICE_VERSION]: "0.1.0"
})
);
const metricReaders = [];
switch (exportType) {
case "prometheus": {
const promExporter = new PrometheusExporter();
metricReaders.push(promExporter);
break;
}
case "otlp": {
const otlpExporter = new OTLPMetricExporter({
url: `${otlpURL}/v1/metrics`,
headers: {
Authorization: `Basic ${btoa(`${otlpUser}:${otlpPassword}`)}`
},
temporalityPreference: AggregationTemporality.DELTA
});
metricReaders.push(
new PeriodicExportingMetricReader({
exporter: otlpExporter,
exportIntervalMillis: otlpPushInterval
})
);
break;
}
default:
throw new Error("Invalid OTEL export type");
}
const meterProvider = new MeterProvider({
resource,
readers: metricReaders
});
opentelemetry.metrics.setGlobalMeterProvider(meterProvider);
registerInstrumentations({
instrumentations: [getNodeAutoInstrumentations()]
});
};
const setupTelemetry = () => {
const appCfg = initEnvConfig();
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
console.log("Initializing telemetry instrumentation");
initTelemetryInstrumentation({
otlpURL: appCfg.OTEL_EXPORT_OTLP_ENDPOINT,
otlpUser: appCfg.OTEL_COLLECTOR_BASIC_AUTH_USERNAME,
otlpPassword: appCfg.OTEL_COLLECTOR_BASIC_AUTH_PASSWORD,
otlpPushInterval: appCfg.OTEL_OTLP_PUSH_INTERVAL,
exportType: appCfg.OTEL_EXPORT_TYPE
});
}
};
void setupTelemetry();

View File

@@ -1,5 +1,3 @@
import "./lib/telemetry/instrumentation";
import dotenv from "dotenv";
import path from "path";
@@ -20,7 +18,6 @@ dotenv.config();
const run = async () => {
const logger = await initLogger();
const appCfg = initEnvConfig(logger);
const db = initDbConnection({
dbConnectionUri: appCfg.DB_CONNECTION_URI,
dbRootCert: appCfg.DB_ROOT_CERT,

View File

@@ -22,7 +22,6 @@ import { TSmtpService } from "@app/services/smtp/smtp-service";
import { globalRateLimiterCfg } from "./config/rateLimiter";
import { addErrorsToResponseSchemas } from "./plugins/add-errors-to-response-schemas";
import { apiMetrics } from "./plugins/api-metrics";
import { fastifyErrHandler } from "./plugins/error-handler";
import { registerExternalNextjs } from "./plugins/external-nextjs";
import { serializerCompiler, validatorCompiler, ZodTypeProvider } from "./plugins/fastify-zod";
@@ -87,10 +86,6 @@ export const main = async ({ db, hsmModule, auditLogDb, smtp, logger, queue, key
// pull ip based on various proxy headers
await server.register(fastifyIp);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
await server.register(apiMetrics);
}
await server.register(fastifySwagger);
await server.register(fastifyFormBody);
await server.register(fastifyErrHandler);

View File

@@ -1,21 +0,0 @@
import opentelemetry from "@opentelemetry/api";
import fp from "fastify-plugin";
export const apiMetrics = fp(async (fastify) => {
const apiMeter = opentelemetry.metrics.getMeter("API");
const latencyHistogram = apiMeter.createHistogram("API_latency", {
unit: "ms"
});
fastify.addHook("onResponse", async (request, reply) => {
const { method } = request;
const route = request.routerPath;
const { statusCode } = reply;
latencyHistogram.record(reply.elapsedTime, {
route,
method,
statusCode
});
});
});

View File

@@ -29,7 +29,7 @@ import {
} from "./identity-aws-auth-types";
type TIdentityAwsAuthServiceFactoryDep = {
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create" | "delete">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create">;
identityAwsAuthDAL: Pick<TIdentityAwsAuthDALFactory, "findOne" | "transaction" | "create" | "updateById" | "delete">;
identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "findOne">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
@@ -346,8 +346,6 @@ export const identityAwsAuthServiceFactory = ({
const revokedIdentityAwsAuth = await identityAwsAuthDAL.transaction(async (tx) => {
const deletedAwsAuth = await identityAwsAuthDAL.delete({ identityId }, tx);
await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.AWS_AUTH }, tx);
return { ...deletedAwsAuth?.[0], orgId: identityMembershipOrg.orgId };
});
return revokedIdentityAwsAuth;

View File

@@ -30,7 +30,7 @@ type TIdentityAzureAuthServiceFactoryDep = {
"findOne" | "transaction" | "create" | "updateById" | "delete"
>;
identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "findOne">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create" | "delete">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create">;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
};
@@ -319,8 +319,6 @@ export const identityAzureAuthServiceFactory = ({
const revokedIdentityAzureAuth = await identityAzureAuthDAL.transaction(async (tx) => {
const deletedAzureAuth = await identityAzureAuthDAL.delete({ identityId }, tx);
await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.AZURE_AUTH }, tx);
return { ...deletedAzureAuth?.[0], orgId: identityMembershipOrg.orgId };
});
return revokedIdentityAzureAuth;

View File

@@ -28,7 +28,7 @@ import {
type TIdentityGcpAuthServiceFactoryDep = {
identityGcpAuthDAL: Pick<TIdentityGcpAuthDALFactory, "findOne" | "transaction" | "create" | "updateById" | "delete">;
identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "findOne">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create" | "delete">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create">;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
};
@@ -365,8 +365,6 @@ export const identityGcpAuthServiceFactory = ({
const revokedIdentityGcpAuth = await identityGcpAuthDAL.transaction(async (tx) => {
const deletedGcpAuth = await identityGcpAuthDAL.delete({ identityId }, tx);
await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.GCP_AUTH }, tx);
return { ...deletedGcpAuth?.[0], orgId: identityMembershipOrg.orgId };
});
return revokedIdentityGcpAuth;

View File

@@ -41,7 +41,7 @@ type TIdentityKubernetesAuthServiceFactoryDep = {
TIdentityKubernetesAuthDALFactory,
"create" | "findOne" | "transaction" | "updateById" | "delete"
>;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create" | "delete">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create">;
identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "findOne" | "findById">;
orgBotDAL: Pick<TOrgBotDALFactory, "findOne" | "transaction" | "create">;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
@@ -622,7 +622,6 @@ export const identityKubernetesAuthServiceFactory = ({
const revokedIdentityKubernetesAuth = await identityKubernetesAuthDAL.transaction(async (tx) => {
const deletedKubernetesAuth = await identityKubernetesAuthDAL.delete({ identityId }, tx);
await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.KUBERNETES_AUTH }, tx);
return { ...deletedKubernetesAuth?.[0], orgId: identityMembershipOrg.orgId };
});
return revokedIdentityKubernetesAuth;

View File

@@ -39,7 +39,7 @@ import {
type TIdentityOidcAuthServiceFactoryDep = {
identityOidcAuthDAL: TIdentityOidcAuthDALFactory;
identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "findOne">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create" | "delete">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "create">;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
orgBotDAL: Pick<TOrgBotDALFactory, "findOne" | "transaction" | "create">;
@@ -539,8 +539,6 @@ export const identityOidcAuthServiceFactory = ({
const revokedIdentityOidcAuth = await identityOidcAuthDAL.transaction(async (tx) => {
const deletedOidcAuth = await identityOidcAuthDAL.delete({ identityId }, tx);
await identityAccessTokenDAL.delete({ identityId, authMethod: IdentityAuthMethod.OIDC_AUTH }, tx);
return { ...deletedOidcAuth?.[0], orgId: identityMembershipOrg.orgId };
});

View File

@@ -86,7 +86,6 @@ services:
- .env
ports:
- 4000:4000
- 9464:9464 # for OTEL collection of Prometheus metrics
environment:
- NODE_ENV=development
- DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
@@ -96,42 +95,6 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"
prometheus:
image: prom/prometheus
volumes:
- ./prometheus.dev.yml:/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yml"
profiles: [metrics]
otel-collector:
image: otel/opentelemetry-collector-contrib
volumes:
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
ports:
- 1888:1888 # pprof extension
- 8888:8888 # Prometheus metrics exposed by the Collector
- 8889:8889 # Prometheus exporter metrics
- 13133:13133 # health_check extension
- 4317:4317 # OTLP gRPC receiver
- 4318:4318 # OTLP http receiver
- 55679:55679 # zpages extension
profiles: [metrics-otel]
grafana:
image: grafana/grafana
container_name: grafana
restart: unless-stopped
environment:
- GF_LOG_LEVEL=debug
ports:
- "3005:3000"
volumes:
- "grafana_storage:/var/lib/grafana"
profiles: [metrics]
frontend:
container_name: infisical-dev-frontend
restart: unless-stopped
@@ -203,4 +166,3 @@ volumes:
driver: local
ldap_data:
ldap_config:
grafana_storage:

View File

@@ -3,3 +3,6 @@ title: "Bulk Create"
openapi: "POST /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -3,3 +3,6 @@ title: "Create"
openapi: "POST /api/v3/secrets/raw/{secretName}"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -3,3 +3,6 @@ title: "Bulk Delete"
openapi: "DELETE /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -3,3 +3,6 @@ title: "Delete"
openapi: "DELETE /api/v3/secrets/raw/{secretName}"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -2,3 +2,7 @@
title: "List"
openapi: "GET /api/v3/secrets/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -3,3 +3,6 @@ title: "Retrieve"
openapi: "GET /api/v3/secrets/raw/{secretName}"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -3,3 +3,6 @@ title: "Bulk Update"
openapi: "PATCH /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -2,3 +2,7 @@
title: "Update"
openapi: "PATCH /api/v3/secrets/raw/{secretName}"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -30,8 +30,7 @@ Used to configure platform-specific security and operational settings
</ParamField>
<ParamField query="TELEMETRY_ENABLED" type="string" default="true" optional>
Telemetry helps us improve Infisical but if you want to disable it you may set
this to `false`.
Telemetry helps us improve Infisical but if you want to dsiable it you may set this to `false`.
</ParamField>
## Data Layer
@@ -66,9 +65,8 @@ DB_READ_REPLICAS=[{"DB_CONNECTION_URI":""}]
Use the command below to encode your certificate:
`echo "<certificate>" | base64`
If not provided it will use master SSL certificate.
If not provided it will use master SSL certificate.
</ParamField>
</Expandable>
</ParamField>
@@ -352,9 +350,8 @@ Optional (for TLS/SSL):
TLS: Available on the same ports (2525, 80, 25, 8025, or 587)
SSL: Available on ports 465, 8465, and 443
</Note>
</Accordion>
</Accordion>
## Authentication
@@ -521,36 +518,3 @@ To help you sync secrets from Infisical to services such as Github and Gitlab, I
OAuth2 client secret for Gitlab integration
</ParamField>
</Accordion>
## Observability
You can configure Infisical to collect and expose telemetry data for analytics and monitoring.
<ParamField
query="OTEL_TELEMETRY_COLLECTION_ENABLED"
type="string"
default="false"
>
Whether or not to collect and expose telemetry data.
</ParamField>
<ParamField query="OTEL_EXPORT_TYPE" type="enum" optional>
Supported types are `prometheus` and `otlp`.
If export type is set to `prometheus`, metric data will be exposed in port 9464 in the `/metrics` path.
If export type is set to `otlp`, you will have to configure a value for `OTEL_EXPORT_OTLP_ENDPOINT`.
</ParamField>
<ParamField query="OTEL_EXPORT_OTLP_ENDPOINT" type="string">
Where telemetry data would be pushed to for collection. This is only
applicable when `OTEL_EXPORT_TYPE` is set to `otlp`.
</ParamField>
<ParamField query="OTEL_COLLECTOR_BASIC_AUTH_USERNAME" type="string">
The username for authenticating with the telemetry collector.
</ParamField>
<ParamField query="OTEL_COLLECTOR_BASIC_AUTH_PASSWORD" type="string">
The password for authenticating with the telemetry collector.
</ParamField>

View File

@@ -1,45 +0,0 @@
extensions:
health_check:
pprof:
zpages:
basicauth/server:
htpasswd:
inline: |
infisical:infisical
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
auth:
authenticator: basicauth/server
prometheus:
config:
scrape_configs:
- job_name: otel-collector
scrape_interval: 30s
static_configs:
- targets: [backend:9464]
metric_relabel_configs:
- action: labeldrop
regex: "service_instance_id|service_name"
processors:
batch:
exporters:
prometheus:
endpoint: "0.0.0.0:8889"
auth:
authenticator: basicauth/server
resource_to_telemetry_conversion:
enabled: true
service:
extensions: [basicauth/server, health_check, pprof, zpages]
pipelines:
metrics:
receivers: [otlp]
processors: [batch]
exporters: [prometheus]

View File

@@ -1,5 +0,0 @@
scrape_configs:
- job_name: "metric-collector"
scrape_interval: 30s
static_configs:
- targets: ["backend:9464"]