Compare commits

...

17 Commits

Author SHA1 Message Date
8d33647739 Merge pull request #2249 from Infisical/maidul-sqhdqwdgvqwjf
patch findProjectUserWorkspaceKey
2024-08-06 22:12:03 +05:30
d1c142e5b1 patch findProjectUserWorkspaceKey 2024-08-06 12:39:06 -04:00
bb1cad0c5b Merge pull request #2223 from Infisical/misc/add-org-level-rate-limit
misc: moved to license-plan-based rate limits
2024-08-06 10:42:57 -04:00
2a1cfe15b4 update text when secrets deleted after integ delete 2024-08-06 10:07:41 -04:00
881d70bc64 Merge pull request #2238 from Infisical/feat/enabled-secrets-deletion-on-integ-removal
feat: added secrets deletion feature on integration removal
2024-08-06 09:54:15 -04:00
902a0b0ed4 Merge pull request #2243 from akhilmhdh/fix/missing-coment-field 2024-08-06 08:18:18 -04:00
ba92192537 misc: removed creation limits completely 2024-08-06 19:41:09 +08:00
26ed8df73c misc: finalized list of license rate limits 2024-08-06 19:14:49 +08:00
c1decab912 misc: addressed comments 2024-08-06 18:58:07 +08:00
=
216c073290 fix: missing comment key in updated project 2024-08-06 16:14:25 +05:30
1070954bdd misc: used destructuring 2024-08-06 02:05:13 +08:00
cc689d3178 feat: added secrets deletion feature on integration removal 2024-08-06 01:52:58 +08:00
0f23b7e1d3 misc: added check for undefined orgId 2024-08-03 02:10:47 +08:00
33193a47ae misc: updated default onprem rate limits 2024-08-03 01:52:04 +08:00
1ad286ca87 misc: name updates and more comments 2024-08-02 22:58:53 +08:00
be7c11a3f5 Merge remote-tracking branch 'origin/main' into misc/add-org-level-rate-limit 2024-08-02 22:42:23 +08:00
55a6740714 misc: moved to plan-based rate limit 2024-08-02 21:37:48 +08:00
30 changed files with 662 additions and 86 deletions

View File

@ -25,6 +25,7 @@
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
"@node-saml/passport-saml": "^4.0.4",
"@octokit/plugin-retry": "^5.0.5",
"@octokit/rest": "^20.0.2",
"@octokit/webhooks-types": "^7.3.1",
"@peculiar/asn1-schema": "^2.3.8",
@ -7812,19 +7813,45 @@
}
},
"node_modules/@octokit/plugin-retry": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.1.tgz",
"integrity": "sha512-SKs+Tz9oj0g4p28qkZwl/topGcb0k0qPNX/i7vBKmDsjoeqnVfFUquqrE/O9oJY7+oLzdCtkiWSXLpLjvl6uog==",
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-5.0.5.tgz",
"integrity": "sha512-sB1RWMhSrre02Atv95K6bhESlJ/sPdZkK/wE/w1IdSCe0yM6FxSjksLa6T7aAvxvxlLKzQEC4KIiqpqyov1Tbg==",
"dependencies": {
"@octokit/request-error": "^5.0.0",
"@octokit/types": "^12.0.0",
"@octokit/request-error": "^4.0.1",
"@octokit/types": "^10.0.0",
"bottleneck": "^2.15.3"
},
"engines": {
"node": ">= 18"
},
"peerDependencies": {
"@octokit/core": ">=5"
"@octokit/core": ">=3"
}
},
"node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": {
"version": "18.1.1",
"resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.1.1.tgz",
"integrity": "sha512-VRaeH8nCDtF5aXWnjPuEMIYf1itK/s3JYyJcWFJT8X9pSNnBtriDf7wlEWsGuhPLl4QIH4xM8fqTXDwJ3Mu6sw=="
},
"node_modules/@octokit/plugin-retry/node_modules/@octokit/request-error": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-4.0.2.tgz",
"integrity": "sha512-uqwUEmZw3x4I9DGYq9fODVAAvcLsPQv97NRycP6syEFu5916M189VnNBW2zANNwqg3OiligNcAey7P0SET843w==",
"dependencies": {
"@octokit/types": "^10.0.0",
"deprecation": "^2.0.0",
"once": "^1.4.0"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/@octokit/plugin-retry/node_modules/@octokit/types": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@octokit/types/-/types-10.0.0.tgz",
"integrity": "sha512-Vm8IddVmhCgU1fxC1eyinpwqzXPEYu0NrYzD3YZjlGjyftdLBTeqNblRC0jmJmgxbJIsQlyogVeGnrNaaMVzIg==",
"dependencies": {
"@octokit/openapi-types": "^18.0.0"
}
},
"node_modules/@octokit/plugin-throttling": {
@ -17396,6 +17423,22 @@
"node": ">=18"
}
},
"node_modules/probot/node_modules/@octokit/plugin-retry": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.1.tgz",
"integrity": "sha512-SKs+Tz9oj0g4p28qkZwl/topGcb0k0qPNX/i7vBKmDsjoeqnVfFUquqrE/O9oJY7+oLzdCtkiWSXLpLjvl6uog==",
"dependencies": {
"@octokit/request-error": "^5.0.0",
"@octokit/types": "^12.0.0",
"bottleneck": "^2.15.3"
},
"engines": {
"node": ">= 18"
},
"peerDependencies": {
"@octokit/core": ">=5"
}
},
"node_modules/probot/node_modules/commander": {
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz",

View File

@ -121,6 +121,7 @@
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
"@node-saml/passport-saml": "^4.0.4",
"@octokit/plugin-retry": "^5.0.5",
"@octokit/rest": "^20.0.2",
"@octokit/webhooks-types": "^7.3.1",
"@peculiar/asn1-schema": "^2.3.8",

View File

@ -18,6 +18,7 @@ import { TOidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-ser
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service";
import { TRateLimitServiceFactory } from "@app/ee/services/rate-limit/rate-limit-service";
import { RateLimitConfiguration } from "@app/ee/services/rate-limit/rate-limit-types";
import { TSamlConfigServiceFactory } from "@app/ee/services/saml-config/saml-config-service";
import { TScimServiceFactory } from "@app/ee/services/scim/scim-service";
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
@ -89,6 +90,7 @@ declare module "fastify" {
id: string;
orgId: string;
};
rateLimits: RateLimitConfiguration;
// passport data
passportUser: {
isUserCompleted: string;

View File

@ -0,0 +1,21 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasCreationLimitCol = await knex.schema.hasColumn(TableName.RateLimit, "creationLimit");
await knex.schema.alterTable(TableName.RateLimit, (t) => {
if (hasCreationLimitCol) {
t.dropColumn("creationLimit");
}
});
}
export async function down(knex: Knex): Promise<void> {
const hasCreationLimitCol = await knex.schema.hasColumn(TableName.RateLimit, "creationLimit");
await knex.schema.alterTable(TableName.RateLimit, (t) => {
if (!hasCreationLimitCol) {
t.integer("creationLimit").defaultTo(30).notNullable();
}
});
}

View File

@ -15,7 +15,6 @@ export const RateLimitSchema = z.object({
authRateLimit: z.number().default(60),
inviteUserRateLimit: z.number().default(30),
mfaRateLimit: z.number().default(20),
creationLimit: z.number().default(30),
publicEndpointLimit: z.number().default(30),
createdAt: z.date(),
updatedAt: z.date()

View File

@ -58,7 +58,6 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => {
authRateLimit: z.number(),
inviteUserRateLimit: z.number(),
mfaRateLimit: z.number(),
creationLimit: z.number(),
publicEndpointLimit: z.number()
}),
response: {

View File

@ -338,6 +338,7 @@ interface DeleteIntegrationEvent {
targetServiceId?: string;
path?: string;
region?: string;
shouldDeleteIntegrationSecrets?: boolean;
};
}

View File

@ -40,7 +40,12 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
secretRotation: true,
caCrl: false,
instanceUserManagement: false,
externalKms: false
externalKms: false,
rateLimits: {
readLimit: 60,
writeLimit: 200,
secretsLimit: 40
}
});
export const setupLicenceRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => {

View File

@ -58,6 +58,11 @@ export type TFeatureSet = {
caCrl: false;
instanceUserManagement: false;
externalKms: false;
rateLimits: {
readLimit: number;
writeLimit: number;
secretsLimit: number;
};
};
export type TOrgPlansTableDTO = {

View File

@ -4,17 +4,16 @@ import { logger } from "@app/lib/logger";
import { TLicenseServiceFactory } from "../license/license-service";
import { TRateLimitDALFactory } from "./rate-limit-dal";
import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types";
import { RateLimitConfiguration, TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types";
let rateLimitMaxConfiguration = {
let rateLimitMaxConfiguration: RateLimitConfiguration = {
readLimit: 60,
publicEndpointLimit: 30,
writeLimit: 200,
secretsLimit: 60,
authRateLimit: 60,
inviteUserRateLimit: 30,
mfaRateLimit: 20,
creationLimit: 30
mfaRateLimit: 20
};
Object.freeze(rateLimitMaxConfiguration);
@ -67,8 +66,7 @@ export const rateLimitServiceFactory = ({ rateLimitDAL, licenseService }: TRateL
secretsLimit: rateLimit.secretsRateLimit,
authRateLimit: rateLimit.authRateLimit,
inviteUserRateLimit: rateLimit.inviteUserRateLimit,
mfaRateLimit: rateLimit.mfaRateLimit,
creationLimit: rateLimit.creationLimit
mfaRateLimit: rateLimit.mfaRateLimit
};
logger.info(`syncRateLimitConfiguration: rate limit configuration: %o`, newRateLimitMaxConfiguration);

View File

@ -5,7 +5,6 @@ export type TRateLimitUpdateDTO = {
authRateLimit: number;
inviteUserRateLimit: number;
mfaRateLimit: number;
creationLimit: number;
publicEndpointLimit: number;
};
@ -14,3 +13,13 @@ export type TRateLimit = {
createdAt: Date;
updatedAt: Date;
} & TRateLimitUpdateDTO;
export type RateLimitConfiguration = {
readLimit: number;
publicEndpointLimit: number;
writeLimit: number;
secretsLimit: number;
authRateLimit: number;
inviteUserRateLimit: number;
mfaRateLimit: number;
};

View File

@ -1,7 +1,6 @@
import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-limit";
import { Redis } from "ioredis";
import { getRateLimiterConfig } from "@app/ee/services/rate-limit/rate-limit-service";
import { getConfig } from "@app/lib/config/env";
export const globalRateLimiterCfg = (): RateLimitPluginOptions => {
@ -22,14 +21,16 @@ export const globalRateLimiterCfg = (): RateLimitPluginOptions => {
// GET endpoints
export const readLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().readLimit,
hook: "preValidation",
max: (req) => req.rateLimits.readLimit,
keyGenerator: (req) => req.realIp
};
// POST, PATCH, PUT, DELETE endpoints
export const writeLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().writeLimit,
hook: "preValidation",
max: (req) => req.rateLimits.writeLimit,
keyGenerator: (req) => req.realIp
};
@ -37,42 +38,40 @@ export const writeLimit: RateLimitOptions = {
export const secretsLimit: RateLimitOptions = {
// secrets, folders, secret imports
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().secretsLimit,
hook: "preValidation",
max: (req) => req.rateLimits.secretsLimit,
keyGenerator: (req) => req.realIp
};
export const authRateLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().authRateLimit,
hook: "preValidation",
max: (req) => req.rateLimits.authRateLimit,
keyGenerator: (req) => req.realIp
};
export const inviteUserRateLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().inviteUserRateLimit,
hook: "preValidation",
max: (req) => req.rateLimits.inviteUserRateLimit,
keyGenerator: (req) => req.realIp
};
export const mfaRateLimit: RateLimitOptions = {
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().mfaRateLimit,
hook: "preValidation",
max: (req) => req.rateLimits.mfaRateLimit,
keyGenerator: (req) => {
return req.headers.authorization?.split(" ")[1] || req.realIp;
}
};
export const creationLimit: RateLimitOptions = {
// identity, project, org
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().creationLimit,
keyGenerator: (req) => req.realIp
};
// Public endpoints to avoid brute force attacks
export const publicEndpointLimit: RateLimitOptions = {
// Read Shared Secrets
timeWindow: 60 * 1000,
max: () => getRateLimiterConfig().publicEndpointLimit,
hook: "preValidation",
max: (req) => req.rateLimits.publicEndpointLimit,
keyGenerator: (req) => req.realIp
};

View File

@ -0,0 +1,38 @@
import fp from "fastify-plugin";
import { getRateLimiterConfig } from "@app/ee/services/rate-limit/rate-limit-service";
import { getConfig } from "@app/lib/config/env";
export const injectRateLimits = fp(async (server) => {
server.decorateRequest("rateLimits", null);
server.addHook("onRequest", async (req) => {
const appCfg = getConfig();
const instanceRateLimiterConfig = getRateLimiterConfig();
if (!req.auth?.orgId) {
// for public endpoints, we always use the instance-wide default rate limits
req.rateLimits = instanceRateLimiterConfig;
return;
}
const { rateLimits, customRateLimits } = await server.services.license.getPlan(req.auth.orgId);
if (customRateLimits && !appCfg.isCloud) {
// we do this because for self-hosted/dedicated instances, we want custom rate limits to be based on admin configuration
// note that the syncing of custom rate limit happens on the instanceRateLimiterConfig object
req.rateLimits = instanceRateLimiterConfig;
return;
}
// we're using the null coalescing operator in order to handle outdated licenses
req.rateLimits = {
readLimit: rateLimits?.readLimit ?? instanceRateLimiterConfig.readLimit,
writeLimit: rateLimits?.writeLimit ?? instanceRateLimiterConfig.writeLimit,
secretsLimit: rateLimits?.secretsLimit ?? instanceRateLimiterConfig.secretsLimit,
publicEndpointLimit: instanceRateLimiterConfig.publicEndpointLimit,
authRateLimit: instanceRateLimiterConfig.authRateLimit,
inviteUserRateLimit: instanceRateLimiterConfig.inviteUserRateLimit,
mfaRateLimit: instanceRateLimiterConfig.mfaRateLimit
};
});
});

View File

@ -184,6 +184,7 @@ import { webhookServiceFactory } from "@app/services/webhook/webhook-service";
import { injectAuditLogInfo } from "../plugins/audit-log";
import { injectIdentity } from "../plugins/auth/inject-identity";
import { injectPermission } from "../plugins/auth/inject-permission";
import { injectRateLimits } from "../plugins/inject-rate-limits";
import { registerSecretScannerGhApp } from "../plugins/secret-scanner";
import { registerV1Routes } from "./v1";
import { registerV2Routes } from "./v2";
@ -896,8 +897,15 @@ export const registerRoutes = async (
folderDAL,
integrationDAL,
integrationAuthDAL,
secretQueueService
secretQueueService,
integrationAuthService,
projectBotService,
secretV2BridgeDAL,
secretImportDAL,
secretDAL,
kmsService
});
const serviceTokenService = serviceTokenServiceFactory({
projectEnvDAL,
serviceTokenDAL,
@ -1142,6 +1150,7 @@ export const registerRoutes = async (
await server.register(injectIdentity, { userDAL, serviceTokenDAL });
await server.register(injectPermission);
await server.register(injectRateLimits);
await server.register(injectAuditLogInfo);
server.route({

View File

@ -3,7 +3,7 @@ import { z } from "zod";
import { IdentitiesSchema, IdentityOrgMembershipsSchema, OrgMembershipRole, OrgRolesSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { IDENTITIES } from "@app/lib/api-docs";
import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@ -16,7 +16,7 @@ export const registerIdentityRouter = async (server: FastifyZodProvider) => {
method: "POST",
url: "/",
config: {
rateLimit: creationLimit
rateLimit: writeLimit
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
schema: {

View File

@ -170,6 +170,12 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
params: z.object({
integrationId: z.string().trim().describe(INTEGRATION.DELETE.integrationId)
}),
querystring: z.object({
shouldDeleteIntegrationSecrets: z
.enum(["true", "false"])
.optional()
.transform((val) => val === "true")
}),
response: {
200: z.object({
integration: IntegrationsSchema
@ -183,7 +189,8 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
actorAuthMethod: req.permission.authMethod,
actor: req.permission.type,
actorOrgId: req.permission.orgId,
id: req.params.integrationId
id: req.params.integrationId,
shouldDeleteIntegrationSecrets: req.query.shouldDeleteIntegrationSecrets
});
await server.services.auditLog.createAuditLog({
@ -205,7 +212,8 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
targetService: integration.targetService,
targetServiceId: integration.targetServiceId,
path: integration.path,
region: integration.region
region: integration.region,
shouldDeleteIntegrationSecrets: req.query.shouldDeleteIntegrationSecrets
// eslint-disable-next-line
}) as any
}

View File

@ -9,7 +9,7 @@ import {
UsersSchema
} from "@app/db/schemas";
import { ORGANIZATIONS } from "@app/lib/api-docs";
import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
@ -307,7 +307,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
method: "POST",
url: "/",
config: {
rateLimit: creationLimit
rateLimit: writeLimit
},
schema: {
body: z.object({

View File

@ -4,7 +4,7 @@ import { z } from "zod";
import { CertificateAuthoritiesSchema, CertificatesSchema, ProjectKeysSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { PROJECTS } from "@app/lib/api-docs";
import { creationLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@ -142,7 +142,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
method: "POST",
url: "/",
config: {
rateLimit: creationLimit
rateLimit: writeLimit
},
schema: {
description: "Create a new project",

View File

@ -0,0 +1,357 @@
import { retry } from "@octokit/plugin-retry";
import { Octokit } from "@octokit/rest";
import { TIntegrationAuths, TIntegrations } from "@app/db/schemas";
import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { IntegrationMetadataSchema } from "../integration/integration-schema";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TIntegrationAuthServiceFactory } from "./integration-auth-service";
import { Integrations } from "./integration-list";
const MAX_SYNC_SECRET_DEPTH = 5;
/**
* Return the secrets in a given [folderId] including secrets from
* nested imported folders recursively.
*/
const getIntegrationSecretsV2 = async (
dto: {
projectId: string;
environment: string;
folderId: string;
depth: number;
decryptor: (value: Buffer | null | undefined) => string;
},
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">,
folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">,
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">
) => {
const content: Record<string, boolean> = {};
if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
logger.info(
`getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
);
return content;
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = secret.key;
content[secretKey] = true;
});
// check if current folder has any imports from other folders
const secretImports = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,
secretImportDAL,
allowedImports: secretImports
});
for (let i = importedSecrets.length - 1; i >= 0; i -= 1) {
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
const importedSecret = importedSecrets[i].secrets[j];
if (!content[importedSecret.key]) {
content[importedSecret.key] = true;
}
}
}
return content;
};
/**
* Return the secrets in a given [folderId] including secrets from
* nested imported folders recursively.
*/
const getIntegrationSecretsV1 = async (
dto: {
projectId: string;
environment: string;
folderId: string;
key: string;
depth: number;
},
secretDAL: Pick<TSecretDALFactory, "findByFolderId">,
folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">,
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">
) => {
let content: Record<string, boolean> = {};
if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
logger.info(
`getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
);
return content;
}
// process secrets in current folder
const secrets = await secretDAL.findByFolderId(dto.folderId);
secrets.forEach((secret) => {
const secretKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secret.secretKeyCiphertext,
iv: secret.secretKeyIV,
tag: secret.secretKeyTag,
key: dto.key
});
content[secretKey] = true;
});
// check if current folder has any imports from other folders
const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
// if no imports then return secrets in the current folder
if (!secretImport) return content;
const importedFolders = await folderDAL.findByManySecretPath(
secretImport.map(({ importEnv, importPath }) => ({
envId: importEnv.id,
secretPath: importPath
}))
);
for await (const folder of importedFolders) {
if (folder) {
// get secrets contained in each imported folder by recursively calling
// this function against the imported folder
const importedSecrets = await getIntegrationSecretsV1(
{
environment: dto.environment,
projectId: dto.projectId,
folderId: folder.id,
key: dto.key,
depth: dto.depth + 1
},
secretDAL,
folderDAL,
secretImportDAL
);
// add the imported secrets to the current folder secrets
content = { ...importedSecrets, ...content };
}
}
return content;
};
export const deleteGithubSecrets = async ({
integration,
secrets,
accessToken
}: {
integration: Omit<TIntegrations, "envId">;
secrets: Record<string, boolean>;
accessToken: string;
}) => {
interface GitHubSecret {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
}
const OctokitWithRetry = Octokit.plugin(retry);
const octokit = new OctokitWithRetry({
auth: accessToken
});
enum GithubScope {
Repo = "github-repo",
Org = "github-org",
Env = "github-env"
}
let encryptedGithubSecrets: GitHubSecret[];
switch (integration.scope) {
case GithubScope.Org: {
encryptedGithubSecrets = (
await octokit.request("GET /orgs/{org}/actions/secrets", {
org: integration.owner as string
})
).data.secrets;
break;
}
case GithubScope.Env: {
encryptedGithubSecrets = (
await octokit.request("GET /repositories/{repository_id}/environments/{environment_name}/secrets", {
repository_id: Number(integration.appId),
environment_name: integration.targetEnvironmentId as string
})
).data.secrets;
break;
}
default: {
encryptedGithubSecrets = (
await octokit.request("GET /repos/{owner}/{repo}/actions/secrets", {
owner: integration.owner as string,
repo: integration.app as string
})
).data.secrets;
break;
}
}
for await (const encryptedSecret of encryptedGithubSecrets) {
if (encryptedSecret.name in secrets) {
switch (integration.scope) {
case GithubScope.Org: {
await octokit.request("DELETE /orgs/{org}/actions/secrets/{secret_name}", {
org: integration.owner as string,
secret_name: encryptedSecret.name
});
break;
}
case GithubScope.Env: {
await octokit.request(
"DELETE /repositories/{repository_id}/environments/{environment_name}/secrets/{secret_name}",
{
repository_id: Number(integration.appId),
environment_name: integration.targetEnvironmentId as string,
secret_name: encryptedSecret.name
}
);
break;
}
default: {
await octokit.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: integration.owner as string,
repo: integration.app as string,
secret_name: encryptedSecret.name
});
break;
}
}
// small delay to prevent hitting API rate limits
await new Promise((resolve) => {
setTimeout(resolve, 50);
});
}
}
};
export const deleteIntegrationSecrets = async ({
integration,
integrationAuth,
integrationAuthService,
projectBotService,
secretV2BridgeDAL,
folderDAL,
secretDAL,
secretImportDAL,
kmsService
}: {
integration: Omit<TIntegrations, "envId"> & {
projectId: string;
environment: {
id: string;
name: string;
slug: string;
};
secretPath: string;
};
integrationAuth: TIntegrationAuths;
integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken" | "getIntegrationAuth">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">;
folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath" | "findBySecretPath">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretDAL: Pick<TSecretDALFactory, "findByFolderId">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
}) => {
const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integration.projectId);
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId: integration.projectId
});
const folder = await folderDAL.findBySecretPath(
integration.projectId,
integration.environment.slug,
integration.secretPath
);
if (!folder) {
throw new NotFoundError({
message: "Folder not found."
});
}
const { accessToken } = await integrationAuthService.getIntegrationAccessToken(
integrationAuth,
shouldUseSecretV2Bridge,
botKey
);
const secrets = shouldUseSecretV2Bridge
? await getIntegrationSecretsV2(
{
environment: integration.environment.id,
projectId: integration.projectId,
folderId: folder.id,
depth: 1,
decryptor: (value) => (value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "")
},
secretV2BridgeDAL,
folderDAL,
secretImportDAL
)
: await getIntegrationSecretsV1(
{
environment: integration.environment.id,
projectId: integration.projectId,
folderId: folder.id,
key: botKey as string,
depth: 1
},
secretDAL,
folderDAL,
secretImportDAL
);
const suffixedSecrets: typeof secrets = {};
const metadata = IntegrationMetadataSchema.parse(integration.metadata);
if (metadata) {
Object.keys(secrets).forEach((key) => {
const prefix = metadata?.secretPrefix || "";
const suffix = metadata?.secretSuffix || "";
const newKey = prefix + key + suffix;
suffixedSecrets[newKey] = secrets[key];
});
}
switch (integration.integration) {
case Integrations.GITHUB: {
await deleteGithubSecrets({
integration,
accessToken,
secrets: Object.keys(suffixedSecrets).length !== 0 ? suffixedSecrets : secrets
});
break;
}
default:
throw new BadRequestError({
message: "Invalid integration"
});
}
};

View File

@ -6,8 +6,15 @@ import { BadRequestError } from "@app/lib/errors";
import { TProjectPermission } from "@app/lib/types";
import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
import { deleteIntegrationSecrets } from "../integration-auth/integration-delete-secret";
import { TKmsServiceFactory } from "../kms/kms-service";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretQueueFactory } from "../secret/secret-queue";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TIntegrationDALFactory } from "./integration-dal";
import {
TCreateIntegrationDTO,
@ -19,9 +26,15 @@ import {
type TIntegrationServiceFactoryDep = {
integrationDAL: TIntegrationDALFactory;
integrationAuthDAL: TIntegrationAuthDALFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
integrationAuthService: TIntegrationAuthServiceFactory;
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findByManySecretPath">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
projectBotService: TProjectBotServiceFactory;
secretQueueService: Pick<TSecretQueueFactory, "syncIntegrations">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretDAL: Pick<TSecretDALFactory, "findByFolderId">;
};
export type TIntegrationServiceFactory = ReturnType<typeof integrationServiceFactory>;
@ -31,7 +44,13 @@ export const integrationServiceFactory = ({
integrationAuthDAL,
folderDAL,
permissionService,
secretQueueService
secretQueueService,
integrationAuthService,
projectBotService,
secretV2BridgeDAL,
secretImportDAL,
kmsService,
secretDAL
}: TIntegrationServiceFactoryDep) => {
const createIntegration = async ({
app,
@ -161,7 +180,14 @@ export const integrationServiceFactory = ({
return updatedIntegration;
};
const deleteIntegration = async ({ actorId, id, actor, actorAuthMethod, actorOrgId }: TDeleteIntegrationDTO) => {
const deleteIntegration = async ({
actorId,
id,
actor,
actorAuthMethod,
actorOrgId,
shouldDeleteIntegrationSecrets
}: TDeleteIntegrationDTO) => {
const integration = await integrationDAL.findById(id);
if (!integration) throw new BadRequestError({ message: "Integration auth not found" });
@ -174,6 +200,22 @@ export const integrationServiceFactory = ({
);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Integrations);
const integrationAuth = await integrationAuthDAL.findById(integration.integrationAuthId);
if (shouldDeleteIntegrationSecrets) {
await deleteIntegrationSecrets({
integration,
integrationAuth,
projectBotService,
integrationAuthService,
secretV2BridgeDAL,
folderDAL,
secretImportDAL,
secretDAL,
kmsService
});
}
const deletedIntegration = await integrationDAL.transaction(async (tx) => {
// delete integration
const deletedIntegrationResult = await integrationDAL.deleteById(id, tx);

View File

@ -63,6 +63,7 @@ export type TUpdateIntegrationDTO = {
export type TDeleteIntegrationDTO = {
id: string;
shouldDeleteIntegrationSecrets?: boolean;
} & Omit<TProjectPermission, "projectId">;
export type TSyncIntegrationDTO = {

View File

@ -46,6 +46,7 @@ export const projectBotDALFactory = (db: TDbClient) => {
const doc = await db
.replicaNode()(TableName.ProjectMembership)
.where(`${TableName.ProjectMembership}.projectId` as "projectId", projectId)
.where(`${TableName.ProjectKeys}.projectId` as "projectId", projectId)
.where(`${TableName.Users}.isGhost` as "isGhost", false)
.join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`)
.join(TableName.ProjectKeys, `${TableName.ProjectMembership}.userId`, `${TableName.ProjectKeys}.receiverId`)

View File

@ -542,8 +542,8 @@ export const reshapeBridgeSecret = (
secretPath,
workspace: workspaceId,
environment,
secretValue: secret.value,
secretComment: secret.comment,
secretValue: secret.value || "",
secretComment: secret.comment || "",
version: secret.version,
type: secret.type,
_id: secret.id,

View File

@ -522,7 +522,7 @@ export const secretV2BridgeServiceFactory = ({
await expandSecretReferences(secretsGroupByKey);
secretsGroupByPath[secretPathKey].forEach((decryptedSecret) => {
// eslint-disable-next-line no-param-reassign
decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value;
decryptedSecret.secretValue = secretsGroupByKey[decryptedSecret.secretKey].value || "";
});
}
}

View File

@ -1,4 +1,4 @@
import { useEffect, useState } from "react";
import { ReactNode, useEffect, useState } from "react";
import { useToggle } from "@app/hooks";
@ -16,6 +16,7 @@ type Props = {
subTitle?: string;
onDeleteApproved: () => Promise<void>;
buttonText?: string;
children?: ReactNode;
};
export const DeleteActionModal = ({
@ -26,7 +27,8 @@ export const DeleteActionModal = ({
onDeleteApproved,
title,
subTitle = "This action is irreversible.",
buttonText = "Delete"
buttonText = "Delete",
children
}: Props): JSX.Element => {
const [inputData, setInputData] = useState("");
const [isLoading, setIsLoading] = useToggle();
@ -97,6 +99,7 @@ export const DeleteActionModal = ({
placeholder={`Type ${deleteKey} here`}
/>
</FormControl>
{children}
</form>
</ModalContent>
</Modal>

View File

@ -110,8 +110,15 @@ export const useCreateIntegration = () => {
export const useDeleteIntegration = () => {
const queryClient = useQueryClient();
return useMutation<{}, {}, { id: string; workspaceId: string }>({
mutationFn: ({ id }) => apiRequest.delete(`/api/v1/integration/${id}`),
return useMutation<
{},
{},
{ id: string; workspaceId: string; shouldDeleteIntegrationSecrets: boolean }
>({
mutationFn: ({ id, shouldDeleteIntegrationSecrets }) =>
apiRequest.delete(
`/api/v1/integration/${id}?shouldDeleteIntegrationSecrets=${shouldDeleteIntegrationSecrets}`
),
onSuccess: (_, { workspaceId }) => {
queryClient.invalidateQueries(workspaceKeys.getWorkspaceIntegrations(workspaceId));
queryClient.invalidateQueries(workspaceKeys.getWorkspaceAuthorization(workspaceId));

View File

@ -5,6 +5,5 @@ export type TRateLimit = {
authRateLimit: number;
inviteUserRateLimit: number;
mfaRateLimit: number;
creationLimit: number;
publicEndpointLimit: number;
};

View File

@ -106,9 +106,13 @@ export const IntegrationsPage = withProjectPermission(
handleProviderIntegration(provider);
};
const handleIntegrationDelete = async (integrationId: string, cb: () => void) => {
const handleIntegrationDelete = async (
integrationId: string,
shouldDeleteIntegrationSecrets: boolean,
cb: () => void
) => {
try {
await deleteIntegration({ id: integrationId, workspaceId });
await deleteIntegration({ id: integrationId, workspaceId, shouldDeleteIntegrationSecrets });
if (cb) cb();
createNotification({
type: "success",
@ -152,7 +156,7 @@ export const IntegrationsPage = withProjectPermission(
isLoading={isIntegrationLoading}
integrations={integrations}
environments={environments}
onIntegrationDelete={({ id }, cb) => handleIntegrationDelete(id, cb)}
onIntegrationDelete={handleIntegrationDelete}
workspaceId={workspaceId}
/>
<CloudIntegrationSection

View File

@ -7,6 +7,7 @@ import { integrationSlugNameMapping } from "public/data/frequentConstants";
import { ProjectPermissionCan } from "@app/components/permissions";
import {
Button,
Checkbox,
DeleteActionModal,
EmptyState,
FormLabel,
@ -16,7 +17,7 @@ import {
Tooltip
} from "@app/components/v2";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context";
import { usePopUp } from "@app/hooks";
import { usePopUp, useToggle } from "@app/hooks";
import { useSyncIntegration } from "@app/hooks/api/integrations/queries";
import { IntegrationMappingBehavior } from "@app/hooks/api/integrations/types";
import { TIntegration } from "@app/hooks/api/types";
@ -25,7 +26,11 @@ type Props = {
environments: Array<{ name: string; slug: string; id: string }>;
integrations?: TIntegration[];
isLoading?: boolean;
onIntegrationDelete: (integration: TIntegration, cb: () => void) => void;
onIntegrationDelete: (
integrationId: string,
shouldDeleteIntegrationSecrets: boolean,
cb: () => void
) => Promise<void>;
workspaceId: string;
};
@ -37,10 +42,12 @@ export const IntegrationsSection = ({
workspaceId
}: Props) => {
const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([
"deleteConfirmation"
"deleteConfirmation",
"deleteSecretsConfirmation"
] as const);
const { mutate: syncIntegration } = useSyncIntegration();
const [shouldDeleteSecrets, setShouldDeleteSecrets] = useToggle(false);
return (
<div className="mb-8">
@ -249,7 +256,10 @@ export const IntegrationsSection = ({
<div className="flex items-end opacity-80 duration-200 hover:opacity-100">
<Tooltip content="Remove Integration">
<IconButton
onClick={() => handlePopUpOpen("deleteConfirmation", integration)}
onClick={() => {
setShouldDeleteSecrets.off();
handlePopUpOpen("deleteConfirmation", integration);
}}
ariaLabel="delete"
isDisabled={!isAllowed}
colorSchema="danger"
@ -281,11 +291,49 @@ export const IntegrationsSection = ({
(popUp?.deleteConfirmation?.data as TIntegration)?.integration ||
""
}
onDeleteApproved={async () =>
onIntegrationDelete(popUp?.deleteConfirmation.data as TIntegration, () =>
handlePopUpClose("deleteConfirmation")
)
}
onDeleteApproved={async () => {
if (shouldDeleteSecrets) {
handlePopUpOpen("deleteSecretsConfirmation");
return;
}
await onIntegrationDelete(
(popUp?.deleteConfirmation.data as TIntegration).id,
false,
() => handlePopUpClose("deleteConfirmation")
);
}}
>
{(popUp?.deleteConfirmation?.data as TIntegration)?.integration === "github" && (
<div className="mt-4">
<Checkbox
id="delete-integration-secrets"
checkIndicatorBg="text-white"
onCheckedChange={() => setShouldDeleteSecrets.toggle()}
>
Delete previously synced secrets from the destination
</Checkbox>
</div>
)}
</DeleteActionModal>
<DeleteActionModal
isOpen={popUp.deleteSecretsConfirmation.isOpen}
title={`Are you sure you also want to delete secrets on ${
(popUp?.deleteConfirmation.data as TIntegration)?.integration
}?`}
subTitle="By confirming, you acknowledge that all secrets managed by this integration will be removed from the destination. This action is irreversible."
onChange={(isOpen) => handlePopUpToggle("deleteSecretsConfirmation", isOpen)}
deleteKey="confirm"
onDeleteApproved={async () => {
await onIntegrationDelete(
(popUp?.deleteConfirmation.data as TIntegration).id,
true,
() => {
handlePopUpClose("deleteSecretsConfirmation");
handlePopUpClose("deleteConfirmation");
}
);
}}
/>
</div>
);

View File

@ -15,7 +15,6 @@ const formSchema = z.object({
authRateLimit: z.number(),
inviteUserRateLimit: z.number(),
mfaRateLimit: z.number(),
creationLimit: z.number(),
publicEndpointLimit: z.number()
});
@ -41,7 +40,6 @@ export const RateLimitPanel = () => {
authRateLimit: rateLimit?.authRateLimit ?? 60,
inviteUserRateLimit: rateLimit?.inviteUserRateLimit ?? 30,
mfaRateLimit: rateLimit?.mfaRateLimit ?? 20,
creationLimit: rateLimit?.creationLimit ?? 30,
publicEndpointLimit: rateLimit?.publicEndpointLimit ?? 30
}
});
@ -60,7 +58,6 @@ export const RateLimitPanel = () => {
authRateLimit,
inviteUserRateLimit,
mfaRateLimit,
creationLimit,
publicEndpointLimit
} = formData;
@ -71,7 +68,6 @@ export const RateLimitPanel = () => {
authRateLimit,
inviteUserRateLimit,
mfaRateLimit,
creationLimit,
publicEndpointLimit
});
createNotification({
@ -210,25 +206,6 @@ export const RateLimitPanel = () => {
</FormControl>
)}
/>
<Controller
control={control}
defaultValue={300}
name="creationLimit"
render={({ field, fieldState: { error } }) => (
<FormControl
label="New resource creation requests per minute"
className="w-72"
isError={Boolean(error)}
errorText={error?.message}
>
<Input
{...field}
value={field.value || ""}
onChange={(e) => field.onChange(Number(e.target.value))}
/>
</FormControl>
)}
/>
<Controller
control={control}
defaultValue={300}