misc: renamed connection check method

This reverts commit 8fc4fd64f8.

.github/workflows/check-api-for-breaking-changes.yml

@@ -40,13 +40,14 @@ jobs:
           REDIS_URL: redis://172.17.0.1:6379
           DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable
           JWT_AUTH_SECRET: something-random
+          ENCRYPTION_KEY: 4bnfe4e407b8921c104518903515b218
       - uses: actions/setup-go@v5
         with:
           go-version: '1.21.5'
       - name: Wait for container to be stable and check logs
         run: |
           SECONDS=0
-          HEALTHY=0
+ r        HEALTHY=0
           while [ $SECONDS -lt 60 ]; do
             if docker ps | grep infisical-api | grep -q healthy; then
               echo "Container is healthy."
@@ -73,4 +74,4 @@ jobs:
         run: |
           docker-compose -f "docker-compose.dev.yml" down
           docker stop infisical-api
-          docker remove infisical-api
+          docker remove infisical-api

.github/workflows/release_build_infisical_cli.yml

@@ -22,6 +22,9 @@ jobs:
             CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }}
             CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }}
             CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }}
+            CLI_TESTS_USER_EMAIL: ${{ secrets.CLI_TESTS_USER_EMAIL }}
+            CLI_TESTS_USER_PASSWORD: ${{ secrets.CLI_TESTS_USER_PASSWORD }}
+            INFISICAL_VAULT_FILE_PASSPHRASE: ${{ secrets.CLI_TESTS_INFISICAL_VAULT_FILE_PASSPHRASE }}
 
     goreleaser:
         runs-on: ubuntu-20.04

.github/workflows/run-cli-tests.yml

@@ -20,7 +20,12 @@ on:
                 required: true
             CLI_TESTS_ENV_SLUG:
                 required: true
-
+            CLI_TESTS_USER_EMAIL:
+                required: true
+            CLI_TESTS_USER_PASSWORD:
+                required: true
+            CLI_TESTS_INFISICAL_VAULT_FILE_PASSPHRASE:
+                required: true
 jobs:
     test:
         defaults:
@@ -43,5 +48,8 @@ jobs:
                   CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }}
                   CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }}
                   CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }}
+                  CLI_TESTS_USER_EMAIL: ${{ secrets.CLI_TESTS_USER_EMAIL }}
+                  CLI_TESTS_USER_PASSWORD: ${{ secrets.CLI_TESTS_USER_PASSWORD }}
+                  INFISICAL_VAULT_FILE_PASSPHRASE: ${{ secrets.CLI_TESTS_INFISICAL_VAULT_FILE_PASSPHRASE }}
 
               run: go test -v -count=1 ./test

Dockerfile.standalone-infisical

@@ -1,7 +1,6 @@
 ARG POSTHOG_HOST=https://app.posthog.com
 ARG POSTHOG_API_KEY=posthog-api-key
 ARG INTERCOM_ID=intercom-id
-ARG SAML_ORG_SLUG=saml-org-slug-default
 
 FROM  node:20-alpine AS base
 
@@ -35,9 +34,7 @@ ENV NEXT_PUBLIC_POSTHOG_API_KEY $POSTHOG_API_KEY
 ARG INTERCOM_ID
 ENV NEXT_PUBLIC_INTERCOM_ID $INTERCOM_ID
 ARG INFISICAL_PLATFORM_VERSION
-ENV NEXT_PUBLIC_INFISICAL_PLATFORM_VERSION $INFISICAL_PLATFORM_VERSION
-ARG SAML_ORG_SLUG
-ENV NEXT_PUBLIC_SAML_ORG_SLUG=$SAML_ORG_SLUG 
+ENV NEXT_PUBLIC_INFISICAL_PLATFORM_VERSION $INFISICAL_PLATFORM_VERSION 
 
 # Build
 RUN npm run build
@@ -55,6 +52,7 @@ VOLUME /app/.next/cache/images
 COPY --chown=non-root-user:nodejs --chmod=555 frontend/scripts ./scripts
 COPY --from=frontend-builder /app/public ./public
 RUN chown non-root-user:nodejs ./public/data
+
 COPY --from=frontend-builder --chown=non-root-user:nodejs /app/.next/standalone ./
 COPY --from=frontend-builder --chown=non-root-user:nodejs /app/.next/static ./.next/static
 
@@ -93,9 +91,18 @@ RUN mkdir frontend-build
 
 # Production stage
 FROM base AS production
+RUN apk add --upgrade --no-cache ca-certificates
 RUN addgroup --system --gid 1001 nodejs \
   && adduser --system --uid 1001 non-root-user
 
+# Give non-root-user permission to update SSL certs
+RUN chown -R non-root-user /etc/ssl/certs
+RUN chown non-root-user /etc/ssl/certs/ca-certificates.crt
+RUN chmod -R u+rwx /etc/ssl/certs
+RUN chmod u+rw /etc/ssl/certs/ca-certificates.crt
+RUN chown non-root-user /usr/sbin/update-ca-certificates
+RUN chmod u+rx /usr/sbin/update-ca-certificates
+
 ## set pre baked keys
 ARG POSTHOG_API_KEY
 ENV NEXT_PUBLIC_POSTHOG_API_KEY=$POSTHOG_API_KEY \
@@ -103,9 +110,6 @@ ENV NEXT_PUBLIC_POSTHOG_API_KEY=$POSTHOG_API_KEY \
 ARG INTERCOM_ID=intercom-id
 ENV NEXT_PUBLIC_INTERCOM_ID=$INTERCOM_ID \
   BAKED_NEXT_PUBLIC_INTERCOM_ID=$INTERCOM_ID
-ARG SAML_ORG_SLUG
-ENV NEXT_PUBLIC_SAML_ORG_SLUG=$SAML_ORG_SLUG \
-  BAKED_NEXT_PUBLIC_SAML_ORG_SLUG=$SAML_ORG_SLUG
 
 WORKDIR / 
 

backend/e2e-test/mocks/keystore.ts

@@ -1,4 +1,5 @@
 import { TKeyStoreFactory } from "@app/keystore/keystore";
+import { Lock } from "@app/lib/red-lock";
 
 export const mockKeyStore = (): TKeyStoreFactory => {
   const store: Record<string, string | number | Buffer> = {};
@@ -25,6 +26,12 @@ export const mockKeyStore = (): TKeyStoreFactory => {
     },
     incrementBy: async () => {
       return 1;
-    }
+    },
+    acquireLock: () => {
+      return Promise.resolve({
+        release: () => {}
+      }) as Promise<Lock>;
+    },
+    waitTillReady: async () => {}
   };
 };

backend/package-lock.json

@@ -51,7 +51,7 @@
         "libsodium-wrappers": "^0.7.13",
         "lodash.isequal": "^4.5.0",
         "ms": "^2.1.3",
-        "mysql2": "^3.9.7",
+        "mysql2": "^3.9.8",
         "nanoid": "^5.0.4",
         "nodemailer": "^6.9.9",
         "ora": "^7.0.1",
@@ -10290,9 +10290,10 @@
       }
     },
     "node_modules/mysql2": {
-      "version": "3.9.7",
-      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.7.tgz",
-      "integrity": "sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==",
+      "version": "3.9.8",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.8.tgz",
+      "integrity": "sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==",
+      "license": "MIT",
       "dependencies": {
         "denque": "^2.1.0",
         "generate-function": "^2.3.1",

backend/package.json

@@ -112,7 +112,7 @@
     "libsodium-wrappers": "^0.7.13",
     "lodash.isequal": "^4.5.0",
     "ms": "^2.1.3",
-    "mysql2": "^3.9.7",
+    "mysql2": "^3.9.8",
     "nanoid": "^5.0.4",
     "nodemailer": "^6.9.9",
     "ora": "^7.0.1",

backend/scripts/generate-schema-types.ts

@@ -35,6 +35,8 @@ const getZodPrimitiveType = (type: string) => {
       return "z.coerce.number()";
     case "text":
       return "z.string()";
+    case "bytea":
+      return "zodBuffer";
     default:
       throw new Error(`Invalid type: ${type}`);
   }
@@ -96,10 +98,15 @@ const main = async () => {
     const columnNames = Object.keys(columns);
 
     let schema = "";
+    const zodImportSet = new Set<string>();
     for (let colNum = 0; colNum < columnNames.length; colNum++) {
       const columnName = columnNames[colNum];
       const colInfo = columns[columnName];
       let ztype = getZodPrimitiveType(colInfo.type);
+      if (["zodBuffer"].includes(ztype)) {
+        zodImportSet.add(ztype);
+      }
+
       // don't put optional on id
       if (colInfo.defaultValue && columnName !== "id") {
         const { defaultValue } = colInfo;
@@ -121,6 +128,8 @@ const main = async () => {
       .split("_")
       .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, "");
 
+    const zodImports = Array.from(zodImportSet);
+
     // the insert and update are changed to zod input type to use default cases
     writeFileSync(
       path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`),
@@ -131,6 +140,8 @@ const main = async () => {
 
 import { z } from "zod";
 
+${zodImports.length ? `import { ${zodImports.join(",")} } from \"@app/lib/zod\";` : ""}
+
 import { TImmutableDBKeys } from "./models";
 
 export const ${pascalCase}Schema = z.object({${schema}});

backend/src/@types/fastify.d.ts

@@ -52,6 +52,8 @@ import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service";
 import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service";
 import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
+import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service";
+import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service";
 import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
 import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service";
 import { TSuperAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
@@ -107,6 +109,7 @@ declare module "fastify" {
       projectKey: TProjectKeyServiceFactory;
       projectRole: TProjectRoleServiceFactory;
       secret: TSecretServiceFactory;
+      secretReplication: TSecretReplicationServiceFactory;
       secretTag: TSecretTagServiceFactory;
       secretImport: TSecretImportServiceFactory;
       projectBot: TProjectBotServiceFactory;
@@ -143,6 +146,7 @@ declare module "fastify" {
       dynamicSecretLease: TDynamicSecretLeaseServiceFactory;
       projectUserAdditionalPrivilege: TProjectUserAdditionalPrivilegeServiceFactory;
       identityProjectAdditionalPrivilege: TIdentityProjectAdditionalPrivilegeServiceFactory;
+      secretSharing: TSecretSharingServiceFactory;
     };
     // this is exclusive use for middlewares in which we need to inject data
     // everywhere else access using service layer

backend/src/@types/knex.d.ts

@@ -98,6 +98,15 @@ import {
   TIntegrations,
   TIntegrationsInsert,
   TIntegrationsUpdate,
+  TKmsKeys,
+  TKmsKeysInsert,
+  TKmsKeysUpdate,
+  TKmsKeyVersions,
+  TKmsKeyVersionsInsert,
+  TKmsKeyVersionsUpdate,
+  TKmsRootConfig,
+  TKmsRootConfigInsert,
+  TKmsRootConfigUpdate,
   TLdapConfigs,
   TLdapConfigsInsert,
   TLdapConfigsUpdate,
@@ -176,6 +185,9 @@ import {
   TSecretImports,
   TSecretImportsInsert,
   TSecretImportsUpdate,
+  TSecretReferences,
+  TSecretReferencesInsert,
+  TSecretReferencesUpdate,
   TSecretRotationOutputs,
   TSecretRotationOutputsInsert,
   TSecretRotationOutputsUpdate,
@@ -186,6 +198,9 @@ import {
   TSecretScanningGitRisks,
   TSecretScanningGitRisksInsert,
   TSecretScanningGitRisksUpdate,
+  TSecretSharing,
+  TSecretSharingInsert,
+  TSecretSharingUpdate,
   TSecretsInsert,
   TSecretSnapshotFolders,
   TSecretSnapshotFoldersInsert,
@@ -237,7 +252,6 @@ import {
   TWebhooksInsert,
   TWebhooksUpdate
 } from "@app/db/schemas";
-import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references";
 
 declare module "knex/types/tables" {
   interface Tables {
@@ -328,6 +342,7 @@ declare module "knex/types/tables" {
       TSecretFolderVersionsInsert,
       TSecretFolderVersionsUpdate
     >;
+    [TableName.SecretSharing]: Knex.CompositeTableType<TSecretSharing, TSecretSharingInsert, TSecretSharingUpdate>;
     [TableName.SecretTag]: Knex.CompositeTableType<TSecretTags, TSecretTagsInsert, TSecretTagsUpdate>;
     [TableName.SecretImport]: Knex.CompositeTableType<TSecretImports, TSecretImportsInsert, TSecretImportsUpdate>;
     [TableName.Integration]: Knex.CompositeTableType<TIntegrations, TIntegrationsInsert, TIntegrationsUpdate>;
@@ -510,5 +525,13 @@ declare module "knex/types/tables" {
       TSecretVersionTagJunctionInsert,
       TSecretVersionTagJunctionUpdate
     >;
+    // KMS service
+    [TableName.KmsServerRootConfig]: Knex.CompositeTableType<
+      TKmsRootConfig,
+      TKmsRootConfigInsert,
+      TKmsRootConfigUpdate
+    >;
+    [TableName.KmsKey]: Knex.CompositeTableType<TKmsKeys, TKmsKeysInsert, TKmsKeysUpdate>;
+    [TableName.KmsKeyVersion]: Knex.CompositeTableType<TKmsKeyVersions, TKmsKeyVersionsInsert, TKmsKeyVersionsUpdate>;
   }
 }

backend/src/db/migrations/20240528190137_secret_sharing.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.SecretSharing))) {
+    await knex.schema.createTable(TableName.SecretSharing, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("name").notNullable();
+      t.text("encryptedValue").notNullable();
+      t.text("iv").notNullable();
+      t.text("tag").notNullable();
+      t.text("hashedHex").notNullable();
+      t.timestamp("expiresAt").notNullable();
+      t.uuid("userId").notNullable();
+      t.uuid("orgId").notNullable();
+      t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE");
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+
+    await createOnUpdateTrigger(knex, TableName.SecretSharing);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.SecretSharing);
+}

backend/src/db/migrations/20240529060752_snap-shot-secret-index-secretversionid.ts

@@ -0,0 +1,21 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesSecretVersionIdExist = await knex.schema.hasColumn(TableName.SnapshotSecret, "secretVersionId");
+  if (await knex.schema.hasTable(TableName.SnapshotSecret)) {
+    await knex.schema.alterTable(TableName.SnapshotSecret, (t) => {
+      if (doesSecretVersionIdExist) t.index("secretVersionId");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesSecretVersionIdExist = await knex.schema.hasColumn(TableName.SnapshotSecret, "secretVersionId");
+  if (await knex.schema.hasTable(TableName.SnapshotSecret)) {
+    await knex.schema.alterTable(TableName.SnapshotSecret, (t) => {
+      if (doesSecretVersionIdExist) t.dropIndex("secretVersionId");
+    });
+  }
+}

backend/src/db/migrations/20240529203152_secret_sharing.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.SecretSharing))) {
+    await knex.schema.createTable(TableName.SecretSharing, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("name").notNullable();
+      t.text("encryptedValue").notNullable();
+      t.text("iv").notNullable();
+      t.text("tag").notNullable();
+      t.text("hashedHex").notNullable();
+      t.timestamp("expiresAt").notNullable();
+      t.uuid("userId").notNullable();
+      t.uuid("orgId").notNullable();
+      t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE");
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+
+    await createOnUpdateTrigger(knex, TableName.SecretSharing);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.SecretSharing);
+}

backend/src/db/migrations/20240530044702_universal-text-in-secret-sharing.ts

@@ -0,0 +1,33 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasExpiresAfterViewsColumn = await knex.schema.hasColumn(TableName.SecretSharing, "expiresAfterViews");
+  const hasSecretNameColumn = await knex.schema.hasColumn(TableName.SecretSharing, "name");
+
+  await knex.schema.alterTable(TableName.SecretSharing, (t) => {
+    if (!hasExpiresAfterViewsColumn) {
+      t.integer("expiresAfterViews");
+    }
+
+    if (hasSecretNameColumn) {
+      t.dropColumn("name");
+    }
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasExpiresAfterViewsColumn = await knex.schema.hasColumn(TableName.SecretSharing, "expiresAfterViews");
+  const hasSecretNameColumn = await knex.schema.hasColumn(TableName.SecretSharing, "name");
+
+  await knex.schema.alterTable(TableName.SecretSharing, (t) => {
+    if (hasExpiresAfterViewsColumn) {
+      t.dropColumn("expiresAfterViews");
+    }
+
+    if (!hasSecretNameColumn) {
+      t.string("name").notNullable();
+    }
+  });
+}

backend/src/db/migrations/20240531220007_secret-replication.ts

@@ -0,0 +1,85 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesSecretImportIsReplicationExist = await knex.schema.hasColumn(TableName.SecretImport, "isReplication");
+  const doesSecretImportIsReplicationSuccessExist = await knex.schema.hasColumn(
+    TableName.SecretImport,
+    "isReplicationSuccess"
+  );
+  const doesSecretImportReplicationStatusExist = await knex.schema.hasColumn(
+    TableName.SecretImport,
+    "replicationStatus"
+  );
+  const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated");
+  const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved");
+
+  if (await knex.schema.hasTable(TableName.SecretImport)) {
+    await knex.schema.alterTable(TableName.SecretImport, (t) => {
+      if (!doesSecretImportIsReplicationExist) t.boolean("isReplication").defaultTo(false);
+      if (!doesSecretImportIsReplicationSuccessExist) t.boolean("isReplicationSuccess").nullable();
+      if (!doesSecretImportReplicationStatusExist) t.text("replicationStatus").nullable();
+      if (!doesSecretImportLastReplicatedExist) t.datetime("lastReplicated").nullable();
+      if (!doesSecretImportIsReservedExist) t.boolean("isReserved").defaultTo(false);
+    });
+  }
+
+  const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved");
+  if (await knex.schema.hasTable(TableName.SecretFolder)) {
+    await knex.schema.alterTable(TableName.SecretFolder, (t) => {
+      if (!doesSecretFolderReservedExist) t.boolean("isReserved").defaultTo(false);
+    });
+  }
+
+  const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn(
+    TableName.SecretApprovalRequest,
+    "isReplicated"
+  );
+  if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) {
+    await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => {
+      if (!doesSecretApprovalRequestIsReplicatedExist) t.boolean("isReplicated");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesSecretImportIsReplicationExist = await knex.schema.hasColumn(TableName.SecretImport, "isReplication");
+  const doesSecretImportIsReplicationSuccessExist = await knex.schema.hasColumn(
+    TableName.SecretImport,
+    "isReplicationSuccess"
+  );
+  const doesSecretImportReplicationStatusExist = await knex.schema.hasColumn(
+    TableName.SecretImport,
+    "replicationStatus"
+  );
+  const doesSecretImportLastReplicatedExist = await knex.schema.hasColumn(TableName.SecretImport, "lastReplicated");
+  const doesSecretImportIsReservedExist = await knex.schema.hasColumn(TableName.SecretImport, "isReserved");
+
+  if (await knex.schema.hasTable(TableName.SecretImport)) {
+    await knex.schema.alterTable(TableName.SecretImport, (t) => {
+      if (doesSecretImportIsReplicationExist) t.dropColumn("isReplication");
+      if (doesSecretImportIsReplicationSuccessExist) t.dropColumn("isReplicationSuccess");
+      if (doesSecretImportReplicationStatusExist) t.dropColumn("replicationStatus");
+      if (doesSecretImportLastReplicatedExist) t.dropColumn("lastReplicated");
+      if (doesSecretImportIsReservedExist) t.dropColumn("isReserved");
+    });
+  }
+
+  const doesSecretFolderReservedExist = await knex.schema.hasColumn(TableName.SecretFolder, "isReserved");
+  if (await knex.schema.hasTable(TableName.SecretFolder)) {
+    await knex.schema.alterTable(TableName.SecretFolder, (t) => {
+      if (doesSecretFolderReservedExist) t.dropColumn("isReserved");
+    });
+  }
+
+  const doesSecretApprovalRequestIsReplicatedExist = await knex.schema.hasColumn(
+    TableName.SecretApprovalRequest,
+    "isReplicated"
+  );
+  if (await knex.schema.hasTable(TableName.SecretApprovalRequest)) {
+    await knex.schema.alterTable(TableName.SecretApprovalRequest, (t) => {
+      if (doesSecretApprovalRequestIsReplicatedExist) t.dropColumn("isReplicated");
+    });
+  }
+}

backend/src/db/migrations/20240603075514_kms.ts

@@ -0,0 +1,56 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.KmsServerRootConfig))) {
+    await knex.schema.createTable(TableName.KmsServerRootConfig, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.binary("encryptedRootKey").notNullable();
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.KmsServerRootConfig);
+
+  if (!(await knex.schema.hasTable(TableName.KmsKey))) {
+    await knex.schema.createTable(TableName.KmsKey, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.binary("encryptedKey").notNullable();
+      t.string("encryptionAlgorithm").notNullable();
+      t.integer("version").defaultTo(1).notNullable();
+      t.string("description");
+      t.boolean("isDisabled").defaultTo(false);
+      t.boolean("isReserved").defaultTo(true);
+      t.string("projectId");
+      t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      t.uuid("orgId");
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.KmsKey);
+
+  if (!(await knex.schema.hasTable(TableName.KmsKeyVersion))) {
+    await knex.schema.createTable(TableName.KmsKeyVersion, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.binary("encryptedKey").notNullable();
+      t.integer("version").notNullable();
+      t.uuid("kmsKeyId").notNullable();
+      t.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.KmsKeyVersion);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.KmsServerRootConfig);
+  await dropOnUpdateTrigger(knex, TableName.KmsServerRootConfig);
+
+  await knex.schema.dropTableIfExists(TableName.KmsKeyVersion);
+  await dropOnUpdateTrigger(knex, TableName.KmsKeyVersion);
+
+  await knex.schema.dropTableIfExists(TableName.KmsKey);
+  await dropOnUpdateTrigger(knex, TableName.KmsKey);
+}

backend/src/db/schemas/index.ts

@@ -30,6 +30,9 @@ export * from "./identity-universal-auths";
 export * from "./incident-contacts";
 export * from "./integration-auths";
 export * from "./integrations";
+export * from "./kms-key-versions";
+export * from "./kms-keys";
+export * from "./kms-root-config";
 export * from "./ldap-configs";
 export * from "./ldap-group-maps";
 export * from "./models";
@@ -57,9 +60,11 @@ export * from "./secret-blind-indexes";
 export * from "./secret-folder-versions";
 export * from "./secret-folders";
 export * from "./secret-imports";
+export * from "./secret-references";
 export * from "./secret-rotation-outputs";
 export * from "./secret-rotations";
 export * from "./secret-scanning-git-risks";
+export * from "./secret-sharing";
 export * from "./secret-snapshot-folders";
 export * from "./secret-snapshot-secrets";
 export * from "./secret-snapshots";

backend/src/db/schemas/kms-key-versions.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const KmsKeyVersionsSchema = z.object({
+  id: z.string().uuid(),
+  encryptedKey: zodBuffer,
+  version: z.number(),
+  kmsKeyId: z.string().uuid()
+});
+
+export type TKmsKeyVersions = z.infer<typeof KmsKeyVersionsSchema>;
+export type TKmsKeyVersionsInsert = Omit<z.input<typeof KmsKeyVersionsSchema>, TImmutableDBKeys>;
+export type TKmsKeyVersionsUpdate = Partial<Omit<z.input<typeof KmsKeyVersionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/kms-keys.ts

@@ -0,0 +1,26 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const KmsKeysSchema = z.object({
+  id: z.string().uuid(),
+  encryptedKey: zodBuffer,
+  encryptionAlgorithm: z.string(),
+  version: z.number().default(1),
+  description: z.string().nullable().optional(),
+  isDisabled: z.boolean().default(false).nullable().optional(),
+  isReserved: z.boolean().default(true).nullable().optional(),
+  projectId: z.string().nullable().optional(),
+  orgId: z.string().uuid().nullable().optional()
+});
+
+export type TKmsKeys = z.infer<typeof KmsKeysSchema>;
+export type TKmsKeysInsert = Omit<z.input<typeof KmsKeysSchema>, TImmutableDBKeys>;
+export type TKmsKeysUpdate = Partial<Omit<z.input<typeof KmsKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/kms-root-config.ts

@@ -0,0 +1,19 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { zodBuffer } from "@app/lib/zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const KmsRootConfigSchema = z.object({
+  id: z.string().uuid(),
+  encryptedRootKey: zodBuffer
+});
+
+export type TKmsRootConfig = z.infer<typeof KmsRootConfigSchema>;
+export type TKmsRootConfigInsert = Omit<z.input<typeof KmsRootConfigSchema>, TImmutableDBKeys>;
+export type TKmsRootConfigUpdate = Partial<Omit<z.input<typeof KmsRootConfigSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/models.ts

@@ -29,6 +29,7 @@ export enum TableName {
   ProjectKeys = "project_keys",
   Secret = "secrets",
   SecretReference = "secret_references",
+  SecretSharing = "secret_sharing",
   SecretBlindIndex = "secret_blind_indexes",
   SecretVersion = "secret_versions",
   SecretFolder = "secret_folders",
@@ -80,7 +81,11 @@ export enum TableName {
   DynamicSecretLease = "dynamic_secret_leases",
   // junction tables with tags
   JnSecretTag = "secret_tag_junction",
-  SecretVersionTag = "secret_version_tag_junction"
+  SecretVersionTag = "secret_version_tag_junction",
+  // KMS Service
+  KmsServerRootConfig = "kms_root_config",
+  KmsKey = "kms_keys",
+  KmsKeyVersion = "kms_key_versions"
 }
 
 export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";

backend/src/db/schemas/secret-approval-requests.ts

@@ -18,7 +18,8 @@ export const SecretApprovalRequestsSchema = z.object({
   statusChangeBy: z.string().uuid().nullable().optional(),
   committerId: z.string().uuid(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  isReplicated: z.boolean().nullable().optional()
 });
 
 export type TSecretApprovalRequests = z.infer<typeof SecretApprovalRequestsSchema>;

backend/src/db/schemas/secret-folders.ts

@@ -14,7 +14,8 @@ export const SecretFoldersSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   envId: z.string().uuid(),
-  parentId: z.string().uuid().nullable().optional()
+  parentId: z.string().uuid().nullable().optional(),
+  isReserved: z.boolean().default(false).nullable().optional()
 });
 
 export type TSecretFolders = z.infer<typeof SecretFoldersSchema>;

backend/src/db/schemas/secret-imports.ts

@@ -15,7 +15,12 @@ export const SecretImportsSchema = z.object({
   position: z.number(),
   createdAt: z.date(),
   updatedAt: z.date(),
-  folderId: z.string().uuid()
+  folderId: z.string().uuid(),
+  isReplication: z.boolean().default(false).nullable().optional(),
+  isReplicationSuccess: z.boolean().nullable().optional(),
+  replicationStatus: z.string().nullable().optional(),
+  lastReplicated: z.date().nullable().optional(),
+  isReserved: z.boolean().default(false).nullable().optional()
 });
 
 export type TSecretImports = z.infer<typeof SecretImportsSchema>;

backend/src/db/schemas/secret-sharing.ts

@@ -0,0 +1,26 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const SecretSharingSchema = z.object({
+  id: z.string().uuid(),
+  encryptedValue: z.string(),
+  iv: z.string(),
+  tag: z.string(),
+  hashedHex: z.string(),
+  expiresAt: z.date(),
+  userId: z.string().uuid(),
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  expiresAfterViews: z.number().nullable().optional()
+});
+
+export type TSecretSharing = z.infer<typeof SecretSharingSchema>;
+export type TSecretSharingInsert = Omit<z.input<typeof SecretSharingSchema>, TImmutableDBKeys>;
+export type TSecretSharingUpdate = Partial<Omit<z.input<typeof SecretSharingSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/users.ts

@@ -23,8 +23,8 @@ export const UsersSchema = z.object({
   isGhost: z.boolean().default(false),
   username: z.string(),
   isEmailVerified: z.boolean().default(false).nullable().optional(),
-  consecutiveFailedMfaAttempts: z.number().optional(),
-  isLocked: z.boolean().optional(),
+  consecutiveFailedMfaAttempts: z.number().default(0).nullable().optional(),
+  isLocked: z.boolean().default(false).nullable().optional(),
   temporaryLockDateEnd: z.date().nullable().optional()
 });
 

backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts

@@ -5,10 +5,15 @@ import { z } from "zod";
 
 import { IdentityProjectAdditionalPrivilegeTemporaryMode } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-types";
 import { IDENTITY_ADDITIONAL_PRIVILEGE } from "@app/lib/api-docs";
+import { BadRequestError } from "@app/lib/errors";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
-import { ProjectPermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas";
+import {
+  ProjectPermissionSchema,
+  ProjectSpecificPrivilegePermissionSchema,
+  SanitizedIdentityPrivilegeSchema
+} from "@app/server/routes/sanitizedSchemas";
 import { AuthMode } from "@app/services/auth/auth-type";
 
 export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: FastifyZodProvider) => {
@@ -39,7 +44,12 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
           })
           .optional()
           .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug),
-        permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
+        permissions: ProjectPermissionSchema.array()
+          .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
+          .optional(),
+        privilegePermission: ProjectSpecificPrivilegePermissionSchema.describe(
+          IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.privilegePermission
+        ).optional()
       }),
       response: {
         200: z.object({
@@ -49,6 +59,18 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
+      const { permissions, privilegePermission } = req.body;
+      if (!permissions && !privilegePermission) {
+        throw new BadRequestError({ message: "Permission or privilegePermission must be provided" });
+      }
+
+      const permission = privilegePermission
+        ? privilegePermission.actions.map((action) => ({
+            action,
+            subject: privilegePermission.subject,
+            conditions: privilegePermission.conditions
+          }))
+        : permissions!;
       const privilege = await server.services.identityProjectAdditionalPrivilege.create({
         actorId: req.permission.id,
         actor: req.permission.type,
@@ -57,7 +79,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
         ...req.body,
         slug: req.body.slug ? slugify(req.body.slug) : slugify(alphaNumericNanoId(12)),
         isTemporary: false,
-        permissions: JSON.stringify(packRules(req.body.permissions))
+        permissions: JSON.stringify(packRules(permission))
       });
       return { privilege };
     }
@@ -90,7 +112,12 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
           })
           .optional()
           .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug),
-        permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions),
+        permissions: ProjectPermissionSchema.array()
+          .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
+          .optional(),
+        privilegePermission: ProjectSpecificPrivilegePermissionSchema.describe(
+          IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.privilegePermission
+        ).optional(),
         temporaryMode: z
           .nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode)
           .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.temporaryMode),
@@ -111,6 +138,19 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
+      const { permissions, privilegePermission } = req.body;
+      if (!permissions && !privilegePermission) {
+        throw new BadRequestError({ message: "Permission or privilegePermission must be provided" });
+      }
+
+      const permission = privilegePermission
+        ? privilegePermission.actions.map((action) => ({
+            action,
+            subject: privilegePermission.subject,
+            conditions: privilegePermission.conditions
+          }))
+        : permissions!;
+
       const privilege = await server.services.identityProjectAdditionalPrivilege.create({
         actorId: req.permission.id,
         actor: req.permission.type,
@@ -119,7 +159,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
         ...req.body,
         slug: req.body.slug ? slugify(req.body.slug) : slugify(alphaNumericNanoId(12)),
         isTemporary: true,
-        permissions: JSON.stringify(packRules(req.body.permissions))
+        permissions: JSON.stringify(packRules(permission))
       });
       return { privilege };
     }
@@ -156,13 +196,16 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
               })
               .describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.newSlug),
             permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions),
+            privilegePermission: ProjectSpecificPrivilegePermissionSchema.describe(
+              IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.privilegePermission
+            ).optional(),
             isTemporary: z.boolean().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.isTemporary),
             temporaryMode: z
               .nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode)
               .describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.temporaryMode),
             temporaryRange: z
               .string()
-              .refine((val) => ms(val) > 0, "Temporary range must be a positive number")
+              .refine((val) => typeof val === "undefined" || ms(val) > 0, "Temporary range must be a positive number")
               .describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.temporaryRange),
             temporaryAccessStartTime: z
               .string()
@@ -179,7 +222,18 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      const updatedInfo = req.body.privilegeDetails;
+      const { permissions, privilegePermission, ...updatedInfo } = req.body.privilegeDetails;
+      if (!permissions && !privilegePermission) {
+        throw new BadRequestError({ message: "Permission or privilegePermission must be provided" });
+      }
+
+      const permission = privilegePermission
+        ? privilegePermission.actions.map((action) => ({
+            action,
+            subject: privilegePermission.subject,
+            conditions: privilegePermission.conditions
+          }))
+        : permissions!;
       const privilege = await server.services.identityProjectAdditionalPrivilege.updateBySlug({
         actorId: req.permission.id,
         actor: req.permission.type,
@@ -190,7 +244,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
         projectSlug: req.body.projectSlug,
         data: {
           ...updatedInfo,
-          permissions: updatedInfo?.permissions ? JSON.stringify(packRules(updatedInfo.permissions)) : undefined
+          permissions: permission ? JSON.stringify(packRules(permission)) : undefined
         }
       });
       return { privilege };

backend/src/ee/routes/v1/org-role-router.ts

@@ -23,7 +23,7 @@ export const registerOrgRoleRouter = async (server: FastifyZodProvider) => {
           .min(1)
           .trim()
           .refine(
-            (val) => !Object.keys(OrgMembershipRole).includes(val),
+            (val) => !Object.values(OrgMembershipRole).includes(val as OrgMembershipRole),
             "Please choose a different slug, the slug you have entered is reserved"
           )
           .refine((v) => slugify(v) === v, {

backend/src/ee/routes/v1/project-role-router.ts

@@ -1,146 +1,232 @@
+import { packRules } from "@casl/ability/extra";
+import slugify from "@sindresorhus/slugify";
 import { z } from "zod";
 
-import { ProjectMembershipsSchema, ProjectRolesSchema } from "@app/db/schemas";
+import { ProjectMembershipRole, ProjectMembershipsSchema, ProjectRolesSchema } from "@app/db/schemas";
+import { PROJECT_ROLE } from "@app/lib/api-docs";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { ProjectPermissionSchema, SanitizedRoleSchema } from "@app/server/routes/sanitizedSchemas";
 import { AuthMode } from "@app/services/auth/auth-type";
 
 export const registerProjectRoleRouter = async (server: FastifyZodProvider) => {
   server.route({
     method: "POST",
-    url: "/:projectId/roles",
+    url: "/:projectSlug/roles",
     config: {
       rateLimit: writeLimit
     },
     schema: {
+      description: "Create a project role",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
       params: z.object({
-        projectId: z.string().trim()
+        projectSlug: z.string().trim().describe(PROJECT_ROLE.CREATE.projectSlug)
       }),
       body: z.object({
-        slug: z.string().trim(),
-        name: z.string().trim(),
-        description: z.string().trim().optional(),
-        permissions: z.any().array()
+        slug: z
+          .string()
+          .toLowerCase()
+          .trim()
+          .min(1)
+          .refine(
+            (val) => !Object.values(ProjectMembershipRole).includes(val as ProjectMembershipRole),
+            "Please choose a different slug, the slug you have entered is reserved"
+          )
+          .refine((v) => slugify(v) === v, {
+            message: "Slug must be a valid"
+          })
+          .describe(PROJECT_ROLE.CREATE.slug),
+        name: z.string().min(1).trim().describe(PROJECT_ROLE.CREATE.name),
+        description: z.string().trim().optional().describe(PROJECT_ROLE.CREATE.description),
+        permissions: ProjectPermissionSchema.array().describe(PROJECT_ROLE.CREATE.permissions)
       }),
       response: {
         200: z.object({
-          role: ProjectRolesSchema
+          role: SanitizedRoleSchema
         })
       }
     },
-    onRequest: verifyAuth([AuthMode.JWT]),
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      const role = await server.services.projectRole.createRole(
-        req.permission.type,
-        req.permission.id,
-        req.params.projectId,
-        req.body,
-        req.permission.authMethod,
-        req.permission.orgId
-      );
+      const role = await server.services.projectRole.createRole({
+        actorAuthMethod: req.permission.authMethod,
+        actorId: req.permission.id,
+        actorOrgId: req.permission.orgId,
+        actor: req.permission.type,
+        projectSlug: req.params.projectSlug,
+        data: {
+          ...req.body,
+          permissions: JSON.stringify(packRules(req.body.permissions))
+        }
+      });
       return { role };
     }
   });
 
   server.route({
     method: "PATCH",
-    url: "/:projectId/roles/:roleId",
+    url: "/:projectSlug/roles/:roleId",
     config: {
       rateLimit: writeLimit
     },
     schema: {
+      description: "Update a project role",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
       params: z.object({
-        projectId: z.string().trim(),
-        roleId: z.string().trim()
+        projectSlug: z.string().trim().describe(PROJECT_ROLE.UPDATE.projectSlug),
+        roleId: z.string().trim().describe(PROJECT_ROLE.UPDATE.roleId)
       }),
       body: z.object({
-        slug: z.string().trim().optional(),
-        name: z.string().trim().optional(),
-        description: z.string().trim().optional(),
-        permissions: z.any().array()
+        slug: z
+          .string()
+          .toLowerCase()
+          .trim()
+          .optional()
+          .describe(PROJECT_ROLE.UPDATE.slug)
+          .refine(
+            (val) =>
+              typeof val === "undefined" ||
+              !Object.values(ProjectMembershipRole).includes(val as ProjectMembershipRole),
+            "Please choose a different slug, the slug you have entered is reserved"
+          )
+          .refine((val) => typeof val === "undefined" || slugify(val) === val, {
+            message: "Slug must be a valid"
+          }),
+        name: z.string().trim().optional().describe(PROJECT_ROLE.UPDATE.name),
+        permissions: ProjectPermissionSchema.array().describe(PROJECT_ROLE.UPDATE.permissions)
       }),
       response: {
         200: z.object({
-          role: ProjectRolesSchema
+          role: SanitizedRoleSchema
         })
       }
     },
-    onRequest: verifyAuth([AuthMode.JWT]),
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      const role = await server.services.projectRole.updateRole(
-        req.permission.type,
-        req.permission.id,
-        req.params.projectId,
-        req.params.roleId,
-        req.body,
-        req.permission.authMethod,
-        req.permission.orgId
-      );
+      const role = await server.services.projectRole.updateRole({
+        actorAuthMethod: req.permission.authMethod,
+        actorId: req.permission.id,
+        actorOrgId: req.permission.orgId,
+        actor: req.permission.type,
+        projectSlug: req.params.projectSlug,
+        roleId: req.params.roleId,
+        data: {
+          ...req.body,
+          permissions: JSON.stringify(packRules(req.body.permissions))
+        }
+      });
       return { role };
     }
   });
 
   server.route({
     method: "DELETE",
-    url: "/:projectId/roles/:roleId",
+    url: "/:projectSlug/roles/:roleId",
     config: {
       rateLimit: writeLimit
     },
     schema: {
+      description: "Delete a project role",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
       params: z.object({
-        projectId: z.string().trim(),
-        roleId: z.string().trim()
+        projectSlug: z.string().trim().describe(PROJECT_ROLE.DELETE.projectSlug),
+        roleId: z.string().trim().describe(PROJECT_ROLE.DELETE.roleId)
       }),
       response: {
         200: z.object({
-          role: ProjectRolesSchema
+          role: SanitizedRoleSchema
         })
       }
     },
-    onRequest: verifyAuth([AuthMode.JWT]),
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      const role = await server.services.projectRole.deleteRole(
-        req.permission.type,
-        req.permission.id,
-        req.params.projectId,
-        req.params.roleId,
-        req.permission.authMethod,
-        req.permission.orgId
-      );
+      const role = await server.services.projectRole.deleteRole({
+        actorAuthMethod: req.permission.authMethod,
+        actorId: req.permission.id,
+        actorOrgId: req.permission.orgId,
+        actor: req.permission.type,
+        projectSlug: req.params.projectSlug,
+        roleId: req.params.roleId
+      });
       return { role };
     }
   });
 
   server.route({
     method: "GET",
-    url: "/:projectId/roles",
+    url: "/:projectSlug/roles",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      description: "List project role",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        projectSlug: z.string().trim().describe(PROJECT_ROLE.LIST.projectSlug)
+      }),
+      response: {
+        200: z.object({
+          roles: ProjectRolesSchema.omit({ permissions: true }).array()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const roles = await server.services.projectRole.listRoles({
+        actorAuthMethod: req.permission.authMethod,
+        actorId: req.permission.id,
+        actorOrgId: req.permission.orgId,
+        actor: req.permission.type,
+        projectSlug: req.params.projectSlug
+      });
+      return { roles };
+    }
+  });
+
+  server.route({
+    method: "GET",
+    url: "/:projectSlug/roles/slug/:slug",
     config: {
       rateLimit: readLimit
     },
     schema: {
       params: z.object({
-        projectId: z.string().trim()
+        projectSlug: z.string().trim().describe(PROJECT_ROLE.GET_ROLE_BY_SLUG.projectSlug),
+        slug: z.string().trim().describe(PROJECT_ROLE.GET_ROLE_BY_SLUG.roleSlug)
       }),
       response: {
         200: z.object({
-          data: z.object({
-            roles: ProjectRolesSchema.omit({ permissions: true })
-              .merge(z.object({ permissions: z.unknown() }))
-              .array()
-          })
+          role: SanitizedRoleSchema
         })
       }
     },
-    onRequest: verifyAuth([AuthMode.JWT]),
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      const roles = await server.services.projectRole.listRoles(
-        req.permission.type,
-        req.permission.id,
-        req.params.projectId,
-        req.permission.authMethod,
-        req.permission.orgId
-      );
-      return { data: { roles } };
+      const role = await server.services.projectRole.getRoleBySlug({
+        actorAuthMethod: req.permission.authMethod,
+        actorId: req.permission.id,
+        actorOrgId: req.permission.orgId,
+        actor: req.permission.type,
+        projectSlug: req.params.projectSlug,
+        roleSlug: req.params.slug
+      });
+      return { role };
     }
   });
 

backend/src/ee/routes/v1/secret-approval-request-router.ts

@@ -32,22 +32,20 @@ export const registerSecretApprovalRequestRouter = async (server: FastifyZodProv
       }),
       response: {
         200: z.object({
-          approvals: SecretApprovalRequestsSchema.merge(
-            z.object({
-              // secretPath: z.string(),
-              policy: z.object({
-                id: z.string(),
-                name: z.string(),
-                approvals: z.number(),
-                approvers: z.string().array(),
-                secretPath: z.string().optional().nullable()
-              }),
-              commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(),
-              environment: z.string(),
-              reviewers: z.object({ member: z.string(), status: z.string() }).array(),
-              approvers: z.string().array()
-            })
-          ).array()
+          approvals: SecretApprovalRequestsSchema.extend({
+            // secretPath: z.string(),
+            policy: z.object({
+              id: z.string(),
+              name: z.string(),
+              approvals: z.number(),
+              approvers: z.string().array(),
+              secretPath: z.string().optional().nullable()
+            }),
+            commits: z.object({ op: z.string(), secretId: z.string().nullable().optional() }).array(),
+            environment: z.string(),
+            reviewers: z.object({ member: z.string(), status: z.string() }).array(),
+            approvers: z.string().array()
+          }).array()
         })
       }
     },

backend/src/ee/services/license/license-dal.ts

@@ -16,6 +16,8 @@ export const licenseDALFactory = (db: TDbClient) => {
             void bd.where({ orgId });
           }
         })
+        .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
+        .where(`${TableName.Users}.isGhost`, false)
         .count();
       return doc?.[0].count;
     } catch (error) {

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -15,9 +15,16 @@ import { ActorType } from "@app/services/auth/auth-type";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
-import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
+import {
+  fnSecretBlindIndexCheck,
+  fnSecretBlindIndexCheckV2,
+  fnSecretBulkDelete,
+  fnSecretBulkInsert,
+  fnSecretBulkUpdate,
+  getAllNestedSecretReferences
+} from "@app/services/secret/secret-fns";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
-import { TSecretServiceFactory } from "@app/services/secret/secret-service";
+import { SecretOperations } from "@app/services/secret/secret-types";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
@@ -32,7 +39,6 @@ import { TSecretApprovalRequestReviewerDALFactory } from "./secret-approval-requ
 import { TSecretApprovalRequestSecretDALFactory } from "./secret-approval-request-secret-dal";
 import {
   ApprovalStatus,
-  CommitType,
   RequestState,
   TApprovalRequestCountDTO,
   TGenerateSecretApprovalRequestDTO,
@@ -45,10 +51,11 @@ import {
 
 type TSecretApprovalRequestServiceFactoryDep = {
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretApprovalRequestDAL: TSecretApprovalRequestDALFactory;
   secretApprovalRequestSecretDAL: TSecretApprovalRequestSecretDALFactory;
   secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
-  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
+  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findSecretPathByFolderIds">;
   secretDAL: TSecretDALFactory;
   secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
   secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
@@ -56,16 +63,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
-  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
-  secretService: Pick<
-    TSecretServiceFactory,
-    | "fnSecretBulkInsert"
-    | "fnSecretBulkUpdate"
-    | "fnSecretBlindIndexCheck"
-    | "fnSecretBulkDelete"
-    | "fnSecretBlindIndexCheckV2"
-  >;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "removeSecretReminder">;
 };
 
 export type TSecretApprovalRequestServiceFactory = ReturnType<typeof secretApprovalRequestServiceFactory>;
@@ -82,7 +80,6 @@ export const secretApprovalRequestServiceFactory = ({
   projectDAL,
   permissionService,
   snapshotService,
-  secretService,
   secretVersionDAL,
   secretQueueService,
   projectBotService
@@ -302,11 +299,12 @@ export const secretApprovalRequestServiceFactory = ({
     const secretApprovalSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
     if (!secretApprovalSecrets) throw new BadRequestError({ message: "No secrets found" });
 
-    const conflicts: Array<{ secretId: string; op: CommitType }> = [];
-    let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Create);
+    const conflicts: Array<{ secretId: string; op: SecretOperations }> = [];
+    let secretCreationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Create);
     if (secretCreationCommits.length) {
-      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
+      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
         folderId,
+        secretDAL,
         inputSecrets: secretCreationCommits.map(({ secretBlindIndex }) => {
           if (!secretBlindIndex) {
             throw new BadRequestError({
@@ -319,17 +317,19 @@ export const secretApprovalRequestServiceFactory = ({
       secretCreationCommits
         .filter(({ secretBlindIndex }) => conflictGroupByBlindIndex[secretBlindIndex || ""])
         .forEach((el) => {
-          conflicts.push({ op: CommitType.Create, secretId: el.id });
+          conflicts.push({ op: SecretOperations.Create, secretId: el.id });
         });
       secretCreationCommits = secretCreationCommits.filter(
         ({ secretBlindIndex }) => !conflictGroupByBlindIndex[secretBlindIndex || ""]
       );
     }
 
-    let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Update);
+    let secretUpdationCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Update);
     if (secretUpdationCommits.length) {
-      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await secretService.fnSecretBlindIndexCheckV2({
+      const { secsGroupedByBlindIndex: conflictGroupByBlindIndex } = await fnSecretBlindIndexCheckV2({
         folderId,
+        secretDAL,
+        userId: "",
         inputSecrets: secretUpdationCommits
           .filter(({ secretBlindIndex, secret }) => secret && secret.secretBlindIndex !== secretBlindIndex)
           .map(({ secretBlindIndex }) => {
@@ -347,7 +347,7 @@ export const secretApprovalRequestServiceFactory = ({
             (secretBlindIndex && conflictGroupByBlindIndex[secretBlindIndex]) || !secretId
         )
         .forEach((el) => {
-          conflicts.push({ op: CommitType.Update, secretId: el.id });
+          conflicts.push({ op: SecretOperations.Update, secretId: el.id });
         });
 
       secretUpdationCommits = secretUpdationCommits.filter(
@@ -356,11 +356,11 @@ export const secretApprovalRequestServiceFactory = ({
       );
     }
 
-    const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
+    const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === SecretOperations.Delete);
     const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
     const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
       const newSecrets = secretCreationCommits.length
-        ? await secretService.fnSecretBulkInsert({
+        ? await fnSecretBulkInsert({
             tx,
             folderId,
             inputSecrets: secretCreationCommits.map((el) => ({
@@ -403,7 +403,7 @@ export const secretApprovalRequestServiceFactory = ({
           })
         : [];
       const updatedSecrets = secretUpdationCommits.length
-        ? await secretService.fnSecretBulkUpdate({
+        ? await fnSecretBulkUpdate({
             folderId,
             projectId,
             tx,
@@ -449,11 +449,13 @@ export const secretApprovalRequestServiceFactory = ({
           })
         : [];
       const deletedSecret = secretDeletionCommits.length
-        ? await secretService.fnSecretBulkDelete({
+        ? await fnSecretBulkDelete({
             projectId,
             folderId,
             tx,
             actorId: "",
+            secretDAL,
+            secretQueueService,
             inputSecrets: secretDeletionCommits.map(({ secretBlindIndex }) => {
               if (!secretBlindIndex) {
                 throw new BadRequestError({
@@ -480,12 +482,14 @@ export const secretApprovalRequestServiceFactory = ({
       };
     });
     await snapshotService.performSnapshot(folderId);
-    const folder = await folderDAL.findById(folderId);
-    // TODO(akhilmhdh-pg):  change query to do secret path from folder
+    const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
+    if (!folder) throw new BadRequestError({ message: "Folder not found" });
     await secretQueueService.syncSecrets({
       projectId,
-      secretPath: "/",
-      environment: folder?.environment.envSlug as string
+      secretPath: folder.path,
+      environmentSlug: folder.environmentSlug,
+      actorId,
+      actor
     });
     return mergeStatus;
   };
@@ -533,9 +537,9 @@ export const secretApprovalRequestServiceFactory = ({
     const commits: Omit<TSecretApprovalRequestsSecretsInsert, "requestId">[] = [];
     const commitTagIds: Record<string, string[]> = {};
     // for created secret approval change
-    const createdSecrets = data[CommitType.Create];
+    const createdSecrets = data[SecretOperations.Create];
     if (createdSecrets && createdSecrets?.length) {
-      const { keyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
         inputSecrets: createdSecrets,
         folderId,
         isNew: true,
@@ -546,7 +550,7 @@ export const secretApprovalRequestServiceFactory = ({
       commits.push(
         ...createdSecrets.map(({ secretName, ...el }) => ({
           ...el,
-          op: CommitType.Create as const,
+          op: SecretOperations.Create as const,
           version: 1,
           secretBlindIndex: keyName2BlindIndex[secretName],
           algorithm: SecretEncryptionAlgo.AES_256_GCM,
@@ -558,12 +562,12 @@ export const secretApprovalRequestServiceFactory = ({
       });
     }
     // not secret approval for update operations
-    const updatedSecrets = data[CommitType.Update];
+    const updatedSecrets = data[SecretOperations.Update];
     if (updatedSecrets && updatedSecrets?.length) {
       // get all blind index
       // Find all those secrets
       // if not throw not found
-      const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex, secrets: secretsToBeUpdated } = await fnSecretBlindIndexCheck({
         inputSecrets: updatedSecrets,
         folderId,
         isNew: false,
@@ -574,8 +578,8 @@ export const secretApprovalRequestServiceFactory = ({
       // now find any secret that needs to update its name
       // same process as above
       const nameUpdatedSecrets = updatedSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
-      const { keyName2BlindIndex: newKeyName2BlindIndex } = await secretService.fnSecretBlindIndexCheck({
-        inputSecrets: nameUpdatedSecrets,
+      const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
+        inputSecrets: nameUpdatedSecrets.map(({ newSecretName }) => ({ secretName: newSecretName as string })),
         folderId,
         isNew: true,
         blindIndexCfg,
@@ -592,14 +596,14 @@ export const secretApprovalRequestServiceFactory = ({
           const secretId = secsGroupedByBlindIndex[keyName2BlindIndex[secretName]][0].id;
           const secretBlindIndex =
             newSecretName && newKeyName2BlindIndex[newSecretName]
-              ? newKeyName2BlindIndex?.[secretName]
+              ? newKeyName2BlindIndex?.[newSecretName]
               : keyName2BlindIndex[secretName];
           // add tags
           if (tagIds?.length) commitTagIds[keyName2BlindIndex[secretName]] = tagIds;
           return {
             ...latestSecretVersions[secretId],
             ...el,
-            op: CommitType.Update as const,
+            op: SecretOperations.Update as const,
             secret: secretId,
             secretVersion: latestSecretVersions[secretId].id,
             secretBlindIndex,
@@ -609,12 +613,12 @@ export const secretApprovalRequestServiceFactory = ({
       );
     }
     // deleted secrets
-    const deletedSecrets = data[CommitType.Delete];
+    const deletedSecrets = data[SecretOperations.Delete];
     if (deletedSecrets && deletedSecrets.length) {
       // get all blind index
       // Find all those secrets
       // if not throw not found
-      const { keyName2BlindIndex, secrets } = await secretService.fnSecretBlindIndexCheck({
+      const { keyName2BlindIndex, secrets } = await fnSecretBlindIndexCheck({
         inputSecrets: deletedSecrets,
         folderId,
         isNew: false,
@@ -635,7 +639,7 @@ export const secretApprovalRequestServiceFactory = ({
           if (!latestSecretVersions[secretId].secretBlindIndex)
             throw new BadRequestError({ message: "Failed to find secret blind index" });
           return {
-            op: CommitType.Delete as const,
+            op: SecretOperations.Delete as const,
             ...latestSecretVersions[secretId],
             secretBlindIndex: latestSecretVersions[secretId].secretBlindIndex as string,
             secret: secretId,

backend/src/ee/services/secret-approval-request/secret-approval-request-types.ts

@@ -1,11 +1,6 @@
 import { TImmutableDBKeys, TSecretApprovalPolicies, TSecretApprovalRequestsSecrets } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
-
-export enum CommitType {
-  Create = "create",
-  Update = "update",
-  Delete = "delete"
-}
+import { SecretOperations } from "@app/services/secret/secret-types";
 
 export enum RequestState {
   Open = "open",
@@ -18,14 +13,14 @@ export enum ApprovalStatus {
   REJECTED = "rejected"
 }
 
-type TApprovalCreateSecret = Omit<
+export type TApprovalCreateSecret = Omit<
   TSecretApprovalRequestsSecrets,
   TImmutableDBKeys | "version" | "algorithm" | "keyEncoding" | "requestId" | "op" | "secretVersion" | "secretBlindIndex"
 > & {
   secretName: string;
   tagIds?: string[];
 };
-type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
+export type TApprovalUpdateSecret = Partial<TApprovalCreateSecret> & {
   secretName: string;
   newSecretName?: string;
   tagIds?: string[];
@@ -36,9 +31,9 @@ export type TGenerateSecretApprovalRequestDTO = {
   secretPath: string;
   policy: TSecretApprovalPolicies;
   data: {
-    [CommitType.Create]?: TApprovalCreateSecret[];
-    [CommitType.Update]?: TApprovalUpdateSecret[];
-    [CommitType.Delete]?: { secretName: string }[];
+    [SecretOperations.Create]?: TApprovalCreateSecret[];
+    [SecretOperations.Update]?: TApprovalUpdateSecret[];
+    [SecretOperations.Delete]?: { secretName: string }[];
   };
 } & TProjectPermission;
 

backend/src/ee/services/secret-replication/secret-replication-constants.ts

@@ -0,0 +1 @@
+export const MAX_REPLICATION_DEPTH = 5;

backend/src/ee/services/secret-replication/secret-replication-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TSecretReplicationDALFactory = ReturnType<typeof secretReplicationDALFactory>;
+
+export const secretReplicationDALFactory = (db: TDbClient) => {
+  const orm = ormify(db, TableName.SecretVersion);
+  return orm;
+};

backend/src/ee/services/secret-replication/secret-replication-service.ts

@@ -0,0 +1,485 @@
+import { SecretType, TSecrets } from "@app/db/schemas";
+import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
+import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
+import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
+import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { BadRequestError } from "@app/lib/errors";
+import { groupBy, unique } from "@app/lib/fn";
+import { logger } from "@app/lib/logger";
+import { alphaNumericNanoId } from "@app/lib/nanoid";
+import { QueueName, TQueueServiceFactory } from "@app/queue";
+import { ActorType } from "@app/services/auth/auth-type";
+import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
+import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
+import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { fnSecretBulkInsert, fnSecretBulkUpdate } from "@app/services/secret/secret-fns";
+import { TSecretQueueFactory, uniqueSecretQueueKey } from "@app/services/secret/secret-queue";
+import { SecretOperations } from "@app/services/secret/secret-types";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { ReservedFolders } from "@app/services/secret-folder/secret-folder-types";
+import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
+import { fnSecretsFromImports } from "@app/services/secret-import/secret-import-fns";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+
+import { MAX_REPLICATION_DEPTH } from "./secret-replication-constants";
+
+type TSecretReplicationServiceFactoryDep = {
+  secretDAL: Pick<
+    TSecretDALFactory,
+    "find" | "findByBlindIndexes" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
+  >;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "updateById" | "findByFolderIds">;
+  folderDAL: Pick<
+    TSecretFolderDALFactory,
+    "findSecretPathByFolderIds" | "findBySecretPath" | "create" | "findOne" | "findByManySecretPath"
+  >;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "find" | "insertMany">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "replicateSecrets">;
+  queueService: Pick<TQueueServiceFactory, "start" | "listen" | "queue" | "stopJobById">;
+  secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
+  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
+  secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret" | "find">;
+  secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;
+  projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findOne">;
+  secretApprovalRequestSecretDAL: Pick<
+    TSecretApprovalRequestSecretDALFactory,
+    "insertMany" | "insertApprovalSecretTags"
+  >;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
+};
+
+export type TSecretReplicationServiceFactory = ReturnType<typeof secretReplicationServiceFactory>;
+const SECRET_IMPORT_SUCCESS_LOCK = 10;
+
+const keystoreReplicationSuccessKey = (jobId: string, secretImportId: string) => `${jobId}-${secretImportId}`;
+const getReplicationKeyLockPrefix = (projectId: string, environmentSlug: string, secretPath: string) =>
+  `REPLICATION_SECRET_${projectId}-${environmentSlug}-${secretPath}`;
+export const getReplicationFolderName = (importId: string) => `${ReservedFolders.SecretReplication}${importId}`;
+
+const getDecryptedKeyValue = (key: string, secret: TSecrets) => {
+  const secretKey = decryptSymmetric128BitHexKeyUTF8({
+    ciphertext: secret.secretKeyCiphertext,
+    iv: secret.secretKeyIV,
+    tag: secret.secretKeyTag,
+    key
+  });
+
+  const secretValue = decryptSymmetric128BitHexKeyUTF8({
+    ciphertext: secret.secretValueCiphertext,
+    iv: secret.secretValueIV,
+    tag: secret.secretValueTag,
+    key
+  });
+  return { key: secretKey, value: secretValue };
+};
+
+export const secretReplicationServiceFactory = ({
+  secretDAL,
+  queueService,
+  secretVersionDAL,
+  secretImportDAL,
+  keyStore,
+  secretVersionTagDAL,
+  secretTagDAL,
+  folderDAL,
+  secretApprovalPolicyService,
+  secretApprovalRequestSecretDAL,
+  secretApprovalRequestDAL,
+  secretQueueService,
+  projectMembershipDAL,
+  projectBotService
+}: TSecretReplicationServiceFactoryDep) => {
+  const getReplicatedSecrets = (
+    botKey: string,
+    localSecrets: TSecrets[],
+    importedSecrets: { secrets: TSecrets[] }[]
+  ) => {
+    const deDupe = new Set<string>();
+    const secrets = localSecrets
+      .filter(({ secretBlindIndex }) => Boolean(secretBlindIndex))
+      .map((el) => {
+        const decryptedSecret = getDecryptedKeyValue(botKey, el);
+        deDupe.add(decryptedSecret.key);
+        return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value };
+      });
+
+    for (let i = importedSecrets.length - 1; i >= 0; i = -1) {
+      importedSecrets[i].secrets.forEach((el) => {
+        const decryptedSecret = getDecryptedKeyValue(botKey, el);
+        if (deDupe.has(decryptedSecret.key) || !el.secretBlindIndex) {
+          return;
+        }
+        deDupe.add(decryptedSecret.key);
+        secrets.push({ ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value });
+      });
+    }
+    return secrets;
+  };
+
+  // IMPORTANT NOTE BEFORE READING THE FUNCTION
+  // SOURCE - Where secrets are copied from
+  // DESTINATION - Where the replicated imports that points to SOURCE from Destination
+  queueService.start(QueueName.SecretReplication, async (job) => {
+    logger.info(job.data, "Replication started");
+    const {
+      secretPath,
+      environmentSlug,
+      projectId,
+      actorId,
+      actor,
+      pickOnlyImportIds,
+      _deDupeReplicationQueue: deDupeReplicationQueue,
+      _deDupeQueue: deDupeQueue,
+      _depth: depth = 0
+    } = job.data;
+    if (depth > MAX_REPLICATION_DEPTH) return;
+
+    const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
+    if (!folder) return;
+
+    // the the replicated imports made to the source. These are the destinations
+    const destinationSecretImports = await secretImportDAL.find({
+      importPath: secretPath,
+      importEnv: folder.envId
+    });
+
+    // CASE: normal mode <- link import  <- replicated import
+    const nonReplicatedDestinationImports = destinationSecretImports.filter(({ isReplication }) => !isReplication);
+    if (nonReplicatedDestinationImports.length) {
+      // keep calling sync secret for all the imports made
+      const importedFolderIds = unique(nonReplicatedDestinationImports, (i) => i.folderId).map(
+        ({ folderId }) => folderId
+      );
+      const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
+      const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
+      await Promise.all(
+        nonReplicatedDestinationImports
+          .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string))
+          // filter out already synced ones
+          .filter(
+            ({ folderId }) =>
+              !deDupeQueue?.[
+                uniqueSecretQueueKey(
+                  foldersGroupedById[folderId][0]?.environmentSlug as string,
+                  foldersGroupedById[folderId][0]?.path as string
+                )
+              ]
+          )
+          .map(({ folderId }) =>
+            secretQueueService.replicateSecrets({
+              projectId,
+              secretPath: foldersGroupedById[folderId][0]?.path as string,
+              environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
+              actorId,
+              actor,
+              _depth: depth + 1,
+              _deDupeReplicationQueue: deDupeReplicationQueue,
+              _deDupeQueue: deDupeQueue
+            })
+          )
+      );
+    }
+
+    let destinationReplicatedSecretImports = destinationSecretImports.filter(({ isReplication }) =>
+      Boolean(isReplication)
+    );
+    destinationReplicatedSecretImports = pickOnlyImportIds
+      ? destinationReplicatedSecretImports.filter(({ id }) => pickOnlyImportIds?.includes(id))
+      : destinationReplicatedSecretImports;
+    if (!destinationReplicatedSecretImports.length) return;
+
+    const botKey = await projectBotService.getBotKey(projectId);
+
+    // these are the secrets to be added in replicated folders
+    const sourceLocalSecrets = await secretDAL.find({ folderId: folder.id, type: SecretType.Shared });
+    const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
+    const sourceImportedSecrets = await fnSecretsFromImports({
+      allowedImports: sourceSecretImports,
+      secretDAL,
+      folderDAL,
+      secretImportDAL
+    });
+    // secrets that gets replicated across imports
+    const sourceSecrets = getReplicatedSecrets(botKey, sourceLocalSecrets, sourceImportedSecrets);
+    const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string);
+
+    const lock = await keyStore.acquireLock(
+      [getReplicationKeyLockPrefix(projectId, environmentSlug, secretPath)],
+      5000
+    );
+
+    try {
+      /*  eslint-disable no-await-in-loop */
+      for (const destinationSecretImport of destinationReplicatedSecretImports) {
+        try {
+          const hasJobCompleted = await keyStore.getItem(
+            keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
+            KeyStorePrefixes.SecretReplication
+          );
+          if (hasJobCompleted) {
+            logger.info(
+              { jobId: job.id, importId: destinationSecretImport.id },
+              "Skipping this job as this has been successfully replicated."
+            );
+            // eslint-disable-next-line
+            continue;
+          }
+
+          const [destinationFolder] = await folderDAL.findSecretPathByFolderIds(projectId, [
+            destinationSecretImport.folderId
+          ]);
+          if (!destinationFolder) throw new BadRequestError({ message: "Imported folder not found" });
+
+          let destinationReplicationFolder = await folderDAL.findOne({
+            parentId: destinationFolder.id,
+            name: getReplicationFolderName(destinationSecretImport.id),
+            isReserved: true
+          });
+          if (!destinationReplicationFolder) {
+            destinationReplicationFolder = await folderDAL.create({
+              parentId: destinationFolder.id,
+              name: getReplicationFolderName(destinationSecretImport.id),
+              envId: destinationFolder.envId,
+              isReserved: true
+            });
+          }
+          const destinationReplicationFolderId = destinationReplicationFolder.id;
+
+          const destinationLocalSecretsFromDB = await secretDAL.find({
+            folderId: destinationReplicationFolderId
+          });
+          const destinationLocalSecrets = destinationLocalSecretsFromDB.map((el) => {
+            const decryptedSecret = getDecryptedKeyValue(botKey, el);
+            return { ...el, secretKey: decryptedSecret.key, secretValue: decryptedSecret.value };
+          });
+
+          const destinationLocalSecretsGroupedByBlindIndex = groupBy(
+            destinationLocalSecrets.filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)),
+            (i) => i.secretBlindIndex as string
+          );
+
+          const locallyCreatedSecrets = sourceSecrets
+            .filter(
+              ({ secretBlindIndex }) => !destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]
+            )
+            .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create
+
+          const locallyUpdatedSecrets = sourceSecrets
+            .filter(
+              ({ secretBlindIndex, secretKey, secretValue }) =>
+                destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] &&
+                // if key or value changed
+                (destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretKey !== secretKey ||
+                  destinationLocalSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretValue !==
+                    secretValue)
+            )
+            .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create
+
+          const locallyDeletedSecrets = destinationLocalSecrets
+            .filter(({ secretBlindIndex }) => !sourceSecretsGroupByBlindIndex[secretBlindIndex as string]?.[0])
+            .map((el) => ({ ...el, operation: SecretOperations.Delete }));
+
+          const isEmtpy =
+            locallyCreatedSecrets.length + locallyUpdatedSecrets.length + locallyDeletedSecrets.length === 0;
+          // eslint-disable-next-line
+          if (isEmtpy) continue;
+
+          const policy = await secretApprovalPolicyService.getSecretApprovalPolicy(
+            projectId,
+            destinationFolder.environmentSlug,
+            destinationFolder.path
+          );
+          // this means it should be a approval request rather than direct replication
+          if (policy && actor === ActorType.USER) {
+            const membership = await projectMembershipDAL.findOne({ projectId, userId: actorId });
+            if (!membership) {
+              logger.error("Project membership not found in %s for user %s", projectId, actorId);
+              return;
+            }
+
+            const localSecretsLatestVersions = destinationLocalSecrets.map(({ id }) => id);
+            const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(
+              destinationReplicationFolderId,
+              localSecretsLatestVersions
+            );
+            await secretApprovalRequestDAL.transaction(async (tx) => {
+              const approvalRequestDoc = await secretApprovalRequestDAL.create(
+                {
+                  folderId: destinationReplicationFolderId,
+                  slug: alphaNumericNanoId(),
+                  policyId: policy.id,
+                  status: "open",
+                  hasMerged: false,
+                  committerId: membership.id,
+                  isReplicated: true
+                },
+                tx
+              );
+              const commits = locallyCreatedSecrets
+                .concat(locallyUpdatedSecrets)
+                .concat(locallyDeletedSecrets)
+                .map((doc) => {
+                  const { operation } = doc;
+                  const localSecret = destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0];
+
+                  return {
+                    op: operation,
+                    keyEncoding: doc.keyEncoding,
+                    algorithm: doc.algorithm,
+                    requestId: approvalRequestDoc.id,
+                    metadata: doc.metadata,
+                    secretKeyIV: doc.secretKeyIV,
+                    secretKeyTag: doc.secretKeyTag,
+                    secretKeyCiphertext: doc.secretKeyCiphertext,
+                    secretValueIV: doc.secretValueIV,
+                    secretValueTag: doc.secretValueTag,
+                    secretValueCiphertext: doc.secretValueCiphertext,
+                    secretBlindIndex: doc.secretBlindIndex,
+                    secretCommentIV: doc.secretCommentIV,
+                    secretCommentTag: doc.secretCommentTag,
+                    secretCommentCiphertext: doc.secretCommentCiphertext,
+                    skipMultilineEncoding: doc.skipMultilineEncoding,
+                    // except create operation other two needs the secret id and version id
+                    ...(operation !== SecretOperations.Create
+                      ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id }
+                      : {})
+                  };
+                });
+              const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx);
+
+              return { ...approvalRequestDoc, commits: approvalCommits };
+            });
+          } else {
+            await secretDAL.transaction(async (tx) => {
+              if (locallyCreatedSecrets.length) {
+                await fnSecretBulkInsert({
+                  folderId: destinationReplicationFolderId,
+                  secretVersionDAL,
+                  secretDAL,
+                  tx,
+                  secretTagDAL,
+                  secretVersionTagDAL,
+                  inputSecrets: locallyCreatedSecrets.map((doc) => {
+                    return {
+                      keyEncoding: doc.keyEncoding,
+                      algorithm: doc.algorithm,
+                      type: doc.type,
+                      metadata: doc.metadata,
+                      secretKeyIV: doc.secretKeyIV,
+                      secretKeyTag: doc.secretKeyTag,
+                      secretKeyCiphertext: doc.secretKeyCiphertext,
+                      secretValueIV: doc.secretValueIV,
+                      secretValueTag: doc.secretValueTag,
+                      secretValueCiphertext: doc.secretValueCiphertext,
+                      secretBlindIndex: doc.secretBlindIndex,
+                      secretCommentIV: doc.secretCommentIV,
+                      secretCommentTag: doc.secretCommentTag,
+                      secretCommentCiphertext: doc.secretCommentCiphertext,
+                      skipMultilineEncoding: doc.skipMultilineEncoding
+                    };
+                  })
+                });
+              }
+              if (locallyUpdatedSecrets.length) {
+                await fnSecretBulkUpdate({
+                  projectId,
+                  folderId: destinationReplicationFolderId,
+                  secretVersionDAL,
+                  secretDAL,
+                  tx,
+                  secretTagDAL,
+                  secretVersionTagDAL,
+                  inputSecrets: locallyUpdatedSecrets.map((doc) => {
+                    return {
+                      filter: {
+                        folderId: destinationReplicationFolderId,
+                        id: destinationLocalSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id
+                      },
+                      data: {
+                        keyEncoding: doc.keyEncoding,
+                        algorithm: doc.algorithm,
+                        type: doc.type,
+                        metadata: doc.metadata,
+                        secretKeyIV: doc.secretKeyIV,
+                        secretKeyTag: doc.secretKeyTag,
+                        secretKeyCiphertext: doc.secretKeyCiphertext,
+                        secretValueIV: doc.secretValueIV,
+                        secretValueTag: doc.secretValueTag,
+                        secretValueCiphertext: doc.secretValueCiphertext,
+                        secretBlindIndex: doc.secretBlindIndex,
+                        secretCommentIV: doc.secretCommentIV,
+                        secretCommentTag: doc.secretCommentTag,
+                        secretCommentCiphertext: doc.secretCommentCiphertext,
+                        skipMultilineEncoding: doc.skipMultilineEncoding
+                      }
+                    };
+                  })
+                });
+              }
+              if (locallyDeletedSecrets.length) {
+                await secretDAL.delete(
+                  {
+                    $in: {
+                      id: locallyDeletedSecrets.map(({ id }) => id)
+                    },
+                    folderId: destinationReplicationFolderId
+                  },
+                  tx
+                );
+              }
+            });
+
+            await secretQueueService.syncSecrets({
+              projectId,
+              secretPath: destinationFolder.path,
+              environmentSlug: destinationFolder.environmentSlug,
+              actorId,
+              actor,
+              _depth: depth + 1,
+              _deDupeReplicationQueue: deDupeReplicationQueue,
+              _deDupeQueue: deDupeQueue
+            });
+          }
+
+          // this is used to avoid multiple times generating secret approval by failed one
+          await keyStore.setItemWithExpiry(
+            keystoreReplicationSuccessKey(job.id as string, destinationSecretImport.id),
+            SECRET_IMPORT_SUCCESS_LOCK,
+            1,
+            KeyStorePrefixes.SecretReplication
+          );
+
+          await secretImportDAL.updateById(destinationSecretImport.id, {
+            lastReplicated: new Date(),
+            replicationStatus: null,
+            isReplicationSuccess: true
+          });
+        } catch (err) {
+          logger.error(
+            err,
+            `Failed to replicate secret with import id=[${destinationSecretImport.id}] env=[${destinationSecretImport.importEnv.slug}] path=[${destinationSecretImport.importPath}]`
+          );
+          await secretImportDAL.updateById(destinationSecretImport.id, {
+            lastReplicated: new Date(),
+            replicationStatus: (err as Error)?.message.slice(0, 500),
+            isReplicationSuccess: false
+          });
+        }
+      }
+      /*  eslint-enable no-await-in-loop */
+    } finally {
+      await lock.release();
+      logger.info(job.data, "Replication finished");
+    }
+  });
+
+  queueService.listen(QueueName.SecretReplication, "failed", (job, err) => {
+    logger.error(err, "Failed to replicate secret", job?.data);
+  });
+};

backend/src/ee/services/secret-replication/secret-replication-types.ts

@@ -0,0 +1,3 @@
+export type TSyncSecretReplicationDTO = {
+  id: string;
+};

backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts

@@ -220,7 +220,7 @@ export const secretSnapshotServiceFactory = ({
       const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id);
       // this will remove all secrets and folders on child
       // due to sql foreign key and link list connection removing the folders removes everything below too
-      const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId }, tx);
+      const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx);
       const deletedTopLevelFolders = groupBy(
         deletedFolders.filter(({ parentId }) => parentId === snapshot.folderId),
         (item) => item.id

backend/src/keystore/keystore.ts

@@ -1,20 +1,75 @@
 import { Redis } from "ioredis";
 
+import { Redlock, Settings } from "@app/lib/red-lock";
+
 export type TKeyStoreFactory = ReturnType<typeof keyStoreFactory>;
 
+// all the key prefixes used must be set here to avoid conflict
+export enum KeyStorePrefixes {
+  SecretReplication = "secret-replication-import-lock"
+}
+
+type TWaitTillReady = {
+  key: string;
+  waitingCb?: () => void;
+  keyCheckCb: (val: string | null) => boolean;
+  waitIteration?: number;
+  delay?: number;
+  jitter?: number;
+};
+
 export const keyStoreFactory = (redisUrl: string) => {
   const redis = new Redis(redisUrl);
+  const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 });
 
-  const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value);
+  const setItem = async (key: string, value: string | number | Buffer, prefix?: string) =>
+    redis.set(prefix ? `${prefix}:${key}` : key, value);
 
-  const getItem = async (key: string) => redis.get(key);
+  const getItem = async (key: string, prefix?: string) => redis.get(prefix ? `${prefix}:${key}` : key);
 
-  const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) =>
-    redis.setex(key, exp, value);
+  const setItemWithExpiry = async (
+    key: string,
+    exp: number | string,
+    value: string | number | Buffer,
+    prefix?: string
+  ) => redis.setex(prefix ? `${prefix}:${key}` : key, exp, value);
 
   const deleteItem = async (key: string) => redis.del(key);
 
   const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
 
-  return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy };
+  const waitTillReady = async ({
+    key,
+    waitingCb,
+    keyCheckCb,
+    waitIteration = 10,
+    delay = 1000,
+    jitter = 200
+  }: TWaitTillReady) => {
+    let attempts = 0;
+    let isReady = keyCheckCb(await getItem(key));
+    while (!isReady) {
+      if (attempts > waitIteration) return;
+      // eslint-disable-next-line
+      await new Promise((resolve) => {
+        waitingCb?.();
+        setTimeout(resolve, Math.max(0, delay + Math.floor((Math.random() * 2 - 1) * jitter)));
+      });
+      attempts += 1;
+      // eslint-disable-next-line
+      isReady = keyCheckCb(await getItem(key, "wait_till_ready"));
+    }
+  };
+
+  return {
+    setItem,
+    getItem,
+    setItemWithExpiry,
+    deleteItem,
+    incrementBy,
+    acquireLock(resources: string[], duration: number, settings?: Partial<Settings>) {
+      return redisLock.acquire(resources, duration, settings);
+    },
+    waitTillReady
+  };
 };

backend/src/lib/api-docs/constants.ts

@@ -225,7 +225,8 @@ export const PROJECT_IDENTITIES = {
     roles: {
       description: "A list of role slugs to assign to the identity project membership.",
       role: "The role slug to assign to the newly created identity project membership.",
-      isTemporary: "Whether the assigned role is temporary.",
+      isTemporary:
+        "Whether the assigned role is temporary. If isTemporary is set true, must provide temporaryMode, temporaryRange and temporaryAccessStartTime.",
       temporaryMode: "Type of temporary expiry.",
       temporaryRange: "Expiry time for temporary access. In relative mode it could be 1s,2m,3h",
       temporaryAccessStartTime: "Time to which the temporary access starts"
@@ -242,7 +243,8 @@ export const PROJECT_IDENTITIES = {
     roles: {
       description: "A list of role slugs to assign to the newly created identity project membership.",
       role: "The role slug to assign to the newly created identity project membership.",
-      isTemporary: "Whether the assigned role is temporary.",
+      isTemporary:
+        "Whether the assigned role is temporary. If isTemporary is set true, must provide temporaryMode, temporaryRange and temporaryAccessStartTime.",
       temporaryMode: "Type of temporary expiry.",
       temporaryRange: "Expiry time for temporary access. In relative mode it could be 1s,2m,3h",
       temporaryAccessStartTime: "Time to which the temporary access starts"
@@ -519,7 +521,8 @@ export const IDENTITY_ADDITIONAL_PRIVILEGE = {
     projectSlug: "The slug of the project of the identity in.",
     identityId: "The ID of the identity to create.",
     slug: "The slug of the privilege to create.",
-    permissions: `The permission object for the privilege.
+    permissions: `@deprecated - use privilegePermission
+The permission object for the privilege.
 - Read secrets
 \`\`\`
 { "permissions": [{"action": "read", "subject": "secrets"]}
@@ -533,6 +536,7 @@ export const IDENTITY_ADDITIONAL_PRIVILEGE = {
 - { "permissions": [{"action": "read", "subject": "secrets", "conditions": { "environment": "dev", "secretPath": { "$glob": "/" } }}] }
 \`\`\`
 `,
+    privilegePermission: "The permission object for the privilege.",
     isPackPermission: "Whether the server should pack(compact) the permission object.",
     isTemporary: "Whether the privilege is temporary.",
     temporaryMode: "Type of temporary access given. Types: relative",
@@ -544,7 +548,8 @@ export const IDENTITY_ADDITIONAL_PRIVILEGE = {
     identityId: "The ID of the identity to update.",
     slug: "The slug of the privilege to update.",
     newSlug: "The new slug of the privilege to update.",
-    permissions: `The permission object for the privilege.
+    permissions: `@deprecated - use privilegePermission
+The permission object for the privilege.
 - Read secrets
 \`\`\`
 { "permissions": [{"action": "read", "subject": "secrets"]}
@@ -558,6 +563,7 @@ export const IDENTITY_ADDITIONAL_PRIVILEGE = {
 - { "permissions": [{"action": "read", "subject": "secrets", "conditions": { "environment": "dev", "secretPath": { "$glob": "/" } }}] }
 \`\`\`
 `,
+    privilegePermission: "The permission object for the privilege.",
     isTemporary: "Whether the privilege is temporary.",
     temporaryMode: "Type of temporary access given. Types: relative",
     temporaryRange: "TTL for the temporay time. Eg: 1m, 1h, 1d",
@@ -655,6 +661,7 @@ export const INTEGRATION = {
     targetServiceId:
       "The service based grouping identifier ID of the external provider. Used in Terraform cloud, Checkly, Railway and NorthFlank",
     owner: "External integration providers service entity owner. Used in Github.",
+    url: "The self-hosted URL of the platform to integrate with",
     path: "Path to save the synced secrets. Used by Gitlab, AWS Parameter Store, Vault",
     region: "AWS region to sync secrets to.",
     scope: "Scope of the provider. Used by Github, Qovery",
@@ -715,3 +722,32 @@ export const AUDIT_LOG_STREAMS = {
     id: "The ID of the audit log stream to get details."
   }
 };
+
+export const PROJECT_ROLE = {
+  CREATE: {
+    projectSlug: "Slug of the project to create the role for.",
+    slug: "The slug of the role.",
+    name: "The name of the role.",
+    description: "The description for the role.",
+    permissions: "The permissions assigned to the role."
+  },
+  UPDATE: {
+    projectSlug: "Slug of the project to update the role for.",
+    roleId: "The ID of the role to update",
+    slug: "The slug of the role.",
+    name: "The name of the role.",
+    description: "The description for the role.",
+    permissions: "The permissions assigned to the role."
+  },
+  DELETE: {
+    projectSlug: "Slug of the project to delete this role for.",
+    roleId: "The ID of the role to update"
+  },
+  GET_ROLE_BY_SLUG: {
+    projectSlug: "The slug of the project.",
+    roleSlug: "The slug of the role to get details"
+  },
+  LIST: {
+    projectSlug: "The slug of the project to list the roles of."
+  }
+};

backend/src/lib/config/env.ts

@@ -75,6 +75,7 @@ const envSchema = z
         .optional()
         .default(process.env.URL_GITLAB_LOGIN ?? GITLAB_URL)
     ), // fallback since URL_GITLAB_LOGIN has been renamed
+    DEFAULT_SAML_ORG_SLUG: zpStr(z.string().optional()).default(process.env.NEXT_PUBLIC_SAML_ORG_SLUG),
     // integration client secrets
     // heroku
     CLIENT_ID_HEROKU: zpStr(z.string().optional()),
@@ -131,7 +132,8 @@ const envSchema = z
     isSecretScanningConfigured:
       Boolean(data.SECRET_SCANNING_GIT_APP_ID) &&
       Boolean(data.SECRET_SCANNING_PRIVATE_KEY) &&
-      Boolean(data.SECRET_SCANNING_WEBHOOK_SECRET)
+      Boolean(data.SECRET_SCANNING_WEBHOOK_SECRET),
+    samlDefaultOrgSlug: data.DEFAULT_SAML_ORG_SLUG
   }));
 
 let envCfg: Readonly<z.infer<typeof envSchema>>;

backend/src/lib/crypto/cipher/cipher.ts

@@ -0,0 +1,49 @@
+import crypto from "crypto";
+
+import { SymmetricEncryption, TSymmetricEncryptionFns } from "./types";
+
+const getIvLength = () => {
+  return 12;
+};
+
+const getTagLength = () => {
+  return 16;
+};
+
+export const symmetricCipherService = (type: SymmetricEncryption): TSymmetricEncryptionFns => {
+  const IV_LENGTH = getIvLength();
+  const TAG_LENGTH = getTagLength();
+
+  const encrypt = (text: Buffer, key: Buffer) => {
+    const iv = crypto.randomBytes(IV_LENGTH);
+    const cipher = crypto.createCipheriv(type, key, iv);
+
+    let encrypted = cipher.update(text);
+    encrypted = Buffer.concat([encrypted, cipher.final()]);
+
+    // Get the authentication tag
+    const tag = cipher.getAuthTag();
+
+    // Concatenate IV, encrypted text, and tag into a single buffer
+    const ciphertextBlob = Buffer.concat([iv, encrypted, tag]);
+    return ciphertextBlob;
+  };
+
+  const decrypt = (ciphertextBlob: Buffer, key: Buffer) => {
+    // Extract the IV, encrypted text, and tag from the buffer
+    const iv = ciphertextBlob.subarray(0, IV_LENGTH);
+    const tag = ciphertextBlob.subarray(-TAG_LENGTH);
+    const encrypted = ciphertextBlob.subarray(IV_LENGTH, -TAG_LENGTH);
+
+    const decipher = crypto.createDecipheriv(type, key, iv);
+    decipher.setAuthTag(tag);
+
+    const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
+    return decrypted;
+  };
+
+  return {
+    encrypt,
+    decrypt
+  };
+};

backend/src/lib/crypto/cipher/index.ts

@@ -0,0 +1,2 @@
+export { symmetricCipherService } from "./cipher";
+export { SymmetricEncryption } from "./types";

backend/src/lib/crypto/cipher/types.ts

@@ -0,0 +1,9 @@
+export enum SymmetricEncryption {
+  AES_GCM_256 = "aes-256-gcm",
+  AES_GCM_128 = "aes-128-gcm"
+}
+
+export type TSymmetricEncryptionFns = {
+  encrypt: (text: Buffer, key: Buffer) => Buffer;
+  decrypt: (blob: Buffer, key: Buffer) => Buffer;
+};

backend/src/lib/crypto/encryption.ts

@@ -11,6 +11,8 @@ import { getConfig } from "../config/env";
 export const decodeBase64 = (s: string) => naclUtils.decodeBase64(s);
 export const encodeBase64 = (u: Uint8Array) => naclUtils.encodeBase64(u);
 
+export const randomSecureBytes = (length = 32) => crypto.randomBytes(length);
+
 export type TDecryptSymmetricInput = {
   ciphertext: string;
   iv: string;

backend/src/lib/crypto/index.ts

@@ -9,7 +9,8 @@ export {
   encryptAsymmetric,
   encryptSymmetric,
   encryptSymmetric128BitHexKeyUTF8,
-  generateAsymmetricKeyPair
+  generateAsymmetricKeyPair,
+  randomSecureBytes
 } from "./encryption";
 export {
   decryptIntegrationAuths,

backend/src/lib/knex/index.ts

@@ -128,7 +128,7 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
       }
       if ($decr) {
         Object.entries($decr).forEach(([incrementField, incrementValue]) => {
-          void query.increment(incrementField, incrementValue);
+          void query.decrement(incrementField, incrementValue);
         });
       }
       const [docs] = await query;

backend/src/lib/red-lock/index.ts

@@ -0,0 +1,682 @@
+/* eslint-disable */
+// Source code credits: https://github.com/mike-marcacci/node-redlock
+// Taken to avoid external dependency
+import { randomBytes, createHash } from "crypto";
+import { EventEmitter } from "events";
+
+// AbortController became available as a global in node version 16. Once version
+// 14 reaches its end-of-life, this can be removed.
+
+import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis";
+
+type Client = IORedisClient | IORedisCluster;
+
+// Define script constants.
+const ACQUIRE_SCRIPT = `
+  -- Return 0 if an entry already exists.
+  for i, key in ipairs(KEYS) do
+    if redis.call("exists", key) == 1 then
+      return 0
+    end
+  end
+
+  -- Create an entry for each provided key.
+  for i, key in ipairs(KEYS) do
+    redis.call("set", key, ARGV[1], "PX", ARGV[2])
+  end
+
+  -- Return the number of entries added.
+  return #KEYS
+`;
+
+const EXTEND_SCRIPT = `
+  -- Return 0 if an entry exists with a *different* lock value.
+  for i, key in ipairs(KEYS) do
+    if redis.call("get", key) ~= ARGV[1] then
+      return 0
+    end
+  end
+
+  -- Update the entry for each provided key.
+  for i, key in ipairs(KEYS) do
+    redis.call("set", key, ARGV[1], "PX", ARGV[2])
+  end
+
+  -- Return the number of entries updated.
+  return #KEYS
+`;
+
+const RELEASE_SCRIPT = `
+  local count = 0
+  for i, key in ipairs(KEYS) do
+    -- Only remove entries for *this* lock value.
+    if redis.call("get", key) == ARGV[1] then
+      redis.pcall("del", key)
+      count = count + 1
+    end
+  end
+
+  -- Return the number of entries removed.
+  return count
+`;
+
+export type ClientExecutionResult =
+  | {
+      client: Client;
+      vote: "for";
+      value: number;
+    }
+  | {
+      client: Client;
+      vote: "against";
+      error: Error;
+    };
+
+/*
+ * This object contains a summary of results.
+ */
+export type ExecutionStats = {
+  readonly membershipSize: number;
+  readonly quorumSize: number;
+  readonly votesFor: Set<Client>;
+  readonly votesAgainst: Map<Client, Error>;
+};
+
+/*
+ * This object contains a summary of results. Because the result of an attempt
+ * can sometimes be determined before all requests are finished, each attempt
+ * contains a Promise that will resolve ExecutionStats once all requests are
+ * finished. A rejection of these promises should be considered undefined
+ * behavior and should cause a crash.
+ */
+export type ExecutionResult = {
+  attempts: ReadonlyArray<Promise<ExecutionStats>>;
+  start: number;
+};
+
+/**
+ *
+ */
+export interface Settings {
+  readonly driftFactor: number;
+  readonly retryCount: number;
+  readonly retryDelay: number;
+  readonly retryJitter: number;
+  readonly automaticExtensionThreshold: number;
+}
+
+// Define default settings.
+const defaultSettings: Readonly<Settings> = {
+  driftFactor: 0.01,
+  retryCount: 10,
+  retryDelay: 200,
+  retryJitter: 100,
+  automaticExtensionThreshold: 500
+};
+
+// Modifyng this object is forbidden.
+Object.freeze(defaultSettings);
+
+/*
+ * This error indicates a failure due to the existence of another lock for one
+ * or more of the requested resources.
+ */
+export class ResourceLockedError extends Error {
+  constructor(public readonly message: string) {
+    super();
+    this.name = "ResourceLockedError";
+  }
+}
+
+/*
+ * This error indicates a failure of an operation to pass with a quorum.
+ */
+export class ExecutionError extends Error {
+  constructor(
+    public readonly message: string,
+    public readonly attempts: ReadonlyArray<Promise<ExecutionStats>>
+  ) {
+    super();
+    this.name = "ExecutionError";
+  }
+}
+
+/*
+ * An object of this type is returned when a resource is successfully locked. It
+ * contains convenience methods `release` and `extend` which perform the
+ * associated Redlock method on itself.
+ */
+export class Lock {
+  constructor(
+    public readonly redlock: Redlock,
+    public readonly resources: string[],
+    public readonly value: string,
+    public readonly attempts: ReadonlyArray<Promise<ExecutionStats>>,
+    public expiration: number
+  ) {}
+
+  async release(): Promise<ExecutionResult> {
+    return this.redlock.release(this);
+  }
+
+  async extend(duration: number): Promise<Lock> {
+    return this.redlock.extend(this, duration);
+  }
+}
+
+export type RedlockAbortSignal = AbortSignal & { error?: Error };
+
+/**
+ * A redlock object is instantiated with an array of at least one redis client
+ * and an optional `options` object. Properties of the Redlock object should NOT
+ * be changed after it is first used, as doing so could have unintended
+ * consequences for live locks.
+ */
+export class Redlock extends EventEmitter {
+  public readonly clients: Set<Client>;
+  public readonly settings: Settings;
+  public readonly scripts: {
+    readonly acquireScript: { value: string; hash: string };
+    readonly extendScript: { value: string; hash: string };
+    readonly releaseScript: { value: string; hash: string };
+  };
+
+  public constructor(
+    clients: Iterable<Client>,
+    settings: Partial<Settings> = {},
+    scripts: {
+      readonly acquireScript?: string | ((script: string) => string);
+      readonly extendScript?: string | ((script: string) => string);
+      readonly releaseScript?: string | ((script: string) => string);
+    } = {}
+  ) {
+    super();
+
+    // Prevent crashes on error events.
+    this.on("error", () => {
+      // Because redlock is designed for high availability, it does not care if
+      // a minority of redis instances/clusters fail at an operation.
+      //
+      // However, it can be helpful to monitor and log such cases. Redlock emits
+      // an "error" event whenever it encounters an error, even if the error is
+      // ignored in its normal operation.
+      //
+      // This function serves to prevent node's default behavior of crashing
+      // when an "error" event is emitted in the absence of listeners.
+    });
+
+    // Create a new array of client, to ensure no accidental mutation.
+    this.clients = new Set(clients);
+    if (this.clients.size === 0) {
+      throw new Error("Redlock must be instantiated with at least one redis client.");
+    }
+
+    // Customize the settings for this instance.
+    this.settings = {
+      driftFactor: typeof settings.driftFactor === "number" ? settings.driftFactor : defaultSettings.driftFactor,
+      retryCount: typeof settings.retryCount === "number" ? settings.retryCount : defaultSettings.retryCount,
+      retryDelay: typeof settings.retryDelay === "number" ? settings.retryDelay : defaultSettings.retryDelay,
+      retryJitter: typeof settings.retryJitter === "number" ? settings.retryJitter : defaultSettings.retryJitter,
+      automaticExtensionThreshold:
+        typeof settings.automaticExtensionThreshold === "number"
+          ? settings.automaticExtensionThreshold
+          : defaultSettings.automaticExtensionThreshold
+    };
+
+    // Use custom scripts and script modifiers.
+    const acquireScript =
+      typeof scripts.acquireScript === "function" ? scripts.acquireScript(ACQUIRE_SCRIPT) : ACQUIRE_SCRIPT;
+    const extendScript =
+      typeof scripts.extendScript === "function" ? scripts.extendScript(EXTEND_SCRIPT) : EXTEND_SCRIPT;
+    const releaseScript =
+      typeof scripts.releaseScript === "function" ? scripts.releaseScript(RELEASE_SCRIPT) : RELEASE_SCRIPT;
+
+    this.scripts = {
+      acquireScript: {
+        value: acquireScript,
+        hash: this._hash(acquireScript)
+      },
+      extendScript: {
+        value: extendScript,
+        hash: this._hash(extendScript)
+      },
+      releaseScript: {
+        value: releaseScript,
+        hash: this._hash(releaseScript)
+      }
+    };
+  }
+
+  /**
+   * Generate a sha1 hash compatible with redis evalsha.
+   */
+  private _hash(value: string): string {
+    return createHash("sha1").update(value).digest("hex");
+  }
+
+  /**
+   * Generate a cryptographically random string.
+   */
+  private _random(): string {
+    return randomBytes(16).toString("hex");
+  }
+
+  /**
+   * This method runs `.quit()` on all client connections.
+   */
+  public async quit(): Promise<void> {
+    const results = [];
+    for (const client of this.clients) {
+      results.push(client.quit());
+    }
+
+    await Promise.all(results);
+  }
+
+  /**
+   * This method acquires a locks on the resources for the duration specified by
+   * the `duration`.
+   */
+  public async acquire(resources: string[], duration: number, settings?: Partial<Settings>): Promise<Lock> {
+    if (Math.floor(duration) !== duration) {
+      throw new Error("Duration must be an integer value in milliseconds.");
+    }
+
+    const value = this._random();
+
+    try {
+      const { attempts, start } = await this._execute(
+        this.scripts.acquireScript,
+        resources,
+        [value, duration],
+        settings
+      );
+
+      // Add 2 milliseconds to the drift to account for Redis expires precision,
+      // which is 1 ms, plus the configured allowable drift factor.
+      const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2;
+
+      return new Lock(this, resources, value, attempts, start + duration - drift);
+    } catch (error) {
+      // If there was an error acquiring the lock, release any partial lock
+      // state that may exist on a minority of clients.
+      await this._execute(this.scripts.releaseScript, resources, [value], {
+        retryCount: 0
+      }).catch(() => {
+        // Any error here will be ignored.
+      });
+
+      throw error;
+    }
+  }
+
+  /**
+   * This method unlocks the provided lock from all servers still persisting it.
+   * It will fail with an error if it is unable to release the lock on a quorum
+   * of nodes, but will make no attempt to restore the lock in the case of a
+   * failure to release. It is safe to re-attempt a release or to ignore the
+   * error, as the lock will automatically expire after its timeout.
+   */
+  public async release(lock: Lock, settings?: Partial<Settings>): Promise<ExecutionResult> {
+    // Immediately invalidate the lock.
+    lock.expiration = 0;
+
+    // Attempt to release the lock.
+    return this._execute(this.scripts.releaseScript, lock.resources, [lock.value], settings);
+  }
+
+  /**
+   * This method extends a valid lock by the provided `duration`.
+   */
+  public async extend(existing: Lock, duration: number, settings?: Partial<Settings>): Promise<Lock> {
+    if (Math.floor(duration) !== duration) {
+      throw new Error("Duration must be an integer value in milliseconds.");
+    }
+
+    // The lock has already expired.
+    if (existing.expiration < Date.now()) {
+      throw new ExecutionError("Cannot extend an already-expired lock.", []);
+    }
+
+    const { attempts, start } = await this._execute(
+      this.scripts.extendScript,
+      existing.resources,
+      [existing.value, duration],
+      settings
+    );
+
+    // Invalidate the existing lock.
+    existing.expiration = 0;
+
+    // Add 2 milliseconds to the drift to account for Redis expires precision,
+    // which is 1 ms, plus the configured allowable drift factor.
+    const drift = Math.round((settings?.driftFactor ?? this.settings.driftFactor) * duration) + 2;
+
+    const replacement = new Lock(this, existing.resources, existing.value, attempts, start + duration - drift);
+
+    return replacement;
+  }
+
+  /**
+   * Execute a script on all clients. The resulting promise is resolved or
+   * rejected as soon as this quorum is reached; the resolution or rejection
+   * will contains a `stats` property that is resolved once all votes are in.
+   */
+  private async _execute(
+    script: { value: string; hash: string },
+    keys: string[],
+    args: (string | number)[],
+    _settings?: Partial<Settings>
+  ): Promise<ExecutionResult> {
+    const settings = _settings
+      ? {
+          ...this.settings,
+          ..._settings
+        }
+      : this.settings;
+
+    // For the purpose of easy config serialization, we treat a retryCount of
+    // -1 a equivalent to Infinity.
+    const maxAttempts = settings.retryCount === -1 ? Infinity : settings.retryCount + 1;
+
+    const attempts: Promise<ExecutionStats>[] = [];
+
+    while (true) {
+      const { vote, stats, start } = await this._attemptOperation(script, keys, args);
+
+      attempts.push(stats);
+
+      // The operation achieved a quorum in favor.
+      if (vote === "for") {
+        return { attempts, start };
+      }
+
+      // Wait before reattempting.
+      if (attempts.length < maxAttempts) {
+        await new Promise((resolve) => {
+          setTimeout(
+            resolve,
+            Math.max(0, settings.retryDelay + Math.floor((Math.random() * 2 - 1) * settings.retryJitter)),
+            undefined
+          );
+        });
+      } else {
+        throw new ExecutionError("The operation was unable to achieve a quorum during its retry window.", attempts);
+      }
+    }
+  }
+
+  private async _attemptOperation(
+    script: { value: string; hash: string },
+    keys: string[],
+    args: (string | number)[]
+  ): Promise<
+    | { vote: "for"; stats: Promise<ExecutionStats>; start: number }
+    | { vote: "against"; stats: Promise<ExecutionStats>; start: number }
+  > {
+    const start = Date.now();
+
+    return await new Promise((resolve) => {
+      const clientResults = [];
+      for (const client of this.clients) {
+        clientResults.push(this._attemptOperationOnClient(client, script, keys, args));
+      }
+
+      const stats: ExecutionStats = {
+        membershipSize: clientResults.length,
+        quorumSize: Math.floor(clientResults.length / 2) + 1,
+        votesFor: new Set<Client>(),
+        votesAgainst: new Map<Client, Error>()
+      };
+
+      let done: () => void;
+      const statsPromise = new Promise<typeof stats>((resolve) => {
+        done = () => resolve(stats);
+      });
+
+      // This is the expected flow for all successful and unsuccessful requests.
+      const onResultResolve = (clientResult: ClientExecutionResult): void => {
+        switch (clientResult.vote) {
+          case "for":
+            stats.votesFor.add(clientResult.client);
+            break;
+          case "against":
+            stats.votesAgainst.set(clientResult.client, clientResult.error);
+            break;
+        }
+
+        // A quorum has determined a success.
+        if (stats.votesFor.size === stats.quorumSize) {
+          resolve({
+            vote: "for",
+            stats: statsPromise,
+            start
+          });
+        }
+
+        // A quorum has determined a failure.
+        if (stats.votesAgainst.size === stats.quorumSize) {
+          resolve({
+            vote: "against",
+            stats: statsPromise,
+            start
+          });
+        }
+
+        // All votes are in.
+        if (stats.votesFor.size + stats.votesAgainst.size === stats.membershipSize) {
+          done();
+        }
+      };
+
+      // This is unexpected and should crash to prevent undefined behavior.
+      const onResultReject = (error: Error): void => {
+        throw error;
+      };
+
+      for (const result of clientResults) {
+        result.then(onResultResolve, onResultReject);
+      }
+    });
+  }
+
+  private async _attemptOperationOnClient(
+    client: Client,
+    script: { value: string; hash: string },
+    keys: string[],
+    args: (string | number)[]
+  ): Promise<ClientExecutionResult> {
+    try {
+      let result: number;
+      try {
+        // Attempt to evaluate the script by its hash.
+        // @ts-expect-error
+        const shaResult = (await client.evalsha(script.hash, keys.length, [...keys, ...args])) as unknown;
+
+        if (typeof shaResult !== "number") {
+          throw new Error(`Unexpected result of type ${typeof shaResult} returned from redis.`);
+        }
+
+        result = shaResult;
+      } catch (error) {
+        // If the redis server does not already have the script cached,
+        // reattempt the request with the script's raw text.
+        if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT")) {
+          throw error;
+        }
+        // @ts-expect-error
+        const rawResult = (await client.eval(script.value, keys.length, [...keys, ...args])) as unknown;
+
+        if (typeof rawResult !== "number") {
+          throw new Error(`Unexpected result of type ${typeof rawResult} returned from redis.`);
+        }
+
+        result = rawResult;
+      }
+
+      // One or more of the resources was already locked.
+      if (result !== keys.length) {
+        throw new ResourceLockedError(
+          `The operation was applied to: ${result} of the ${keys.length} requested resources.`
+        );
+      }
+
+      return {
+        vote: "for",
+        client,
+        value: result
+      };
+    } catch (error) {
+      if (!(error instanceof Error)) {
+        throw new Error(`Unexpected type ${typeof error} thrown with value: ${error}`);
+      }
+
+      // Emit the error on the redlock instance for observability.
+      this.emit("error", error);
+
+      return {
+        vote: "against",
+        client,
+        error
+      };
+    }
+  }
+
+  /**
+   * Wrap and execute a routine in the context of an auto-extending lock,
+   * returning a promise of the routine's value. In the case that auto-extension
+   * fails, an AbortSignal will be updated to indicate that abortion of the
+   * routine is in order, and to pass along the encountered error.
+   *
+   * @example
+   * ```ts
+   * await redlock.using([senderId, recipientId], 5000, { retryCount: 5 }, async (signal) => {
+   *   const senderBalance = await getBalance(senderId);
+   *   const recipientBalance = await getBalance(recipientId);
+   *
+   *   if (senderBalance < amountToSend) {
+   *     throw new Error("Insufficient balance.");
+   *   }
+   *
+   *   // The abort signal will be true if:
+   *   // 1. the above took long enough that the lock needed to be extended
+   *   // 2. redlock was unable to extend the lock
+   *   //
+   *   // In such a case, exclusivity can no longer be guaranteed for further
+   *   // operations, and should be handled as an exceptional case.
+   *   if (signal.aborted) {
+   *     throw signal.error;
+   *   }
+   *
+   *   await setBalances([
+   *     {id: senderId, balance: senderBalance - amountToSend},
+   *     {id: recipientId, balance: recipientBalance + amountToSend},
+   *   ]);
+   * });
+   * ```
+   */
+
+  public async using<T>(
+    resources: string[],
+    duration: number,
+    settings: Partial<Settings>,
+    routine?: (signal: RedlockAbortSignal) => Promise<T>
+  ): Promise<T>;
+
+  public async using<T>(
+    resources: string[],
+    duration: number,
+    routine: (signal: RedlockAbortSignal) => Promise<T>
+  ): Promise<T>;
+
+  public async using<T>(
+    resources: string[],
+    duration: number,
+    settingsOrRoutine: undefined | Partial<Settings> | ((signal: RedlockAbortSignal) => Promise<T>),
+    optionalRoutine?: (signal: RedlockAbortSignal) => Promise<T>
+  ): Promise<T> {
+    if (Math.floor(duration) !== duration) {
+      throw new Error("Duration must be an integer value in milliseconds.");
+    }
+
+    const settings =
+      settingsOrRoutine && typeof settingsOrRoutine !== "function"
+        ? {
+            ...this.settings,
+            ...settingsOrRoutine
+          }
+        : this.settings;
+
+    const routine = optionalRoutine ?? settingsOrRoutine;
+    if (typeof routine !== "function") {
+      throw new Error("INVARIANT: routine is not a function.");
+    }
+
+    if (settings.automaticExtensionThreshold > duration - 100) {
+      throw new Error(
+        "A lock `duration` must be at least 100ms greater than the `automaticExtensionThreshold` setting."
+      );
+    }
+
+    // The AbortController/AbortSignal pattern allows the routine to be notified
+    // of a failure to extend the lock, and subsequent expiration. In the event
+    // of an abort, the error object will be made available at `signal.error`.
+    const controller = new AbortController();
+
+    const signal = controller.signal as RedlockAbortSignal;
+
+    function queue(): void {
+      timeout = setTimeout(
+        () => (extension = extend()),
+        lock.expiration - Date.now() - settings.automaticExtensionThreshold
+      );
+    }
+
+    async function extend(): Promise<void> {
+      timeout = undefined;
+
+      try {
+        lock = await lock.extend(duration);
+        queue();
+      } catch (error) {
+        if (!(error instanceof Error)) {
+          throw new Error(`Unexpected thrown ${typeof error}: ${error}.`);
+        }
+
+        if (lock.expiration > Date.now()) {
+          return (extension = extend());
+        }
+
+        signal.error = error instanceof Error ? error : new Error(`${error}`);
+        controller.abort();
+      }
+    }
+
+    let timeout: undefined | NodeJS.Timeout;
+    let extension: undefined | Promise<void>;
+    let lock = await this.acquire(resources, duration, settings);
+    queue();
+
+    try {
+      return await routine(signal);
+    } finally {
+      // Clean up the timer.
+      if (timeout) {
+        clearTimeout(timeout);
+        timeout = undefined;
+      }
+
+      // Wait for an in-flight extension to finish.
+      if (extension) {
+        await extension.catch(() => {
+          // An error here doesn't matter at all, because the routine has
+          // already completed, and a release will be attempted regardless. The
+          // only reason for waiting here is to prevent possible contention
+          // between the extension and release.
+        });
+      }
+
+      await lock.release();
+    }
+  }
+}

backend/src/lib/zod/index.ts

@@ -7,3 +7,7 @@ export const zpStr = <T extends ZodTypeAny>(schema: T, opt: { stripNull: boolean
     if (typeof val !== "string") return val;
     return val.trim() || undefined;
   }, schema);
+
+export const zodBuffer = z.custom<Buffer>((data) => Buffer.isBuffer(data) || data instanceof Uint8Array, {
+  message: "Expected binary data (Buffer Or Uint8Array)"
+});

backend/src/queue/queue-service.ts

@@ -7,6 +7,7 @@ import {
   TScanFullRepoEventPayload,
   TScanPushEventPayload
 } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
+import { TSyncSecretsDTO } from "@app/services/secret/secret-types";
 
 export enum QueueName {
   SecretRotation = "secret-rotation",
@@ -21,7 +22,9 @@ export enum QueueName {
   SecretFullRepoScan = "secret-full-repo-scan",
   SecretPushEventScan = "secret-push-event-scan",
   UpgradeProjectToGhost = "upgrade-project-to-ghost",
-  DynamicSecretRevocation = "dynamic-secret-revocation"
+  DynamicSecretRevocation = "dynamic-secret-revocation",
+  SecretReplication = "secret-replication",
+  SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
 }
 
 export enum QueueJobs {
@@ -37,7 +40,9 @@ export enum QueueJobs {
   SecretScan = "secret-scan",
   UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
   DynamicSecretRevocation = "dynamic-secret-revocation",
-  DynamicSecretPruning = "dynamic-secret-pruning"
+  DynamicSecretPruning = "dynamic-secret-pruning",
+  SecretReplication = "secret-replication",
+  SecretSync = "secret-sync" // parent queue to push integration sync, webhook, and secret replication
 }
 
 export type TQueueJobTypes = {
@@ -116,6 +121,14 @@ export type TQueueJobTypes = {
           dynamicSecretCfgId: string;
         };
       };
+  [QueueName.SecretReplication]: {
+    name: QueueJobs.SecretReplication;
+    payload: TSyncSecretsDTO;
+  };
+  [QueueName.SecretSync]: {
+    name: QueueJobs.SecretSync;
+    payload: TSyncSecretsDTO;
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
@@ -132,7 +145,7 @@ export const queueServiceFactory = (redisUrl: string) => {
 
   const start = <T extends QueueName>(
     name: T,
-    jobFn: (job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>) => Promise<void>,
+    jobFn: (job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>, token?: string) => Promise<void>,
     queueSettings: Omit<QueueOptions, "connection"> = {}
   ) => {
     if (queueContainer[name]) {
@@ -166,7 +179,7 @@ export const queueServiceFactory = (redisUrl: string) => {
     name: T,
     job: TQueueJobTypes[T]["name"],
     data: TQueueJobTypes[T]["payload"],
-    opts: JobsOptions & { jobId?: string }
+    opts?: JobsOptions & { jobId?: string }
   ) => {
     const q = queueContainer[name];
 

backend/src/server/config/rateLimiter.ts

@@ -28,7 +28,7 @@ export const readLimit: RateLimitOptions = {
 // POST, PATCH, PUT, DELETE endpoints
 export const writeLimit: RateLimitOptions = {
   timeWindow: 60 * 1000,
-  max: 50,
+  max: 200, // (too low, FA having issues so increasing it - maidul)
   keyGenerator: (req) => req.realIp
 };
 
@@ -66,3 +66,11 @@ export const creationLimit: RateLimitOptions = {
   max: 30,
   keyGenerator: (req) => req.realIp
 };
+
+// Public endpoints to avoid brute force attacks
+export const publicEndpointLimit: RateLimitOptions = {
+  // Shared Secrets
+  timeWindow: 60 * 1000,
+  max: 30,
+  keyGenerator: (req) => req.realIp
+};

backend/src/server/routes/index.ts

@@ -44,6 +44,7 @@ import { secretApprovalRequestDALFactory } from "@app/ee/services/secret-approva
 import { secretApprovalRequestReviewerDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-reviewer-dal";
 import { secretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
 import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
+import { secretReplicationServiceFactory } from "@app/ee/services/secret-replication/secret-replication-service";
 import { secretRotationDALFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
 import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue";
 import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
@@ -96,6 +97,9 @@ import { integrationDALFactory } from "@app/services/integration/integration-dal
 import { integrationServiceFactory } from "@app/services/integration/integration-service";
 import { integrationAuthDALFactory } from "@app/services/integration-auth/integration-auth-dal";
 import { integrationAuthServiceFactory } from "@app/services/integration-auth/integration-auth-service";
+import { kmsDALFactory } from "@app/services/kms/kms-dal";
+import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
+import { kmsServiceFactory } from "@app/services/kms/kms-service";
 import { incidentContactDALFactory } from "@app/services/org/incident-contacts-dal";
 import { orgBotDALFactory } from "@app/services/org/org-bot-dal";
 import { orgDALFactory } from "@app/services/org/org-dal";
@@ -130,6 +134,8 @@ import { secretFolderServiceFactory } from "@app/services/secret-folder/secret-f
 import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal";
 import { secretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
 import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service";
+import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal";
+import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service";
 import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service";
 import { serviceTokenDALFactory } from "@app/services/service-token/service-token-dal";
@@ -238,8 +244,8 @@ export const registerRoutes = async (
   const sapApproverDAL = secretApprovalPolicyApproverDALFactory(db);
   const secretApprovalPolicyDAL = secretApprovalPolicyDALFactory(db);
   const secretApprovalRequestDAL = secretApprovalRequestDALFactory(db);
-  const sarReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
-  const sarSecretDAL = secretApprovalRequestSecretDALFactory(db);
+  const secretApprovalRequestReviewerDAL = secretApprovalRequestReviewerDALFactory(db);
+  const secretApprovalRequestSecretDAL = secretApprovalRequestSecretDALFactory(db);
 
   const secretRotationDAL = secretRotationDALFactory(db);
   const snapshotDAL = snapshotDALFactory(db);
@@ -253,10 +259,14 @@ export const registerRoutes = async (
   const groupProjectMembershipRoleDAL = groupProjectMembershipRoleDALFactory(db);
   const userGroupMembershipDAL = userGroupMembershipDALFactory(db);
   const secretScanningDAL = secretScanningDALFactory(db);
+  const secretSharingDAL = secretSharingDALFactory(db);
   const licenseDAL = licenseDALFactory(db);
   const dynamicSecretDAL = dynamicSecretDALFactory(db);
   const dynamicSecretLeaseDAL = dynamicSecretLeaseDALFactory(db);
 
+  const kmsDAL = kmsDALFactory(db);
+  const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
+
   const permissionService = permissionServiceFactory({
     permissionDAL,
     orgRoleDAL,
@@ -265,6 +275,12 @@ export const registerRoutes = async (
     projectDAL
   });
   const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore });
+  const kmsService = kmsServiceFactory({
+    kmsRootConfigDAL,
+    keyStore,
+    kmsDAL
+  });
+
   const trustedIpService = trustedIpServiceFactory({
     licenseService,
     projectDAL,
@@ -285,7 +301,7 @@ export const registerRoutes = async (
     permissionService,
     auditLogStreamDAL
   });
-  const sapService = secretApprovalPolicyServiceFactory({
+  const secretApprovalPolicyService = secretApprovalPolicyServiceFactory({
     projectMembershipDAL,
     projectEnvDAL,
     secretApprovalPolicyApproverDAL: sapApproverDAL,
@@ -486,7 +502,7 @@ export const registerRoutes = async (
     projectBotDAL,
     projectMembershipDAL,
     secretApprovalRequestDAL,
-    secretApprovalSecretDAL: sarSecretDAL,
+    secretApprovalSecretDAL: secretApprovalRequestSecretDAL,
     projectUserMembershipRoleDAL
   });
 
@@ -523,7 +539,8 @@ export const registerRoutes = async (
     permissionService,
     projectRoleDAL,
     projectUserMembershipRoleDAL,
-    identityProjectMembershipRoleDAL
+    identityProjectMembershipRoleDAL,
+    projectDAL
   });
 
   const snapshotService = secretSnapshotServiceFactory({
@@ -583,6 +600,7 @@ export const registerRoutes = async (
     secretVersionTagDAL
   });
   const secretImportService = secretImportServiceFactory({
+    licenseService,
     projectEnvDAL,
     folderDAL,
     permissionService,
@@ -611,19 +629,24 @@ export const registerRoutes = async (
     projectEnvDAL,
     projectBotService
   });
-  const sarService = secretApprovalRequestServiceFactory({
+
+  const secretSharingService = secretSharingServiceFactory({
+    permissionService,
+    secretSharingDAL
+  });
+
+  const secretApprovalRequestService = secretApprovalRequestServiceFactory({
     permissionService,
     projectBotService,
     folderDAL,
     secretDAL,
     secretTagDAL,
-    secretApprovalRequestSecretDAL: sarSecretDAL,
-    secretApprovalRequestReviewerDAL: sarReviewerDAL,
+    secretApprovalRequestSecretDAL,
+    secretApprovalRequestReviewerDAL,
     projectDAL,
     secretVersionDAL,
     secretBlindIndexDAL,
     secretApprovalRequestDAL,
-    secretService,
     snapshotService,
     secretVersionTagDAL,
     secretQueueService
@@ -652,6 +675,23 @@ export const registerRoutes = async (
     accessApprovalPolicyApproverDAL
   });
 
+  const secretReplicationService = secretReplicationServiceFactory({
+    secretTagDAL,
+    secretVersionTagDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretImportDAL,
+    keyStore,
+    queueService,
+    folderDAL,
+    secretApprovalPolicyService,
+    secretBlindIndexDAL,
+    secretApprovalRequestDAL,
+    secretApprovalRequestSecretDAL,
+    secretQueueService,
+    projectMembershipDAL,
+    projectBotService
+  });
   const secretRotationQueue = secretRotationQueueFactory({
     telemetryService,
     secretRotationDAL,
@@ -784,7 +824,8 @@ export const registerRoutes = async (
   const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({
     auditLogDAL,
     queueService,
-    identityAccessTokenDAL
+    identityAccessTokenDAL,
+    secretSharingDAL
   });
 
   await superAdminService.initServerCfg();
@@ -794,6 +835,7 @@ export const registerRoutes = async (
 
   await telemetryQueue.startTelemetryCheck();
   await dailyResourceCleanUp.startCleanUp();
+  await kmsService.startService();
 
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {
@@ -815,6 +857,7 @@ export const registerRoutes = async (
     projectEnv: projectEnvService,
     projectRole: projectRoleService,
     secret: secretService,
+    secretReplication: secretReplicationService,
     secretTag: secretTagService,
     folder: folderService,
     secretImport: secretImportService,
@@ -831,10 +874,10 @@ export const registerRoutes = async (
     identityGcpAuth: identityGcpAuthService,
     identityAwsAuth: identityAwsAuthService,
     identityAzureAuth: identityAzureAuthService,
-    secretApprovalPolicy: sapService,
     accessApprovalPolicy: accessApprovalPolicyService,
     accessApprovalRequest: accessApprovalRequestService,
-    secretApprovalRequest: sarService,
+    secretApprovalPolicy: secretApprovalPolicyService,
+    secretApprovalRequest: secretApprovalRequestService,
     secretRotation: secretRotationService,
     dynamicSecret: dynamicSecretService,
     dynamicSecretLease: dynamicSecretLeaseService,
@@ -850,7 +893,8 @@ export const registerRoutes = async (
     secretBlindIndex: secretBlindIndexService,
     telemetry: telemetryService,
     projectUserAdditionalPrivilege: projectUserAdditionalPrivilegeService,
-    identityProjectAdditionalPrivilege: identityProjectAdditionalPrivilegeService
+    identityProjectAdditionalPrivilege: identityProjectAdditionalPrivilegeService,
+    secretSharing: secretSharingService
   });
 
   server.decorate<FastifyZodProvider["store"]>("store", {
@@ -875,7 +919,8 @@ export const registerRoutes = async (
           emailConfigured: z.boolean().optional(),
           inviteOnlySignup: z.boolean().optional(),
           redisConfigured: z.boolean().optional(),
-          secretScanningConfigured: z.boolean().optional()
+          secretScanningConfigured: z.boolean().optional(),
+          samlDefaultOrgSlug: z.string().optional()
         })
       }
     },
@@ -888,7 +933,8 @@ export const registerRoutes = async (
         emailConfigured: cfg.isSmtpConfigured,
         inviteOnlySignup: Boolean(serverCfg.allowSignUp),
         redisConfigured: cfg.isRedisConfigured,
-        secretScanningConfigured: cfg.isSecretScanningConfigured
+        secretScanningConfigured: cfg.isSecretScanningConfigured,
+        samlDefaultOrgSlug: cfg.samlDefaultOrgSlug
       };
     }
   });

backend/src/server/routes/sanitizedSchemas.ts

@@ -4,6 +4,7 @@ import {
   DynamicSecretsSchema,
   IdentityProjectAdditionalPrivilegeSchema,
   IntegrationAuthsSchema,
+  ProjectRolesSchema,
   SecretApprovalPoliciesSchema,
   UsersSchema
 } from "@app/db/schemas";
@@ -88,10 +89,38 @@ export const ProjectPermissionSchema = z.object({
     .optional()
 });
 
+export const ProjectSpecificPrivilegePermissionSchema = z.object({
+  actions: z
+    .nativeEnum(ProjectPermissionActions)
+    .describe("Describe what action an entity can take. Possible actions: create, edit, delete, and read")
+    .array()
+    .min(1),
+  subject: z
+    .enum([ProjectPermissionSub.Secrets])
+    .describe("The entity this permission pertains to. Possible options: secrets, environments"),
+  conditions: z
+    .object({
+      environment: z.string().describe("The environment slug this permission should allow."),
+      secretPath: z
+        .object({
+          $glob: z
+            .string()
+            .min(1)
+            .describe("The secret path this permission should allow. Can be a glob pattern such as /folder-name/*/** ")
+        })
+        .optional()
+    })
+    .describe("When specified, only matching conditions will be allowed to access given resource.")
+});
+
 export const SanitizedIdentityPrivilegeSchema = IdentityProjectAdditionalPrivilegeSchema.extend({
   permissions: UnpackedPermissionSchema.array()
 });
 
+export const SanitizedRoleSchema = ProjectRolesSchema.extend({
+  permissions: UnpackedPermissionSchema.array()
+});
+
 export const SanitizedDynamicSecretSchema = DynamicSecretsSchema.omit({
   inputIV: true,
   inputTag: true,

backend/src/server/routes/v1/index.ts

@@ -19,6 +19,7 @@ import { registerProjectMembershipRouter } from "./project-membership-router";
 import { registerProjectRouter } from "./project-router";
 import { registerSecretFolderRouter } from "./secret-folder-router";
 import { registerSecretImportRouter } from "./secret-import-router";
+import { registerSecretSharingRouter } from "./secret-sharing-router";
 import { registerSecretTagRouter } from "./secret-tag-router";
 import { registerSsoRouter } from "./sso-router";
 import { registerUserActionRouter } from "./user-action-router";
@@ -65,4 +66,5 @@ export const registerV1Routes = async (server: FastifyZodProvider) => {
   await server.register(registerIntegrationAuthRouter, { prefix: "/integration-auth" });
   await server.register(registerWebhookRouter, { prefix: "/webhooks" });
   await server.register(registerIdentityRouter, { prefix: "/identities" });
+  await server.register(registerSecretSharingRouter, { prefix: "/secret-sharing" });
 };

backend/src/server/routes/v1/integration-router.ts

@@ -42,6 +42,7 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
         targetService: z.string().trim().optional().describe(INTEGRATION.CREATE.targetService),
         targetServiceId: z.string().trim().optional().describe(INTEGRATION.CREATE.targetServiceId),
         owner: z.string().trim().optional().describe(INTEGRATION.CREATE.owner),
+        url: z.string().trim().optional().describe(INTEGRATION.CREATE.url),
         path: z.string().trim().optional().describe(INTEGRATION.CREATE.path),
         region: z.string().trim().optional().describe(INTEGRATION.CREATE.region),
         scope: z.string().trim().optional().describe(INTEGRATION.CREATE.scope),

backend/src/server/routes/v1/project-router.ts

@@ -8,7 +8,6 @@ import {
   UsersSchema
 } from "@app/db/schemas";
 import { PROJECTS } from "@app/lib/api-docs";
-import { BadRequestError } from "@app/lib/errors";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
@@ -193,19 +192,18 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
       }
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
-    handler: async () => {
-      // const workspace = await server.services.project.deleteProject({
-      //   filter: {
-      //     type: ProjectFilterType.ID,
-      //     projectId: req.params.workspaceId
-      //   },
-      //   actorId: req.permission.id,
-      //   actorAuthMethod: req.permission.authMethod,
-      //   actor: req.permission.type,
-      //   actorOrgId: req.permission.orgId
-      // });
-      // return { workspace };
-      throw new BadRequestError({ message: "Project delete has been paused temporarily, please try again later" });
+    handler: async (req) => {
+      const workspace = await server.services.project.deleteProject({
+        filter: {
+          type: ProjectFilterType.ID,
+          projectId: req.params.workspaceId
+        },
+        actorId: req.permission.id,
+        actorAuthMethod: req.permission.authMethod,
+        actor: req.permission.type,
+        actorOrgId: req.permission.orgId
+      });
+      return { workspace };
     }
   });
 

backend/src/server/routes/v1/secret-import-router.ts

@@ -29,7 +29,8 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
         import: z.object({
           environment: z.string().trim().describe(SECRET_IMPORTS.CREATE.import.environment),
           path: z.string().trim().transform(removeTrailingSlash).describe(SECRET_IMPORTS.CREATE.import.path)
-        })
+        }),
+        isReplication: z.boolean().default(false)
       }),
       response: {
         200: z.object({
@@ -210,6 +211,49 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
     }
   });
 
+  server.route({
+    method: "POST",
+    url: "/:secretImportId/replication-resync",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      description: "Resync secret replication of secret imports",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        secretImportId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.secretImportId)
+      }),
+      body: z.object({
+        workspaceId: z.string().trim().describe(SECRET_IMPORTS.UPDATE.workspaceId),
+        environment: z.string().trim().describe(SECRET_IMPORTS.UPDATE.environment),
+        path: z.string().trim().default("/").transform(removeTrailingSlash).describe(SECRET_IMPORTS.UPDATE.path)
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { message } = await server.services.secretImport.resyncSecretImportReplication({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        id: req.params.secretImportId,
+        ...req.body,
+        projectId: req.body.workspaceId
+      });
+
+      return { message };
+    }
+  });
+
   server.route({
     method: "GET",
     url: "/",
@@ -232,11 +276,9 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
         200: z.object({
           message: z.string(),
           secretImports: SecretImportsSchema.omit({ importEnv: true })
-            .merge(
-              z.object({
-                importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
-              })
-            )
+            .extend({
+              importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() })
+            })
             .array()
         })
       }

backend/src/server/routes/v1/secret-sharing-router.ts

@@ -0,0 +1,145 @@
+import { z } from "zod";
+
+import { SecretSharingSchema } from "@app/db/schemas";
+import { publicEndpointLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { AuthMode } from "@app/services/auth/auth-type";
+
+export const registerSecretSharingRouter = async (server: FastifyZodProvider) => {
+  server.route({
+    method: "GET",
+    url: "/",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      response: {
+        200: z.array(SecretSharingSchema)
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const sharedSecrets = await req.server.services.secretSharing.getSharedSecrets({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        orgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId
+      });
+
+      return sharedSecrets;
+    }
+  });
+
+  server.route({
+    method: "GET",
+    url: "/public/:id",
+    config: {
+      rateLimit: publicEndpointLimit
+    },
+    schema: {
+      params: z.object({
+        id: z.string().uuid()
+      }),
+      querystring: z.object({
+        hashedHex: z.string()
+      }),
+      response: {
+        200: SecretSharingSchema.pick({
+          encryptedValue: true,
+          iv: true,
+          tag: true,
+          expiresAt: true,
+          expiresAfterViews: true
+        })
+      }
+    },
+    handler: async (req) => {
+      const sharedSecret = await req.server.services.secretSharing.getActiveSharedSecretByIdAndHashedHex(
+        req.params.id,
+        req.query.hashedHex
+      );
+      if (!sharedSecret) return undefined;
+      return {
+        encryptedValue: sharedSecret.encryptedValue,
+        iv: sharedSecret.iv,
+        tag: sharedSecret.tag,
+        expiresAt: sharedSecret.expiresAt,
+        expiresAfterViews: sharedSecret.expiresAfterViews
+      };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        encryptedValue: z.string(),
+        iv: z.string(),
+        tag: z.string(),
+        hashedHex: z.string(),
+        expiresAt: z
+          .string()
+          .refine((date) => date === undefined || new Date(date) > new Date(), "Expires at should be a future date"),
+        expiresAfterViews: z.number()
+      }),
+      response: {
+        200: z.object({
+          id: z.string().uuid()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { encryptedValue, iv, tag, hashedHex, expiresAt, expiresAfterViews } = req.body;
+      const sharedSecret = await req.server.services.secretSharing.createSharedSecret({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        orgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        encryptedValue,
+        iv,
+        tag,
+        hashedHex,
+        expiresAt: new Date(expiresAt),
+        expiresAfterViews
+      });
+      return { id: sharedSecret.id };
+    }
+  });
+
+  server.route({
+    method: "DELETE",
+    url: "/:sharedSecretId",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      params: z.object({
+        sharedSecretId: z.string().uuid()
+      }),
+      response: {
+        200: SecretSharingSchema
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { sharedSecretId } = req.params;
+      const deletedSharedSecret = await req.server.services.secretSharing.deleteSharedSecretById({
+        actor: req.permission.type,
+        actorId: req.permission.id,
+        orgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        sharedSecretId
+      });
+
+      return { ...deletedSharedSecret };
+    }
+  });
+};

backend/src/server/routes/v3/secret-router.ts

@@ -9,7 +9,6 @@ import {
   ServiceTokenScopes
 } from "@app/db/schemas";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
-import { CommitType } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
 import { RAW_SECRETS, SECRETS } from "@app/lib/api-docs";
 import { BadRequestError } from "@app/lib/errors";
 import { removeTrailingSlash } from "@app/lib/fn";
@@ -19,6 +18,7 @@ import { getUserAgentType } from "@app/server/plugins/audit-log";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { ActorType, AuthMode } from "@app/services/auth/auth-type";
 import { ProjectFilterType } from "@app/services/project/project-types";
+import { SecretOperations } from "@app/services/secret/secret-types";
 import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
 
 import { secretRawSchema } from "../sanitizedSchemas";
@@ -902,7 +902,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Create]: [
+              [SecretOperations.Create]: [
                 {
                   secretName: req.params.secretName,
                   secretValueCiphertext,
@@ -1084,7 +1084,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Update]: [
+              [SecretOperations.Update]: [
                 {
                   secretName: req.params.secretName,
                   newSecretName,
@@ -1234,7 +1234,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Delete]: [
+              [SecretOperations.Delete]: [
                 {
                   secretName: req.params.secretName
                 }
@@ -1364,7 +1364,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Create]: inputSecrets
+              [SecretOperations.Create]: inputSecrets
             }
           });
 
@@ -1491,7 +1491,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Update]: inputSecrets.filter(({ type }) => type === "shared")
+              [SecretOperations.Update]: inputSecrets.filter(({ type }) => type === "shared")
             }
           });
 
@@ -1606,7 +1606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
             projectId,
             policy,
             data: {
-              [CommitType.Delete]: inputSecrets.filter(({ type }) => type === "shared")
+              [SecretOperations.Delete]: inputSecrets.filter(({ type }) => type === "shared")
             }
           });
           await server.services.auditLog.createAuditLog({

backend/src/services/integration-auth/integration-auth-service.ts

@@ -199,6 +199,7 @@ export const integrationAuthServiceFactory = ({
       projectId,
       namespace,
       integration,
+      url,
       algorithm: SecretEncryptionAlgo.AES_256_GCM,
       keyEncoding: SecretKeyEncoding.UTF8,
       ...(integration === Integrations.GCP_SECRET_MANAGER

backend/src/services/integration-auth/integration-list.ts

@@ -30,7 +30,8 @@ export enum Integrations {
   DIGITAL_OCEAN_APP_PLATFORM = "digital-ocean-app-platform",
   CLOUD_66 = "cloud-66",
   NORTHFLANK = "northflank",
-  HASURA_CLOUD = "hasura-cloud"
+  HASURA_CLOUD = "hasura-cloud",
+  RUNDECK = "rundeck"
 }
 
 export enum IntegrationType {
@@ -368,6 +369,15 @@ export const getIntegrationOptions = async () => {
       type: "pat",
       clientId: "",
       docsLink: ""
+    },
+    {
+      name: "Rundeck",
+      slug: "rundeck",
+      image: "Rundeck.svg",
+      isAvailable: true,
+      type: "pat",
+      clientId: "",
+      docsLink: ""
     }
   ];
 

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -27,6 +27,7 @@ import { z } from "zod";
 import { SecretType, TIntegrationAuths, TIntegrations, TSecrets } from "@app/db/schemas";
 import { request } from "@app/lib/config/request";
 import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
 import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/secret/secret-types";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
@@ -521,18 +522,42 @@ const syncSecretsAWSParameterStore = async ({
             .promise();
         }
         // case: secret exists in AWS parameter store
-      } else if (awsParameterStoreSecretsObj[key].Value !== secrets[key].value) {
-        // case: secret value doesn't match one in AWS parameter store
+      } else {
         // -> update secret
-        await ssm
-          .putParameter({
-            Name: `${integration.path}${key}`,
-            Type: "SecureString",
-            Value: secrets[key].value,
-            Overwrite: true
-            // Tags: metadata.secretAWSTag ? [{ Key: metadata.secretAWSTag.key, Value: metadata.secretAWSTag.value }] : []
-          })
-          .promise();
+        if (awsParameterStoreSecretsObj[key].Value !== secrets[key].value) {
+          await ssm
+            .putParameter({
+              Name: `${integration.path}${key}`,
+              Type: "SecureString",
+              Value: secrets[key].value,
+              Overwrite: true
+            })
+            .promise();
+        }
+
+        if (awsParameterStoreSecretsObj[key].Name) {
+          try {
+            await ssm
+              .addTagsToResource({
+                ResourceType: "Parameter",
+                ResourceId: awsParameterStoreSecretsObj[key].Name as string,
+                Tags: metadata.secretAWSTag
+                  ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({
+                      Key: tag.key,
+                      Value: tag.value
+                    }))
+                  : []
+              })
+              .promise();
+          } catch (err) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            if ((err as any).code === "AccessDeniedException") {
+              logger.error(
+                `AWS Parameter Store Error [integration=${integration.id}]: double check AWS account permissions (refer to the Infisical docs)`
+              );
+            }
+          }
+        }
       }
 
       await new Promise((resolve) => {
@@ -2725,6 +2750,20 @@ const syncSecretsCloudflarePages = async ({
       }
     }
   );
+
+  const metadata = z.record(z.any()).parse(integration.metadata);
+  if (metadata.shouldAutoRedeploy) {
+    await request.post(
+      `${IntegrationUrls.CLOUDFLARE_PAGES_API_URL}/client/v4/accounts/${accessId}/pages/projects/${integration.app}/deployments`,
+      {},
+      {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          Accept: "application/json"
+        }
+      }
+    );
+  }
 };
 
 /**
@@ -3330,6 +3369,82 @@ const syncSecretsHasuraCloud = async ({
   }
 };
 
+/** Sync/push [secrets] to Rundeck
+ * @param {Object} obj
+ * @param {TIntegrations} obj.integration - integration details
+ * @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values)
+ * @param {String} obj.accessToken - access token for Rundeck integration
+ */
+const syncSecretsRundeck = async ({
+  integration,
+  secrets,
+  accessToken
+}: {
+  integration: TIntegrations;
+  secrets: Record<string, { value: string; comment?: string }>;
+  accessToken: string;
+}) => {
+  interface RundeckSecretResource {
+    name: string;
+  }
+  interface RundeckSecretsGetRes {
+    resources: RundeckSecretResource[];
+  }
+
+  let existingRundeckSecrets: string[] = [];
+
+  try {
+    const listResult = await request.get<RundeckSecretsGetRes>(
+      `${integration.url}/api/44/storage/${integration.path}`,
+      {
+        headers: {
+          "X-Rundeck-Auth-Token": accessToken
+        }
+      }
+    );
+
+    existingRundeckSecrets = listResult.data.resources.map((res) => res.name);
+  } catch (err) {
+    logger.info("No existing rundeck secrets");
+  }
+
+  try {
+    for await (const [key, value] of Object.entries(secrets)) {
+      if (existingRundeckSecrets.includes(key)) {
+        await request.put(`${integration.url}/api/44/storage/${integration.path}/${key}`, value.value, {
+          headers: {
+            "X-Rundeck-Auth-Token": accessToken,
+            "Content-Type": "application/x-rundeck-data-password"
+          }
+        });
+      } else {
+        await request.post(`${integration.url}/api/44/storage/${integration.path}/${key}`, value.value, {
+          headers: {
+            "X-Rundeck-Auth-Token": accessToken,
+            "Content-Type": "application/x-rundeck-data-password"
+          }
+        });
+      }
+    }
+
+    for await (const existingSecret of existingRundeckSecrets) {
+      if (!(existingSecret in secrets)) {
+        await request.delete(`${integration.url}/api/44/storage/${integration.path}/${existingSecret}`, {
+          headers: {
+            "X-Rundeck-Auth-Token": accessToken
+          }
+        });
+      }
+    }
+  } catch (err: unknown) {
+    throw new Error(
+      `Ensure that the provided Rundeck URL is accessible by Infisical and that the linked API token has sufficient permissions.\n\n${
+        (err as Error).message
+      }`
+    );
+  }
+};
+
 /**
  * Sync/push [secrets] to [app] in integration named [integration]
  *
@@ -3596,6 +3711,13 @@ export const syncIntegrationSecrets = async ({
         accessToken
       });
       break;
+    case Integrations.RUNDECK:
+      await syncSecretsRundeck({
+        integration,
+        secrets,
+        accessToken
+      });
+      break;
     default:
       throw new BadRequestError({ message: "Invalid integration" });
   }

backend/src/services/integration/integration-service.ts

@@ -43,6 +43,7 @@ export const integrationServiceFactory = ({
     scope,
     actorId,
     region,
+    url,
     isActive,
     metadata,
     secretPath,
@@ -87,6 +88,7 @@ export const integrationServiceFactory = ({
       region,
       scope,
       owner,
+      url,
       appId,
       path,
       app,

backend/src/services/integration/integration-types.ts

@@ -12,6 +12,7 @@ export type TCreateIntegrationDTO = {
   targetService?: string;
   targetServiceId?: string;
   owner?: string;
+  url?: string;
   path?: string;
   region?: string;
   scope?: string;

backend/src/services/kms/kms-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TKmsDALFactory = ReturnType<typeof kmsDALFactory>;
+
+export const kmsDALFactory = (db: TDbClient) => {
+  const kmsOrm = ormify(db, TableName.KmsKey);
+  return kmsOrm;
+};

backend/src/services/kms/kms-root-config-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TKmsRootConfigDALFactory = ReturnType<typeof kmsRootConfigDALFactory>;
+
+export const kmsRootConfigDALFactory = (db: TDbClient) => {
+  const kmsOrm = ormify(db, TableName.KmsServerRootConfig);
+  return kmsOrm;
+};

backend/src/services/kms/kms-service.ts

@@ -0,0 +1,126 @@
+import { TKeyStoreFactory } from "@app/keystore/keystore";
+import { getConfig } from "@app/lib/config/env";
+import { randomSecureBytes } from "@app/lib/crypto";
+import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
+import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+
+import { TKmsDALFactory } from "./kms-dal";
+import { TKmsRootConfigDALFactory } from "./kms-root-config-dal";
+import { TDecryptWithKmsDTO, TEncryptWithKmsDTO, TGenerateKMSDTO } from "./kms-types";
+
+type TKmsServiceFactoryDep = {
+  kmsDAL: TKmsDALFactory;
+  kmsRootConfigDAL: Pick<TKmsRootConfigDALFactory, "findById" | "create">;
+  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "waitTillReady" | "setItemWithExpiry">;
+};
+
+export type TKmsServiceFactory = ReturnType<typeof kmsServiceFactory>;
+
+const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+
+const KMS_ROOT_CREATION_WAIT_KEY = "wait_till_ready_kms_root_key";
+const KMS_ROOT_CREATION_WAIT_TIME = 10;
+
+// akhilmhdh: Don't edit this value. This is measured for blob concatination in kms
+const KMS_VERSION = "v01";
+const KMS_VERSION_BLOB_LENGTH = 3;
+export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsServiceFactoryDep) => {
+  let ROOT_ENCRYPTION_KEY = Buffer.alloc(0);
+
+  // this is used symmetric encryption
+  const generateKmsKey = async ({ scopeId, scopeType, isReserved = true }: TGenerateKMSDTO) => {
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    const kmsKeyMaterial = randomSecureBytes(32);
+    const encryptedKeyMaterial = cipher.encrypt(kmsKeyMaterial, ROOT_ENCRYPTION_KEY);
+
+    const { encryptedKey, ...doc } = await kmsDAL.create({
+      version: 1,
+      encryptedKey: encryptedKeyMaterial,
+      encryptionAlgorithm: SymmetricEncryption.AES_GCM_256,
+      isReserved,
+      orgId: scopeType === "org" ? scopeId : undefined,
+      projectId: scopeType === "project" ? scopeId : undefined
+    });
+    return doc;
+  };
+
+  const encrypt = async ({ kmsId, plainText }: TEncryptWithKmsDTO) => {
+    const kmsDoc = await kmsDAL.findById(kmsId);
+    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+    // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+
+    const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+    const encryptedPlainTextBlob = cipher.encrypt(plainText, kmsKey);
+
+    // Buffer#1 encrypted text + Buffer#2 version number
+    const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
+    const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
+    return { cipherTextBlob };
+  };
+
+  const decrypt = async ({ cipherTextBlob: versionedCipherTextBlob, kmsId }: TDecryptWithKmsDTO) => {
+    const kmsDoc = await kmsDAL.findById(kmsId);
+    if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
+    // akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
+
+    const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
+    const decryptedBlob = cipher.decrypt(cipherTextBlob, kmsKey);
+    return decryptedBlob;
+  };
+
+  const startService = async () => {
+    const appCfg = getConfig();
+    // This will switch to a seal process and HMS flow in future
+    const encryptionKey = appCfg.ENCRYPTION_KEY || appCfg.ROOT_ENCRYPTION_KEY;
+    // if root key its base64 encoded
+    const isBase64 = !appCfg.ENCRYPTION_KEY;
+    if (!encryptionKey) throw new Error("Root encryption key not found for KMS service.");
+    const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8");
+
+    const lock = await keyStore.acquireLock([`KMS_ROOT_CFG_LOCK`], 3000, { retryCount: 3 }).catch(() => null);
+    if (!lock) {
+      await keyStore.waitTillReady({
+        key: KMS_ROOT_CREATION_WAIT_KEY,
+        keyCheckCb: (val) => val === "true",
+        waitingCb: () => logger.info("KMS. Waiting for leader to finish creation of KMS Root Key")
+      });
+    }
+
+    // check if KMS root key was already generated and saved in DB
+    const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
+    const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
+    if (kmsRootConfig) {
+      if (lock) await lock.release();
+      logger.info("KMS: Encrypted ROOT Key found from DB. Decrypting.");
+      const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer);
+      // set the flag so that other instancen nodes can start
+      await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
+      logger.info("KMS: Loading ROOT Key into Memory.");
+      ROOT_ENCRYPTION_KEY = decryptedRootKey;
+      return;
+    }
+
+    logger.info("KMS: Generating ROOT Key");
+    const newRootKey = randomSecureBytes(32);
+    const encryptedRootKey = cipher.encrypt(newRootKey, encryptionKeyBuffer);
+    // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
+    await kmsRootConfigDAL.create({ encryptedRootKey, id: KMS_ROOT_CONFIG_UUID });
+
+    // set the flag so that other instancen nodes can start
+    await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
+    logger.info("KMS: Saved and loaded ROOT Key into memory");
+    if (lock) await lock.release();
+    ROOT_ENCRYPTION_KEY = newRootKey;
+  };
+
+  return {
+    startService,
+    generateKmsKey,
+    encrypt,
+    decrypt
+  };
+};

backend/src/services/kms/kms-types.ts

@@ -0,0 +1,15 @@
+export type TGenerateKMSDTO = {
+  scopeType: "project" | "org";
+  scopeId: string;
+  isReserved?: boolean;
+};
+
+export type TEncryptWithKmsDTO = {
+  kmsId: string;
+  plainText: Buffer;
+};
+
+export type TDecryptWithKmsDTO = {
+  kmsId: string;
+  cipherTextBlob: Buffer;
+};

backend/src/services/project-role/project-role-service.ts

@@ -1,25 +1,30 @@
-import { ForbiddenError } from "@casl/ability";
-import { packRules } from "@casl/ability/extra";
+import { ForbiddenError, MongoAbility, RawRuleOf } from "@casl/ability";
+import { PackRule, packRules, unpackRules } from "@casl/ability/extra";
 
-import { ProjectMembershipRole, TOrgRolesUpdate, TProjectRolesInsert } from "@app/db/schemas";
+import { ProjectMembershipRole } from "@app/db/schemas";
+import { UnpackedPermissionSchema } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import {
   projectAdminPermissions,
   projectMemberPermissions,
   projectNoAccessPermissions,
   ProjectPermissionActions,
+  ProjectPermissionSet,
   ProjectPermissionSub,
   projectViewerPermission
 } from "@app/ee/services/permission/project-permission";
 import { BadRequestError } from "@app/lib/errors";
 
-import { ActorAuthMethod, ActorType } from "../auth/auth-type";
+import { ActorAuthMethod } from "../auth/auth-type";
 import { TIdentityProjectMembershipRoleDALFactory } from "../identity-project/identity-project-membership-role-dal";
+import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
 import { TProjectRoleDALFactory } from "./project-role-dal";
+import { TCreateRoleDTO, TDeleteRoleDTO, TGetRoleBySlugDTO, TListRolesDTO, TUpdateRoleDTO } from "./project-role-types";
 
 type TProjectRoleServiceFactoryDep = {
   projectRoleDAL: TProjectRoleDALFactory;
+  projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getUserProjectPermission">;
   identityProjectMembershipRoleDAL: TIdentityProjectMembershipRoleDALFactory;
   projectUserMembershipRoleDAL: TProjectUserMembershipRoleDALFactory;
@@ -27,20 +32,68 @@ type TProjectRoleServiceFactoryDep = {
 
 export type TProjectRoleServiceFactory = ReturnType<typeof projectRoleServiceFactory>;
 
+const unpackPermissions = (permissions: unknown) =>
+  UnpackedPermissionSchema.array().parse(
+    unpackRules((permissions || []) as PackRule<RawRuleOf<MongoAbility<ProjectPermissionSet>>>[])
+  );
+
+const getPredefinedRoles = (projectId: string, roleFilter?: ProjectMembershipRole) => {
+  return [
+    {
+      id: "b11b49a9-09a9-4443-916a-4246f9ff2c69", // dummy userid
+      projectId,
+      name: "Admin",
+      slug: ProjectMembershipRole.Admin,
+      permissions: projectAdminPermissions,
+      description: "Full administrative access over a project",
+      createdAt: new Date(),
+      updatedAt: new Date()
+    },
+    {
+      id: "b11b49a9-09a9-4443-916a-4246f9ff2c70", // dummy user for zod validation in response
+      projectId,
+      name: "Developer",
+      slug: ProjectMembershipRole.Member,
+      permissions: projectMemberPermissions,
+      description: "Limited read/write role in a project",
+      createdAt: new Date(),
+      updatedAt: new Date()
+    },
+    {
+      id: "b11b49a9-09a9-4443-916a-4246f9ff2c71", // dummy user for zod validation in response
+      projectId,
+      name: "Viewer",
+      slug: ProjectMembershipRole.Viewer,
+      permissions: projectViewerPermission,
+      description: "Only read role in a project",
+      createdAt: new Date(),
+      updatedAt: new Date()
+    },
+    {
+      id: "b11b49a9-09a9-4443-916a-4246f9ff2c72", // dummy user for zod validation in response
+      projectId,
+      name: "No Access",
+      slug: ProjectMembershipRole.NoAccess,
+      permissions: projectNoAccessPermissions,
+      description: "No access to any resources in the project",
+      createdAt: new Date(),
+      updatedAt: new Date()
+    }
+  ].filter(({ slug }) => !roleFilter || roleFilter.includes(slug));
+};
+
 export const projectRoleServiceFactory = ({
   projectRoleDAL,
   permissionService,
   identityProjectMembershipRoleDAL,
-  projectUserMembershipRoleDAL
+  projectUserMembershipRoleDAL,
+  projectDAL
 }: TProjectRoleServiceFactoryDep) => {
-  const createRole = async (
-    actor: ActorType,
-    actorId: string,
-    projectId: string,
-    data: Omit<TProjectRolesInsert, "projectId">,
-    actorAuthMethod: ActorAuthMethod,
-    actorOrgId: string | undefined
-  ) => {
+  const createRole = async ({ projectSlug, data, actor, actorId, actorAuthMethod, actorOrgId }: TCreateRoleDTO) => {
+    const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
+    if (!project) throw new BadRequestError({ message: "Project not found" });
+    const projectId = project.id;
+
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
@@ -53,21 +106,54 @@ export const projectRoleServiceFactory = ({
     if (existingRole) throw new BadRequestError({ name: "Create Role", message: "Duplicate role" });
     const role = await projectRoleDAL.create({
       ...data,
-      projectId,
-      permissions: JSON.stringify(data.permissions)
+      projectId
     });
-    return role;
+    return { ...role, permissions: unpackPermissions(role.permissions) };
   };
 
-  const updateRole = async (
-    actor: ActorType,
-    actorId: string,
-    projectId: string,
-    roleId: string,
-    data: Omit<TOrgRolesUpdate, "orgId">,
-    actorAuthMethod: ActorAuthMethod,
-    actorOrgId: string | undefined
-  ) => {
+  const getRoleBySlug = async ({
+    actor,
+    actorId,
+    projectSlug,
+    actorAuthMethod,
+    actorOrgId,
+    roleSlug
+  }: TGetRoleBySlugDTO) => {
+    const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
+    if (!project) throw new BadRequestError({ message: "Project not found" });
+    const projectId = project.id;
+
+    const { permission } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Role);
+    if (roleSlug !== "custom" && Object.values(ProjectMembershipRole).includes(roleSlug as ProjectMembershipRole)) {
+      const predefinedRole = getPredefinedRoles(projectId, roleSlug as ProjectMembershipRole)[0];
+      return { ...predefinedRole, permissions: UnpackedPermissionSchema.array().parse(predefinedRole.permissions) };
+    }
+
+    const customRole = await projectRoleDAL.findOne({ slug: roleSlug, projectId });
+    if (!customRole) throw new BadRequestError({ message: "Role not found" });
+    return { ...customRole, permissions: unpackPermissions(customRole.permissions) };
+  };
+
+  const updateRole = async ({
+    roleId,
+    projectSlug,
+    actorOrgId,
+    actorAuthMethod,
+    actorId,
+    actor,
+    data
+  }: TUpdateRoleDTO) => {
+    const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
+    if (!project) throw new BadRequestError({ message: "Project not found" });
+    const projectId = project.id;
+
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
@@ -81,22 +167,16 @@ export const projectRoleServiceFactory = ({
       if (existingRole && existingRole.id !== roleId)
         throw new BadRequestError({ name: "Update Role", message: "Duplicate role" });
     }
-    const [updatedRole] = await projectRoleDAL.update(
-      { id: roleId, projectId },
-      { ...data, permissions: data.permissions ? JSON.stringify(data.permissions) : undefined }
-    );
+    const [updatedRole] = await projectRoleDAL.update({ id: roleId, projectId }, data);
     if (!updatedRole) throw new BadRequestError({ message: "Role not found", name: "Update role" });
-    return updatedRole;
+    return { ...updatedRole, permissions: unpackPermissions(updatedRole.permissions) };
   };
 
-  const deleteRole = async (
-    actor: ActorType,
-    actorId: string,
-    projectId: string,
-    roleId: string,
-    actorAuthMethod: ActorAuthMethod,
-    actorOrgId: string | undefined
-  ) => {
+  const deleteRole = async ({ actor, actorId, actorAuthMethod, actorOrgId, projectSlug, roleId }: TDeleteRoleDTO) => {
+    const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
+    if (!project) throw new BadRequestError({ message: "Project not found" });
+    const projectId = project.id;
+
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
@@ -125,16 +205,14 @@ export const projectRoleServiceFactory = ({
     const [deletedRole] = await projectRoleDAL.delete({ id: roleId, projectId });
     if (!deletedRole) throw new BadRequestError({ message: "Role not found", name: "Delete role" });
 
-    return deletedRole;
+    return { ...deletedRole, permissions: unpackPermissions(deletedRole.permissions) };
   };
 
-  const listRoles = async (
-    actor: ActorType,
-    actorId: string,
-    projectId: string,
-    actorAuthMethod: ActorAuthMethod,
-    actorOrgId: string | undefined
-  ) => {
+  const listRoles = async ({ projectSlug, actorOrgId, actorAuthMethod, actorId, actor }: TListRolesDTO) => {
+    const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
+    if (!project) throw new BadRequestError({ message: "Project not found" });
+    const projectId = project.id;
+
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
@@ -144,52 +222,7 @@ export const projectRoleServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Role);
     const customRoles = await projectRoleDAL.find({ projectId });
-    const roles = [
-      {
-        id: "b11b49a9-09a9-4443-916a-4246f9ff2c69", // dummy userid
-        projectId,
-        name: "Admin",
-        slug: ProjectMembershipRole.Admin,
-        description: "Complete administration access over the project",
-        permissions: packRules(projectAdminPermissions),
-        createdAt: new Date(),
-        updatedAt: new Date()
-      },
-      {
-        id: "b11b49a9-09a9-4443-916a-4246f9ff2c70", // dummy user for zod validation in response
-        projectId,
-        name: "Developer",
-        slug: ProjectMembershipRole.Member,
-        description: "Non-administrative role in an project",
-        permissions: packRules(projectMemberPermissions),
-        createdAt: new Date(),
-        updatedAt: new Date()
-      },
-      {
-        id: "b11b49a9-09a9-4443-916a-4246f9ff2c71", // dummy user for zod validation in response
-        projectId,
-        name: "Viewer",
-        slug: ProjectMembershipRole.Viewer,
-        description: "Non-administrative role in an project",
-        permissions: packRules(projectViewerPermission),
-        createdAt: new Date(),
-        updatedAt: new Date()
-      },
-      {
-        id: "b11b49a9-09a9-4443-916a-4246f9ff2c72", // dummy user for zod validation in response
-        projectId,
-        name: "No Access",
-        slug: "no-access",
-        description: "No access to any resources in the project",
-        permissions: packRules(projectNoAccessPermissions),
-        createdAt: new Date(),
-        updatedAt: new Date()
-      },
-      ...(customRoles || []).map(({ permissions, ...data }) => ({
-        ...data,
-        permissions
-      }))
-    ];
+    const roles = [...getPredefinedRoles(projectId), ...(customRoles || [])];
 
     return roles;
   };
@@ -209,5 +242,5 @@ export const projectRoleServiceFactory = ({
     return { permissions: packRules(permission.rules), membership };
   };
 
-  return { createRole, updateRole, deleteRole, listRoles, getUserPermission };
+  return { createRole, updateRole, deleteRole, listRoles, getUserPermission, getRoleBySlug };
 };

backend/src/services/project-role/project-role-types.ts

@@ -0,0 +1,27 @@
+import { TOrgRolesUpdate, TProjectRolesInsert } from "@app/db/schemas";
+import { TProjectPermission } from "@app/lib/types";
+
+export type TCreateRoleDTO = {
+  data: Omit<TProjectRolesInsert, "projectId">;
+  projectSlug: string;
+} & Omit<TProjectPermission, "projectId">;
+
+export type TGetRoleBySlugDTO = {
+  roleSlug: string;
+  projectSlug: string;
+} & Omit<TProjectPermission, "projectId">;
+
+export type TUpdateRoleDTO = {
+  roleId: string;
+  data: Omit<TOrgRolesUpdate, "orgId">;
+  projectSlug: string;
+} & Omit<TProjectPermission, "projectId">;
+
+export type TDeleteRoleDTO = {
+  roleId: string;
+  projectSlug: string;
+} & Omit<TProjectPermission, "projectId">;
+
+export type TListRolesDTO = {
+  projectSlug: string;
+} & Omit<TProjectPermission, "projectId">;

backend/src/services/resource-cleanup/resource-cleanup-queue.ts

@@ -3,10 +3,12 @@ import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
 
 import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
+import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
 
 type TDailyResourceCleanUpQueueServiceFactoryDep = {
   auditLogDAL: Pick<TAuditLogDALFactory, "pruneAuditLog">;
   identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "removeExpiredTokens">;
+  secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
   queueService: TQueueServiceFactory;
 };
 
@@ -15,12 +17,14 @@ export type TDailyResourceCleanUpQueueServiceFactory = ReturnType<typeof dailyRe
 export const dailyResourceCleanUpQueueServiceFactory = ({
   auditLogDAL,
   queueService,
-  identityAccessTokenDAL
+  identityAccessTokenDAL,
+  secretSharingDAL
 }: TDailyResourceCleanUpQueueServiceFactoryDep) => {
   queueService.start(QueueName.DailyResourceCleanUp, async () => {
     logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
     await auditLogDAL.pruneAuditLog();
     await identityAccessTokenDAL.removeExpiredTokens();
+    await secretSharingDAL.pruneExpiredSharedSecrets();
     logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
   });
 

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -169,6 +169,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str
           // this is for root condition
           //  if the given folder id is root folder id then intial path is set as / instead of /root
           //  if not root folder the path here will be /<folder name>
+          depth: 1,
           path: db.raw(`CONCAT('/', (CASE WHEN "parentId" is NULL THEN '' ELSE ${TableName.SecretFolder}.name END))`),
           child: db.raw("NULL::uuid"),
           environmentSlug: `${TableName.Environment}.slug`
@@ -185,6 +186,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str
               .select({
                 // then we join join this folder name behind previous as we are going from child to parent
                 // the root folder check is used to avoid last / and also root name in folders
+                depth: db.raw("parent.depth + 1"),
                 path: db.raw(
                   `CONCAT( CASE 
                   WHEN  ${TableName.SecretFolder}."parentId" is NULL THEN '' 
@@ -199,7 +201,7 @@ const sqlFindSecretPathByFolderId = (db: Knex, projectId: string, folderIds: str
         );
     })
     .select("*")
-    .from<TSecretFolders & { child: string | null; path: string; environmentSlug: string }>("parent");
+    .from<TSecretFolders & { child: string | null; path: string; environmentSlug: string; depth: number }>("parent");
 
 export type TSecretFolderDALFactory = ReturnType<typeof secretFolderDALFactory>;
 // never change this. If u do write a migration for it
@@ -260,12 +262,23 @@ export const secretFolderDALFactory = (db: TDbClient) => {
     try {
       const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds);
 
+      //  travelling all the way from leaf node to root contains real path
       const rootFolders = groupBy(
         folders.filter(({ parentId }) => parentId === null),
         (i) => i.child || i.id // root condition then child and parent will null
       );
+      const actualFolders = groupBy(
+        folders.filter(({ depth }) => depth === 1),
+        (i) => i.id // root condition then child and parent will null
+      );
 
-      return folderIds.map((folderId) => rootFolders[folderId]?.[0]);
+      return folderIds.map((folderId) => {
+        if (!rootFolders[folderId]?.[0]) return;
+
+        const actualId = rootFolders[folderId][0].child || rootFolders[folderId][0].id;
+        const folder = actualFolders[actualId][0];
+        return { ...folder, path: rootFolders[folderId]?.[0].path };
+      });
     } catch (error) {
       throw new DatabaseError({ error, name: "Find by secret path" });
     }

backend/src/services/secret-folder/secret-folder-service.ts

@@ -253,7 +253,7 @@ export const secretFolderServiceFactory = ({
     const env = await projectEnvDAL.findOne({ projectId, slug: environment });
     if (!env) throw new BadRequestError({ message: "Environment not found", name: "Update folder" });
     const folder = await folderDAL
-      .findOne({ envId: env.id, id, parentId: parentFolder.id })
+      .findOne({ envId: env.id, id, parentId: parentFolder.id, isReserved: false })
       // now folder api accepts id based change
       // this is for cli backward compatiability and when cli removes this, we will remove this logic
       .catch(() => folderDAL.findOne({ envId: env.id, name: id, parentId: parentFolder.id }));
@@ -276,7 +276,11 @@ export const secretFolderServiceFactory = ({
     }
 
     const newFolder = await folderDAL.transaction(async (tx) => {
-      const [doc] = await folderDAL.update({ envId: env.id, id: folder.id, parentId: parentFolder.id }, { name }, tx);
+      const [doc] = await folderDAL.update(
+        { envId: env.id, id: folder.id, parentId: parentFolder.id, isReserved: false },
+        { name },
+        tx
+      );
       await folderVersionDAL.create(
         {
           name: doc.name,
@@ -324,7 +328,12 @@ export const secretFolderServiceFactory = ({
       if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
 
       const [doc] = await folderDAL.delete(
-        { envId: env.id, [uuidValidate(idOrName) ? "id" : "name"]: idOrName, parentId: parentFolder.id },
+        {
+          envId: env.id,
+          [uuidValidate(idOrName) ? "id" : "name"]: idOrName,
+          parentId: parentFolder.id,
+          isReserved: false
+        },
         tx
       );
       if (!doc) throw new BadRequestError({ message: "Folder not found", name: "Delete folder" });
@@ -354,7 +363,7 @@ export const secretFolderServiceFactory = ({
     const parentFolder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!parentFolder) return [];
 
-    const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id });
+    const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id, isReserved: false });
 
     return folders;
   };

backend/src/services/secret-folder/secret-folder-types.ts

@@ -1,5 +1,9 @@
 import { TProjectPermission } from "@app/lib/types";
 
+export enum ReservedFolders {
+  SecretReplication = "__reserve_replication_"
+}
+
 export type TCreateFolderDTO = {
   environment: string;
   path: string;

backend/src/services/secret-folder/secret-folder-version-dal.ts

@@ -15,7 +15,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
     try {
       const docs = await (tx || db)(TableName.SecretFolderVersion)
         .join(TableName.SecretFolder, `${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`)
-        .where({ parentId: folderId })
+        .where({ parentId: folderId, isReserved: false })
         .join<TSecretFolderVersions>(
           (tx || db)(TableName.SecretFolderVersion)
             .groupBy("envId", "folderId")

backend/src/services/secret-import/secret-import-dal.ts

@@ -20,14 +20,14 @@ export const secretImportDALFactory = (db: TDbClient) => {
     return lastPos?.position || 0;
   };
 
-  const updateAllPosition = async (folderId: string, pos: number, targetPos: number, tx?: Knex) => {
+  const updateAllPosition = async (folderId: string, pos: number, targetPos: number, positionInc = 1, tx?: Knex) => {
     try {
       if (targetPos === -1) {
         // this means delete
         await (tx || db)(TableName.SecretImport)
           .where({ folderId })
           .andWhere("position", ">", pos)
-          .decrement("position", 1);
+          .decrement("position", positionInc);
         return;
       }
 
@@ -36,13 +36,13 @@ export const secretImportDALFactory = (db: TDbClient) => {
           .where({ folderId })
           .where("position", "<=", targetPos)
           .andWhere("position", ">", pos)
-          .decrement("position", 1);
+          .decrement("position", positionInc);
       } else {
         await (tx || db)(TableName.SecretImport)
           .where({ folderId })
           .where("position", ">=", targetPos)
           .andWhere("position", "<", pos)
-          .increment("position", 1);
+          .increment("position", positionInc);
       }
     } catch (error) {
       throw new DatabaseError({ error, name: "Update position" });
@@ -74,6 +74,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
     try {
       const docs = await (tx || db)(TableName.SecretImport)
         .whereIn("folderId", folderIds)
+        .where("isReplication", false)
         .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
         .select(
           db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports,

backend/src/services/secret-import/secret-import-fns.ts

@@ -79,7 +79,7 @@ export const fnSecretsFromImports = async ({
   let secretsFromDeeperImports: TSecretImportSecrets[] = [];
   if (deeperImports.length) {
     secretsFromDeeperImports = await fnSecretsFromImports({
-      allowedImports: deeperImports,
+      allowedImports: deeperImports.filter(({ isReplication }) => !isReplication),
       secretImportDAL,
       folderDAL,
       secretDAL,

backend/src/services/secret-import/secret-import-service.ts

@@ -1,7 +1,12 @@
+import path from "node:path";
+
 import { ForbiddenError, subject } from "@casl/ability";
 
+import { TableName } from "@app/db/schemas";
+import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
+import { getReplicationFolderName } from "@app/ee/services/secret-replication/secret-replication-service";
 import { BadRequestError } from "@app/lib/errors";
 
 import { TProjectDALFactory } from "../project/project-dal";
@@ -16,6 +21,7 @@ import {
   TDeleteSecretImportDTO,
   TGetSecretImportsDTO,
   TGetSecretsFromImportDTO,
+  TResyncSecretImportReplicationDTO,
   TUpdateSecretImportDTO
 } from "./secret-import-types";
 
@@ -26,7 +32,8 @@ type TSecretImportServiceFactoryDep = {
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
   projectEnvDAL: TProjectEnvDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
-  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "replicateSecrets">;
+  licenseService: Pick<TLicenseServiceFactory, "getPlan">;
 };
 
 const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" });
@@ -40,7 +47,8 @@ export const secretImportServiceFactory = ({
   folderDAL,
   projectDAL,
   secretDAL,
-  secretQueueService
+  secretQueueService,
+  licenseService
 }: TSecretImportServiceFactoryDep) => {
   const createImport = async ({
     environment,
@@ -50,7 +58,8 @@ export const secretImportServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     projectId,
-    path
+    isReplication,
+    path: secretPath
   }: TCreateSecretImportDTO) => {
     const { permission } = await permissionService.getProjectPermission(
       actor,
@@ -63,7 +72,7 @@ export const secretImportServiceFactory = ({
     // check if user has permission to import into destination  path
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Create,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
     // check if user has permission to import from target path
@@ -74,10 +83,18 @@ export const secretImportServiceFactory = ({
         secretPath: data.path
       })
     );
+    if (isReplication) {
+      const plan = await licenseService.getPlan(actorOrgId);
+      if (!plan.secretApproval) {
+        throw new BadRequestError({
+          message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication."
+        });
+      }
+    }
 
     await projectDAL.checkProjectUpgradeStatus(projectId);
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" });
 
     const [importEnv] = await projectEnvDAL.findBySlugs(projectId, [data.environment]);
@@ -88,35 +105,62 @@ export const secretImportServiceFactory = ({
       const existingImport = await secretImportDAL.findOne({
         folderId: sourceFolder.id,
         importEnv: folder.environment.id,
-        importPath: path
+        importPath: secretPath
       });
       if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" });
     }
 
     const secImport = await secretImportDAL.transaction(async (tx) => {
       const lastPos = await secretImportDAL.findLastImportPosition(folder.id, tx);
-      return secretImportDAL.create(
+      const doc = await secretImportDAL.create(
         {
           folderId: folder.id,
           position: lastPos + 1,
           importEnv: importEnv.id,
-          importPath: data.path
+          importPath: data.path,
+          isReplication
         },
         tx
       );
+      if (doc.isReplication) {
+        await secretImportDAL.create(
+          {
+            folderId: folder.id,
+            position: lastPos + 2,
+            isReserved: true,
+            importEnv: folder.environment.id,
+            importPath: path.join(secretPath, getReplicationFolderName(doc.id))
+          },
+          tx
+        );
+      }
+      return doc;
     });
 
-    await secretQueueService.syncSecrets({
-      secretPath: secImport.importPath,
-      projectId,
-      environment: importEnv.slug
-    });
+    if (secImport.isReplication && sourceFolder) {
+      await secretQueueService.replicateSecrets({
+        secretPath: secImport.importPath,
+        projectId,
+        environmentSlug: importEnv.slug,
+        pickOnlyImportIds: [secImport.id],
+        actorId,
+        actor
+      });
+    } else {
+      await secretQueueService.syncSecrets({
+        secretPath,
+        projectId,
+        environmentSlug: environment,
+        actorId,
+        actor
+      });
+    }
 
     return { ...secImport, importEnv };
   };
 
   const updateImport = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -135,10 +179,10 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Edit,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" });
 
     const secImpDoc = await secretImportDAL.findOne({ folderId: folder.id, id });
@@ -158,7 +202,7 @@ export const secretImportServiceFactory = ({
       const existingImport = await secretImportDAL.findOne({
         folderId: sourceFolder.id,
         importEnv: folder.environment.id,
-        importPath: path
+        importPath: secretPath
       });
       if (existingImport) throw new BadRequestError({ message: "Cyclic import not allowed" });
     }
@@ -167,12 +211,31 @@ export const secretImportServiceFactory = ({
       const secImp = await secretImportDAL.findOne({ folderId: folder.id, id });
       if (!secImp) throw ERR_SEC_IMP_NOT_FOUND;
       if (data.position) {
-        await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, tx);
+        if (secImp.isReplication) {
+          await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 2, tx);
+        } else {
+          await secretImportDAL.updateAllPosition(folder.id, secImp.position, data.position, 1, tx);
+        }
+      }
+      if (secImp.isReplication) {
+        const replicationFolderPath = path.join(secretPath, getReplicationFolderName(secImp.id));
+        await secretImportDAL.update(
+          {
+            folderId: folder.id,
+            importEnv: folder.environment.id,
+            importPath: replicationFolderPath,
+            isReserved: true
+          },
+          { position: data?.position ? data.position + 1 : undefined },
+          tx
+        );
       }
       const [doc] = await secretImportDAL.update(
         { id, folderId: folder.id },
         {
-          position: data?.position,
+          // when moving replicated import, the position is meant for reserved import
+          // replicated one should always be behind the reserved import
+          position: data.position,
           importEnv: data?.environment ? importedEnv.id : undefined,
           importPath: data?.path
         },
@@ -184,7 +247,7 @@ export const secretImportServiceFactory = ({
   };
 
   const deleteImport = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -202,16 +265,34 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Delete,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Delete import" });
 
     const secImport = await secretImportDAL.transaction(async (tx) => {
       const [doc] = await secretImportDAL.delete({ folderId: folder.id, id }, tx);
       if (!doc) throw new BadRequestError({ name: "Sec imp del", message: "Secret import doc not found" });
-      await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, tx);
+      if (doc.isReplication) {
+        const replicationFolderPath = path.join(secretPath, getReplicationFolderName(doc.id));
+        const replicatedFolder = await folderDAL.findBySecretPath(projectId, environment, replicationFolderPath, tx);
+        if (replicatedFolder) {
+          await secretImportDAL.delete(
+            {
+              folderId: folder.id,
+              importEnv: folder.environment.id,
+              importPath: replicationFolderPath,
+              isReserved: true
+            },
+            tx
+          );
+          await folderDAL.deleteById(replicatedFolder.id, tx);
+        }
+        await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 2, tx);
+      } else {
+        await secretImportDAL.updateAllPosition(folder.id, doc.position, -1, 1, tx);
+      }
 
       const importEnv = await projectEnvDAL.findById(doc.importEnv);
       if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" });
@@ -219,16 +300,91 @@ export const secretImportServiceFactory = ({
     });
 
     await secretQueueService.syncSecrets({
-      secretPath: path,
+      secretPath,
       projectId,
-      environment
+      environmentSlug: environment,
+      actor,
+      actorId
     });
 
     return secImport;
   };
 
+  const resyncSecretImportReplication = async ({
+    environment,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    projectId,
+    path: secretPath,
+    id: secretImportDocId
+  }: TResyncSecretImportReplicationDTO) => {
+    const { permission, membership } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    // check if user has permission to import into destination  path
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Create,
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
+    );
+
+    const plan = await licenseService.getPlan(actorOrgId);
+    if (!plan.secretApproval) {
+      throw new BadRequestError({
+        message: "Failed to create secret replication due to plan restriction. Upgrade plan to create replication."
+      });
+    }
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+    if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Update import" });
+
+    const [secretImportDoc] = await secretImportDAL.find({
+      folderId: folder.id,
+      [`${TableName.SecretImport}.id` as "id"]: secretImportDocId
+    });
+    if (!secretImportDoc) throw new BadRequestError({ message: "Failed to find secret import" });
+
+    if (!secretImportDoc.isReplication) throw new BadRequestError({ message: "Import is not in replication mode" });
+
+    // check if user has permission to import from target path
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Create,
+      subject(ProjectPermissionSub.Secrets, {
+        environment: secretImportDoc.importEnv.slug,
+        secretPath: secretImportDoc.importPath
+      })
+    );
+
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
+    const sourceFolder = await folderDAL.findBySecretPath(
+      projectId,
+      secretImportDoc.importEnv.slug,
+      secretImportDoc.importPath
+    );
+
+    if (membership && sourceFolder) {
+      await secretQueueService.replicateSecrets({
+        secretPath: secretImportDoc.importPath,
+        projectId,
+        environmentSlug: secretImportDoc.importEnv.slug,
+        pickOnlyImportIds: [secretImportDoc.id],
+        actorId,
+        actor
+      });
+    }
+
+    return { message: "replication started" };
+  };
+
   const getImports = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -245,10 +401,10 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Read,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
 
     const secImports = await secretImportDAL.find({ folderId: folder.id });
@@ -256,7 +412,7 @@ export const secretImportServiceFactory = ({
   };
 
   const getSecretsFromImports = async ({
-    path,
+    path: secretPath,
     environment,
     projectId,
     actor,
@@ -273,13 +429,13 @@ export const secretImportServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Read,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) return [];
     // this will already order by position
     // so anything based on this order will also be in right position
-    const secretImports = await secretImportDAL.find({ folderId: folder.id });
+    const secretImports = await secretImportDAL.find({ folderId: folder.id, isReplication: false });
 
     const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
       permission.can(
@@ -299,6 +455,7 @@ export const secretImportServiceFactory = ({
     deleteImport,
     getImports,
     getSecretsFromImports,
+    resyncSecretImportReplication,
     fnSecretsFromImports
   };
 };

backend/src/services/secret-import/secret-import-types.ts

@@ -7,6 +7,7 @@ export type TCreateSecretImportDTO = {
     environment: string;
     path: string;
   };
+  isReplication?: boolean;
 } & TProjectPermission;
 
 export type TUpdateSecretImportDTO = {
@@ -16,6 +17,12 @@ export type TUpdateSecretImportDTO = {
   data: Partial<{ environment: string; path: string; position: number }>;
 } & TProjectPermission;
 
+export type TResyncSecretImportReplicationDTO = {
+  environment: string;
+  path: string;
+  id: string;
+} & TProjectPermission;
+
 export type TDeleteSecretImportDTO = {
   environment: string;
   path: string;

backend/src/services/secret-sharing/secret-sharing-dal.ts

@@ -0,0 +1,27 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { DatabaseError } from "@app/lib/errors";
+import { ormify } from "@app/lib/knex";
+
+export type TSecretSharingDALFactory = ReturnType<typeof secretSharingDALFactory>;
+
+export const secretSharingDALFactory = (db: TDbClient) => {
+  const sharedSecretOrm = ormify(db, TableName.SecretSharing);
+
+  const pruneExpiredSharedSecrets = async (tx?: Knex) => {
+    try {
+      const today = new Date();
+      const docs = await (tx || db)(TableName.SecretSharing).where("expiresAt", "<", today).del();
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "pruneExpiredSharedSecrets" });
+    }
+  };
+
+  return {
+    ...sharedSecretOrm,
+    pruneExpiredSharedSecrets
+  };
+};

backend/src/services/secret-sharing/secret-sharing-service.ts

@@ -0,0 +1,84 @@
+import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
+import { UnauthorizedError } from "@app/lib/errors";
+
+import { TSecretSharingDALFactory } from "./secret-sharing-dal";
+import { TCreateSharedSecretDTO, TDeleteSharedSecretDTO, TSharedSecretPermission } from "./secret-sharing-types";
+
+type TSecretSharingServiceFactoryDep = {
+  permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
+  secretSharingDAL: TSecretSharingDALFactory;
+};
+
+export type TSecretSharingServiceFactory = ReturnType<typeof secretSharingServiceFactory>;
+
+export const secretSharingServiceFactory = ({
+  permissionService,
+  secretSharingDAL
+}: TSecretSharingServiceFactoryDep) => {
+  const createSharedSecret = async (createSharedSecretInput: TCreateSharedSecretDTO) => {
+    const {
+      actor,
+      actorId,
+      orgId,
+      actorAuthMethod,
+      actorOrgId,
+      encryptedValue,
+      iv,
+      tag,
+      hashedHex,
+      expiresAt,
+      expiresAfterViews
+    } = createSharedSecretInput;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    if (!permission) throw new UnauthorizedError({ name: "User not in org" });
+    const newSharedSecret = await secretSharingDAL.create({
+      encryptedValue,
+      iv,
+      tag,
+      hashedHex,
+      expiresAt,
+      expiresAfterViews,
+      userId: actorId,
+      orgId
+    });
+    return { id: newSharedSecret.id };
+  };
+
+  const getSharedSecrets = async (getSharedSecretsInput: TSharedSecretPermission) => {
+    const { actor, actorId, orgId, actorAuthMethod, actorOrgId } = getSharedSecretsInput;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    if (!permission) throw new UnauthorizedError({ name: "User not in org" });
+    const userSharedSecrets = await secretSharingDAL.find({ userId: actorId, orgId }, { sort: [["expiresAt", "asc"]] });
+    return userSharedSecrets;
+  };
+
+  const getActiveSharedSecretByIdAndHashedHex = async (sharedSecretId: string, hashedHex: string) => {
+    const sharedSecret = await secretSharingDAL.findOne({ id: sharedSecretId, hashedHex });
+    if (sharedSecret.expiresAt && sharedSecret.expiresAt < new Date()) {
+      return;
+    }
+    if (sharedSecret.expiresAfterViews != null && sharedSecret.expiresAfterViews >= 0) {
+      if (sharedSecret.expiresAfterViews === 0) {
+        await secretSharingDAL.deleteById(sharedSecretId);
+        return;
+      }
+      await secretSharingDAL.updateById(sharedSecretId, { $decr: { expiresAfterViews: 1 } });
+    }
+    return sharedSecret;
+  };
+
+  const deleteSharedSecretById = async (deleteSharedSecretInput: TDeleteSharedSecretDTO) => {
+    const { actor, actorId, orgId, actorAuthMethod, actorOrgId, sharedSecretId } = deleteSharedSecretInput;
+    const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    if (!permission) throw new UnauthorizedError({ name: "User not in org" });
+    const deletedSharedSecret = await secretSharingDAL.deleteById(sharedSecretId);
+    return deletedSharedSecret;
+  };
+
+  return {
+    createSharedSecret,
+    getSharedSecrets,
+    deleteSharedSecretById,
+    getActiveSharedSecretByIdAndHashedHex
+  };
+};

backend/src/services/secret-sharing/secret-sharing-types.ts

@@ -0,0 +1,22 @@
+import { ActorAuthMethod, ActorType } from "../auth/auth-type";
+
+export type TSharedSecretPermission = {
+  actor: ActorType;
+  actorId: string;
+  actorAuthMethod: ActorAuthMethod;
+  actorOrgId: string;
+  orgId: string;
+};
+
+export type TCreateSharedSecretDTO = {
+  encryptedValue: string;
+  iv: string;
+  tag: string;
+  hashedHex: string;
+  expiresAt: Date;
+  expiresAfterViews: number;
+} & TSharedSecretPermission;
+
+export type TDeleteSharedSecretDTO = {
+  sharedSecretId: string;
+} & TSharedSecretPermission;

backend/src/services/secret/secret-fns.ts

@@ -32,6 +32,8 @@ import {
   TCreateManySecretsRawFn,
   TCreateManySecretsRawFnFactory,
   TFnSecretBlindIndexCheck,
+  TFnSecretBlindIndexCheckV2,
+  TFnSecretBulkDelete,
   TFnSecretBulkInsert,
   TFnSecretBulkUpdate,
   TUpdateManySecretsRawFn,
@@ -149,7 +151,8 @@ export const recursivelyGetSecretPaths = ({
 
     // Fetch all folders in env once with a single query
     const folders = await folderDAL.find({
-      envId: env.id
+      envId: env.id,
+      isReserved: false
     });
 
     // Build the folder hierarchy map
@@ -396,6 +399,30 @@ export const decryptSecretRaw = (
   };
 };
 
+// this is used when secret blind index already exist
+// mainly for secret approval
+export const fnSecretBlindIndexCheckV2 = async ({
+  inputSecrets,
+  folderId,
+  userId,
+  secretDAL
+}: TFnSecretBlindIndexCheckV2) => {
+  if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
+    throw new BadRequestError({ message: "Missing user id for personal secret" });
+  }
+  const secrets = await secretDAL.findByBlindIndexes(
+    folderId,
+    inputSecrets.map(({ secretBlindIndex, type }) => ({
+      blindIndex: secretBlindIndex,
+      type: type || SecretType.Shared
+    })),
+    userId
+  );
+  const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
+
+  return { secsGroupedByBlindIndex, secrets };
+};
+
 /**
  * Grabs and processes nested secret references from a string
  *
@@ -598,6 +625,35 @@ export const fnSecretBulkUpdate = async ({
   return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
 };
 
+export const fnSecretBulkDelete = async ({
+  folderId,
+  inputSecrets,
+  tx,
+  actorId,
+  secretDAL,
+  secretQueueService
+}: TFnSecretBulkDelete) => {
+  const deletedSecrets = await secretDAL.deleteMany(
+    inputSecrets.map(({ type, secretBlindIndex }) => ({
+      blindIndex: secretBlindIndex,
+      type
+    })),
+    folderId,
+    actorId,
+    tx
+  );
+
+  await Promise.allSettled(
+    deletedSecrets
+      .filter(({ secretReminderRepeatDays }) => Boolean(secretReminderRepeatDays))
+      .map(({ id, secretReminderRepeatDays }) =>
+        secretQueueService.removeSecretReminder({ secretId: id, repeatDays: secretReminderRepeatDays as number })
+      )
+  );
+
+  return deletedSecrets;
+};
+
 export const createManySecretsRawFnFactory = ({
   projectDAL,
   projectBotDAL,

backend/src/services/secret/secret-queue.ts

@@ -28,7 +28,12 @@ import { TWebhookDALFactory } from "../webhook/webhook-dal";
 import { fnTriggerWebhook } from "../webhook/webhook-fns";
 import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
-import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
+import {
+  TCreateSecretReminderDTO,
+  THandleReminderDTO,
+  TRemoveSecretReminderDTO,
+  TSyncSecretsDTO
+} from "./secret-types";
 
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 type TSecretQueueFactoryDep = {
@@ -59,8 +64,10 @@ export type TGetSecrets = {
 };
 
 const MAX_SYNC_SECRET_DEPTH = 5;
-const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`;
+export const uniqueSecretQueueKey = (environment: string, secretPath: string) =>
+  `secret-queue-dedupe-${environment}-${secretPath}`;
 
+type TIntegrationSecret = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
 export const secretQueueFactory = ({
   queueService,
   integrationDAL,
@@ -81,68 +88,6 @@ export const secretQueueFactory = ({
   secretTagDAL,
   secretVersionTagDAL
 }: TSecretQueueFactoryDep) => {
-  const createManySecretsRawFn = createManySecretsRawFnFactory({
-    projectDAL,
-    projectBotDAL,
-    secretDAL,
-    secretVersionDAL,
-    secretBlindIndexDAL,
-    secretTagDAL,
-    secretVersionTagDAL,
-    folderDAL
-  });
-
-  const updateManySecretsRawFn = updateManySecretsRawFnFactory({
-    projectDAL,
-    projectBotDAL,
-    secretDAL,
-    secretVersionDAL,
-    secretBlindIndexDAL,
-    secretTagDAL,
-    secretVersionTagDAL,
-    folderDAL
-  });
-
-  const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
-    await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
-      attempts: 3,
-      delay: 1000,
-      backoff: {
-        type: "exponential",
-        delay: 3000
-      },
-      removeOnComplete: true,
-      removeOnFail: true
-    });
-  };
-
-  const syncSecrets = async ({
-    deDupeQueue = {},
-    ...dto
-  }: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
-    const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
-    if (deDupeQueue?.[deDuplicationKey]) {
-      return;
-    }
-    // eslint-disable-next-line
-    deDupeQueue[deDuplicationKey] = true;
-    logger.info(
-      `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environment}] [path=${dto.secretPath}]`
-    );
-    await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
-      jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
-      removeOnFail: true,
-      removeOnComplete: true,
-      delay: 1000,
-      attempts: 5,
-      backoff: {
-        type: "exponential",
-        delay: 3000
-      }
-    });
-    await syncIntegrations({ ...dto, deDupeQueue });
-  };
-
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
     const appCfg = getConfig();
     await queueService.stopRepeatableJob(
@@ -237,8 +182,27 @@ export const secretQueueFactory = ({
       }
     }
   };
+  const createManySecretsRawFn = createManySecretsRawFnFactory({
+    projectDAL,
+    projectBotDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL
+  });
 
-  type Content = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
+  const updateManySecretsRawFn = updateManySecretsRawFnFactory({
+    projectDAL,
+    projectBotDAL,
+    secretDAL,
+    secretVersionDAL,
+    secretBlindIndexDAL,
+    secretTagDAL,
+    secretVersionTagDAL,
+    folderDAL
+  });
 
   /**
    * Return the secrets in a given [folderId] including secrets from
@@ -251,7 +215,7 @@ export const secretQueueFactory = ({
     key: string;
     depth: number;
   }) => {
-    let content: Content = {};
+    let content: TIntegrationSecret = {};
     if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
       logger.info(
         `getIntegrationSecrets: secret depth exceeded for [projectId=${dto.projectId}] [folderId=${dto.folderId}] [depth=${dto.depth}]`
@@ -301,7 +265,7 @@ export const secretQueueFactory = ({
     await expandSecrets(content);
 
     // check if current folder has any imports from other folders
-    const secretImport = await secretImportDAL.find({ folderId: dto.folderId });
+    const secretImport = await secretImportDAL.find({ folderId: dto.folderId, isReplication: false });
 
     // if no imports then return secrets in the current folder
     if (!secretImport) return content;
@@ -333,8 +297,122 @@ export const secretQueueFactory = ({
     return content;
   };
 
+  const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
+    await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
+      attempts: 3,
+      delay: 1000,
+      backoff: {
+        type: "exponential",
+        delay: 3000
+      },
+      removeOnComplete: true,
+      removeOnFail: true
+    });
+  };
+
+  const replicateSecrets = async (dto: Omit<TSyncSecretsDTO, "deDupeQueue">) => {
+    await queueService.queue(QueueName.SecretReplication, QueueJobs.SecretReplication, dto, {
+      attempts: 3,
+      backoff: {
+        type: "exponential",
+        delay: 2000
+      },
+      removeOnComplete: true,
+      removeOnFail: true
+    });
+  };
+
+  const syncSecrets = async <T extends boolean = false>({
+    // seperate de-dupe queue for integration sync and replication sync
+    _deDupeQueue: deDupeQueue = {},
+    _depth: depth = 0,
+    _deDupeReplicationQueue: deDupeReplicationQueue = {},
+    ...dto
+  }: TSyncSecretsDTO<T>) => {
+    logger.info(
+      `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environmentSlug}] [path=${dto.secretPath}]`
+    );
+    const deDuplicationKey = uniqueSecretQueueKey(dto.environmentSlug, dto.secretPath);
+    if (
+      !dto.excludeReplication
+        ? deDupeReplicationQueue?.[deDuplicationKey]
+        : deDupeQueue?.[deDuplicationKey] || depth > MAX_SYNC_SECRET_DEPTH
+    ) {
+      return;
+    }
+    // eslint-disable-next-line
+    deDupeQueue[deDuplicationKey] = true;
+    // eslint-disable-next-line
+    deDupeReplicationQueue[deDuplicationKey] = true;
+    await queueService.queue(
+      QueueName.SecretSync,
+      QueueJobs.SecretSync,
+      {
+        ...dto,
+        _deDupeQueue: deDupeQueue,
+        _deDupeReplicationQueue: deDupeReplicationQueue,
+        _depth: depth
+      } as TSyncSecretsDTO,
+      {
+        removeOnFail: true,
+        removeOnComplete: true,
+        delay: 1000,
+        attempts: 5,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        }
+      }
+    );
+  };
+
+  queueService.start(QueueName.SecretSync, async (job) => {
+    const {
+      _deDupeQueue: deDupeQueue,
+      _deDupeReplicationQueue: deDupeReplicationQueue,
+      _depth: depth,
+      secretPath,
+      projectId,
+      environmentSlug: environment,
+      excludeReplication,
+      actorId,
+      actor
+    } = job.data;
+
+    await queueService.queue(
+      QueueName.SecretWebhook,
+      QueueJobs.SecWebhook,
+      { environment, projectId, secretPath },
+      {
+        jobId: `secret-webhook-${environment}-${projectId}-${secretPath}`,
+        removeOnFail: { count: 5 },
+        removeOnComplete: true,
+        delay: 1000,
+        attempts: 5,
+        backoff: {
+          type: "exponential",
+          delay: 3000
+        }
+      }
+    );
+    await syncIntegrations({ secretPath, projectId, environment, deDupeQueue });
+    if (!excludeReplication) {
+      await replicateSecrets({
+        _deDupeReplicationQueue: deDupeReplicationQueue,
+        _depth: depth,
+        projectId,
+        secretPath,
+        actorId,
+        actor,
+        excludeReplication,
+        environmentSlug: environment
+      });
+    }
+  });
+
   queueService.start(QueueName.IntegrationSync, async (job) => {
     const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
+    if (depth > MAX_SYNC_SECRET_DEPTH) return;
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
@@ -348,7 +426,8 @@ export const secretQueueFactory = ({
       const linkSourceDto = {
         projectId,
         importEnv: folder.environment.id,
-        importPath: secretPath
+        importPath: secretPath,
+        isReplication: false
       };
       const imports = await secretImportDAL.find(linkSourceDto);
 
@@ -356,30 +435,31 @@ export const secretQueueFactory = ({
         // keep calling sync secret for all the imports made
         const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
         const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
-        const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
+        const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
         logger.info(
           `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
         );
         await Promise.all(
           imports
-            .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
+            .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string))
             // filter out already synced ones
             .filter(
               ({ folderId }) =>
                 !deDupeQueue[
-                  uniqueIntegrationKey(
-                    foldersGroupedById[folderId][0].environmentSlug,
-                    foldersGroupedById[folderId][0].path
+                  uniqueSecretQueueKey(
+                    foldersGroupedById[folderId][0]?.environmentSlug as string,
+                    foldersGroupedById[folderId][0]?.path as string
                   )
                 ]
             )
             .map(({ folderId }) =>
               syncSecrets({
-                depth: depth + 1,
                 projectId,
-                secretPath: foldersGroupedById[folderId][0].path,
-                environment: foldersGroupedById[folderId][0].environmentSlug,
-                deDupeQueue
+                secretPath: foldersGroupedById[folderId][0]?.path as string,
+                environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
+                _deDupeQueue: deDupeQueue,
+                _depth: depth + 1,
+                excludeReplication: true
               })
             )
         );
@@ -393,30 +473,31 @@ export const secretQueueFactory = ({
       if (secretReferences.length) {
         const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
         const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
-        const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id);
+        const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string);
         logger.info(
           `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
         );
         await Promise.all(
           secretReferences
-            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path))
+            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path))
             // filter out already synced ones
             .filter(
               ({ folderId }) =>
                 !deDupeQueue[
-                  uniqueIntegrationKey(
-                    referencedFoldersGroupedById[folderId][0].environmentSlug,
-                    referencedFoldersGroupedById[folderId][0].path
+                  uniqueSecretQueueKey(
+                    referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
+                    referencedFoldersGroupedById[folderId][0]?.path as string
                   )
                 ]
             )
             .map(({ folderId }) =>
               syncSecrets({
-                depth: depth + 1,
                 projectId,
-                secretPath: referencedFoldersGroupedById[folderId][0].path,
-                environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
-                deDupeQueue
+                secretPath: referencedFoldersGroupedById[folderId][0]?.path as string,
+                environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
+                _deDupeQueue: deDupeQueue,
+                _depth: depth + 1,
+                excludeReplication: true
               })
             )
         );
@@ -546,10 +627,11 @@ export const secretQueueFactory = ({
 
   return {
     // depth is internal only field thus no need to make it available outside
-    syncSecrets: (dto: TGetSecrets) => syncSecrets(dto),
+    syncSecrets,
     syncIntegrations,
     addSecretReminder,
     removeSecretReminder,
-    handleSecretReminder
+    handleSecretReminder,
+    replicateSecrets
   };
 };

backend/src/services/secret/secret-service.ts

@@ -35,6 +35,7 @@ import { TSecretDALFactory } from "./secret-dal";
 import {
   decryptSecretRaw,
   fnSecretBlindIndexCheck,
+  fnSecretBulkDelete,
   fnSecretBulkInsert,
   fnSecretBulkUpdate,
   getAllNestedSecretReferences,
@@ -53,8 +54,6 @@ import {
   TDeleteManySecretRawDTO,
   TDeleteSecretDTO,
   TDeleteSecretRawDTO,
-  TFnSecretBlindIndexCheckV2,
-  TFnSecretBulkDelete,
   TGetASecretDTO,
   TGetASecretRawDTO,
   TGetSecretsDTO,
@@ -139,53 +138,6 @@ export const secretServiceFactory = ({
     return secretBlindIndex;
   };
 
-  const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
-    const deletedSecrets = await secretDAL.deleteMany(
-      inputSecrets.map(({ type, secretBlindIndex }) => ({
-        blindIndex: secretBlindIndex,
-        type
-      })),
-      folderId,
-      actorId,
-      tx
-    );
-
-    for (const s of deletedSecrets) {
-      if (s.secretReminderRepeatDays) {
-        // eslint-disable-next-line no-await-in-loop
-        await secretQueueService
-          .removeSecretReminder({
-            secretId: s.id,
-            repeatDays: s.secretReminderRepeatDays
-          })
-          .catch((err) => {
-            logger.error(err, `Failed to delete secret reminder for secret with ID ${s?.id}`);
-          });
-      }
-    }
-
-    return deletedSecrets;
-  };
-
-  // this is used when secret blind index already exist
-  // mainly for secret approval
-  const fnSecretBlindIndexCheckV2 = async ({ inputSecrets, folderId, userId }: TFnSecretBlindIndexCheckV2) => {
-    if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
-      throw new BadRequestError({ message: "Missing user id for personal secret" });
-    }
-    const secrets = await secretDAL.findByBlindIndexes(
-      folderId,
-      inputSecrets.map(({ secretBlindIndex, type }) => ({
-        blindIndex: secretBlindIndex,
-        type: type || SecretType.Shared
-      })),
-      userId
-    );
-    const secsGroupedByBlindIndex = groupBy(secrets, (i) => i.secretBlindIndex as string);
-
-    return { secsGroupedByBlindIndex, secrets };
-  };
-
   const createSecret = async ({
     path,
     actor,
@@ -283,8 +235,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
-    // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
+    await secretQueueService.syncSecrets({
+      secretPath: path,
+      actorId,
+      actor,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
     return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
   };
 
@@ -413,8 +370,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
-    // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
     return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
   };
 
@@ -470,6 +432,8 @@ export const secretServiceFactory = ({
         projectId,
         folderId,
         actorId,
+        secretDAL,
+        secretQueueService,
         inputSecrets: [
           {
             type: inputSecret.type as SecretType,
@@ -481,8 +445,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
-
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
     // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
     return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
   };
@@ -551,7 +520,8 @@ export const secretServiceFactory = ({
 
     if (includeImports) {
       const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
-      const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
+      const allowedImports = secretImports.filter(({ importEnv, importPath, isReplication }) =>
+        !isReplication &&
         // if its service token allow full access over imported one
         actor === ActorType.SERVICE
           ? true
@@ -656,7 +626,7 @@ export const secretServiceFactory = ({
     // then search for imported secrets
     // here we consider the import order also thus starting from bottom
     if (!secret && includeImports) {
-      const secretImports = await secretImportDAL.find({ folderId });
+      const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
       const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
         // if its service token allow full access over imported one
         actor === ActorType.SERVICE
@@ -767,7 +737,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
 
     return newSecrets;
   };
@@ -867,7 +843,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
 
     return secrets;
   };
@@ -917,6 +899,8 @@ export const secretServiceFactory = ({
 
     const secretsDeleted = await secretDAL.transaction(async (tx) =>
       fnSecretBulkDelete({
+        secretDAL,
+        secretQueueService,
         inputSecrets: inputSecrets.map(({ type, secretName }) => ({
           secretBlindIndex: keyName2BlindIndex[secretName],
           type
@@ -929,7 +913,13 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
+    await secretQueueService.syncSecrets({
+      actor,
+      actorId,
+      secretPath: path,
+      projectId,
+      environmentSlug: folder.environment.slug
+    });
 
     return secretsDeleted;
   };
@@ -1109,9 +1099,6 @@ export const secretServiceFactory = ({
       skipMultilineEncoding
     });
 
-    await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1150,8 +1137,6 @@ export const secretServiceFactory = ({
     });
 
     await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1181,9 +1166,6 @@ export const secretServiceFactory = ({
       actorAuthMethod
     });
 
-    await snapshotService.performSnapshot(secret.folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1232,9 +1214,6 @@ export const secretServiceFactory = ({
       })
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return secrets.map((secret) =>
       decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
     );
@@ -1286,9 +1265,6 @@ export const secretServiceFactory = ({
       })
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return secrets.map((secret) =>
       decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
     );
@@ -1322,9 +1298,6 @@ export const secretServiceFactory = ({
       secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
     });
 
-    await snapshotService.performSnapshot(secrets[0].folderId);
-    await secretQueueService.syncSecrets({ secretPath, projectId, environment });
-
     return secrets.map((secret) =>
       decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
     );
@@ -1448,7 +1421,12 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folder.id);
-    await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
+    await secretQueueService.syncSecrets({
+      secretPath,
+      projectId: project.id,
+      environmentSlug: environment,
+      excludeReplication: true
+    });
 
     return {
       ...updatedSecret[0],
@@ -1550,7 +1528,12 @@ export const secretServiceFactory = ({
     );
 
     await snapshotService.performSnapshot(folder.id);
-    await secretQueueService.syncSecrets({ secretPath, projectId: project.id, environment });
+    await secretQueueService.syncSecrets({
+      secretPath,
+      projectId: project.id,
+      environmentSlug: environment,
+      excludeReplication: true
+    });
 
     return {
       ...updatedSecret[0],
@@ -1624,12 +1607,6 @@ export const secretServiceFactory = ({
     updateManySecretsRaw,
     deleteManySecretsRaw,
     getSecretVersions,
-    backfillSecretReferences,
-    // external services function
-    fnSecretBulkDelete,
-    fnSecretBulkUpdate,
-    fnSecretBlindIndexCheck,
-    fnSecretBulkInsert,
-    fnSecretBlindIndexCheckV2
+    backfillSecretReferences
   };
 };

backend/src/services/secret/secret-types.ts

@@ -11,6 +11,8 @@ import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/se
 import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
+import { ActorType } from "../auth/auth-type";
+
 type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
 
 type TPartialInputSecret = Pick<TSecrets, "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id">;
@@ -264,6 +266,10 @@ export type TFnSecretBulkDelete = {
   inputSecrets: Array<{ type: SecretType; secretBlindIndex: string }>;
   actorId: string;
   tx?: Knex;
+  secretDAL: Pick<TSecretDALFactory, "deleteMany">;
+  secretQueueService: {
+    removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
+  };
 };
 
 export type TFnSecretBlindIndexCheck = {
@@ -277,6 +283,7 @@ export type TFnSecretBlindIndexCheck = {
 
 // when blind index is already present
 export type TFnSecretBlindIndexCheckV2 = {
+  secretDAL: Pick<TSecretDALFactory, "findByBlindIndexes">;
   folderId: string;
   userId?: string;
   inputSecrets: Array<{ secretBlindIndex: string; type?: SecretType }>;
@@ -363,3 +370,27 @@ export type TUpdateManySecretsRawFn = {
   }[];
   userId?: string;
 };
+
+export enum SecretOperations {
+  Create = "create",
+  Update = "update",
+  Delete = "delete"
+}
+
+export type TSyncSecretsDTO<T extends boolean = false> = {
+  _deDupeQueue?: Record<string, boolean>;
+  _deDupeReplicationQueue?: Record<string, boolean>;
+  _depth?: number;
+  secretPath: string;
+  projectId: string;
+  environmentSlug: string;
+  // cases for just doing sync integration and webhook
+  excludeReplication?: T;
+} & (T extends true
+  ? object
+  : {
+      actor: ActorType;
+      actorId: string;
+      // used for import creation to trigger replication
+      pickOnlyImportIds?: string[];
+    });

backend/src/services/secret/secret-version-dal.ts

@@ -89,6 +89,7 @@ export const secretVersionDALFactory = (db: TDbClient) => {
 
   const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
     try {
+      if (!secretIds.length) return {};
       const docs: Array<TSecretVersions & { max: number }> = await (tx || db)(TableName.SecretVersion)
         .where("folderId", folderId)
         .whereIn(`${TableName.SecretVersion}.secretId`, secretIds)

cli/.gitignore

@@ -1,3 +1,4 @@
 .infisical.json
 dist/
 agent-config.test.yaml
+.test.env

cli/go.mod

@@ -3,7 +3,9 @@ module github.com/Infisical/infisical-merge
 go 1.21
 
 require (
+	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
 	github.com/charmbracelet/lipgloss v0.5.0
+	github.com/creack/pty v1.1.21
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/fatih/semgroup v1.2.0
 	github.com/gitleaks/go-gitdiff v0.8.0
@@ -29,7 +31,6 @@ require (
 require (
 	github.com/alessio/shellescape v1.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
-	github.com/bradleyjkemp/cupaloy/v2 v2.8.0 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/danieljoos/wincred v1.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

cli/go.sum

@@ -74,6 +74,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
+github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=
 github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

cli/packages/cmd/agent.go

@@ -15,7 +15,6 @@ import (
 	"path"
 	"runtime"
 	"slices"
-	"strings"
 	"sync"
 	"syscall"
 	"text/template"
@@ -257,19 +256,6 @@ func WriteBytesToFile(data *bytes.Buffer, outputPath string) error {
 	return err
 }
 
-func appendAPIEndpoint(address string) string {
-	// Ensure the address does not already end with "/api"
-	if strings.HasSuffix(address, "/api") {
-		return address
-	}
-
-	// Check if the address ends with a slash and append accordingly
-	if address[len(address)-1] == '/' {
-		return address + "api"
-	}
-	return address + "/api"
-}
-
 func ParseAgentConfig(configFile []byte) (*Config, error) {
 	var rawConfig struct {
 		Infisical InfisicalConfig `yaml:"infisical"`
@@ -290,7 +276,7 @@ func ParseAgentConfig(configFile []byte) (*Config, error) {
 		rawConfig.Infisical.Address = DEFAULT_INFISICAL_CLOUD_URL
 	}
 
-	config.INFISICAL_URL = appendAPIEndpoint(rawConfig.Infisical.Address)
+	config.INFISICAL_URL = util.AppendAPIEndpoint(rawConfig.Infisical.Address)
 
 	log.Info().Msgf("Infisical instance address set to %s", rawConfig.Infisical.Address)
 

cli/packages/cmd/login.go

@@ -101,7 +101,7 @@ var loginCmd = &cobra.Command{
 				//set domainQuery to false
 				if !overrideDomain {
 					domainQuery = false
-					config.INFISICAL_URL = config.INFISICAL_URL_MANUAL_OVERRIDE
+					config.INFISICAL_URL = util.AppendAPIEndpoint(config.INFISICAL_URL_MANUAL_OVERRIDE)
 				}
 
 			}

cli/packages/cmd/root.go

@@ -43,6 +43,7 @@ func init() {
 	rootCmd.PersistentFlags().Bool("silent", false, "Disable output of tip/info messages. Useful when running in scripts or CI/CD pipelines.")
 	rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
 		silent, err := cmd.Flags().GetBool("silent")
+		config.INFISICAL_URL = util.AppendAPIEndpoint(config.INFISICAL_URL)
 		if err != nil {
 			util.HandleError(err)
 		}

cli/packages/cmd/secrets.go

@@ -170,6 +170,11 @@ var secretsSetCmd = &cobra.Command{
 			util.HandleError(err, "Unable to get your local config details")
 		}
 
+		secretType, err := cmd.Flags().GetString("type")
+		if err != nil || (secretType != util.SECRET_TYPE_SHARED && secretType != util.SECRET_TYPE_PERSONAL) {
+			util.HandleError(err, "Unable to parse secret type")
+		}
+
 		loggedInUserDetails, err := util.GetCurrentLoggedInUserDetails()
 		if err != nil {
 			util.HandleError(err, "Unable to authenticate")
@@ -179,6 +184,7 @@ var secretsSetCmd = &cobra.Command{
 			util.PrintErrorMessageAndExit("Your login session has expired, please run [infisical login] and try again")
 		}
 
+
 		httpClient := resty.New().
 			SetAuthToken(loggedInUserDetails.UserCredentials.JTWToken).
 			SetHeader("Accept", "application/json")
@@ -223,7 +229,16 @@ var secretsSetCmd = &cobra.Command{
 		secretsToModify := []api.Secret{}
 		secretOperations := []SecretSetOperation{}
 
-		secretByKey := getSecretsByKeys(secrets)
+		sharedSecretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets))
+		personalSecretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets))
+	
+		for _, secret := range secrets {
+			if secret.Type == util.SECRET_TYPE_PERSONAL {
+				personalSecretMapByName[secret.Key] = secret
+			} else {
+				sharedSecretMapByName[secret.Key] = secret
+			}
+		}
 
 		for _, arg := range args {
 			splitKeyValueFromArg := strings.SplitN(arg, "=", 2)
@@ -251,7 +266,16 @@ var secretsSetCmd = &cobra.Command{
 				util.HandleError(err, "unable to encrypt your secrets")
 			}
 
-			if existingSecret, ok := secretByKey[key]; ok {
+			var existingSecret models.SingleEnvironmentVariable
+			var doesSecretExist bool
+
+			if secretType == util.SECRET_TYPE_SHARED {
+				existingSecret, doesSecretExist = sharedSecretMapByName[key]
+			} else {
+				existingSecret, doesSecretExist = personalSecretMapByName[key]
+			}
+
+			if doesSecretExist {
 				// case: secret exists in project so it needs to be modified
 				encryptedSecretDetails := api.Secret{
 					ID:                    existingSecret.ID,
@@ -291,7 +315,7 @@ var secretsSetCmd = &cobra.Command{
 					SecretValueIV:         base64.StdEncoding.EncodeToString(encryptedValue.Nonce),
 					SecretValueTag:        base64.StdEncoding.EncodeToString(encryptedValue.AuthTag),
 					SecretValueHash:       hashedValue,
-					Type:                  util.SECRET_TYPE_SHARED,
+					Type:                  secretType,
 					PlainTextKey:          key,
 				}
 				secretsToCreate = append(secretsToCreate, encryptedSecretDetails)
@@ -781,6 +805,7 @@ func init() {
 	secretsCmd.Flags().Bool("secret-overriding", true, "Prioritizes personal secrets, if any, with the same name over shared secrets")
 	secretsCmd.AddCommand(secretsSetCmd)
 	secretsSetCmd.Flags().String("path", "/", "set secrets within a folder path")
+	secretsSetCmd.Flags().String("type", util.SECRET_TYPE_SHARED, "the type of secret to create: personal or shared")
 
 	// Only supports logged in users (JWT auth)
 	secretsSetCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {

cli/packages/cmd/user.go

@@ -237,7 +237,7 @@ func NewDomainPrompt() (string, error) {
 		return "", err
 	}
 
-	return domain, nil
+	return util.AppendAPIEndpoint(domain), nil
 }
 
 func LoggedInUsersPrompt(profiles []string) (string, error) {

cli/packages/util/common.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+
+	"github.com/Infisical/infisical-merge/packages/config"
 )
 
 func GetHomeDir() (string, error) {
@@ -21,7 +23,7 @@ func WriteToFile(fileName string, dataToWrite []byte, filePerm os.FileMode) erro
 	return nil
 }
 
-func CheckIsConnectedToInternet() (ok bool) {
-	_, err := http.Get("http://clients3.google.com/generate_204")
+func ValidateInfisicalAPIConnection() (ok bool) {
+	_, err := http.Get(fmt.Sprintf("%v/status", config.INFISICAL_URL))
 	return err == nil
 }

cli/packages/util/credentials.go

@@ -88,7 +88,7 @@ func GetCurrentLoggedInUserDetails() (LoggedInUserDetails, error) {
 		//configFile.LoggedInUserDomain
 		//if not empty set as infisical url
 		if configFile.LoggedInUserDomain != "" {
-			config.INFISICAL_URL = configFile.LoggedInUserDomain
+			config.INFISICAL_URL = AppendAPIEndpoint(configFile.LoggedInUserDomain)
 		}
 
 		isAuthenticated := api.CallIsAuthenticated(httpClient)

cli/packages/util/helper.go

@@ -233,3 +233,16 @@ func getCurrentBranch() (string, error) {
 	}
 	return path.Base(strings.TrimSpace(out.String())), nil
 }
+
+func AppendAPIEndpoint(address string) string {
+	// Ensure the address does not already end with "/api"
+	if strings.HasSuffix(address, "/api") {
+		return address
+	}
+
+	// Check if the address ends with a slash and append accordingly
+	if address[len(address)-1] == '/' {
+		return address + "api"
+	}
+	return address + "/api"
+}

cli/packages/util/secrets.go

@@ -307,32 +307,33 @@ func FilterSecretsByTag(plainTextSecrets []models.SingleEnvironmentVariable, tag
 }
 
 func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectConfigFilePath string) ([]models.SingleEnvironmentVariable, error) {
-	isConnected := CheckIsConnectedToInternet()
 	var secretsToReturn []models.SingleEnvironmentVariable
 	// var serviceTokenDetails api.GetServiceTokenDetailsResponse
 	var errorToReturn error
 
 	if params.InfisicalToken == "" && params.UniversalAuthAccessToken == "" {
-		if isConnected {
-			log.Debug().Msg("GetAllEnvironmentVariables: Connected to internet, checking logged in creds")
-
-			if projectConfigFilePath == "" {
-				RequireLocalWorkspaceFile()
-			} else {
-				ValidateWorkspaceFile(projectConfigFilePath)
-			}
-
-			RequireLogin()
+		if projectConfigFilePath == "" {
+			RequireLocalWorkspaceFile()
+		} else {
+			ValidateWorkspaceFile(projectConfigFilePath)
 		}
 
+		RequireLogin()
+
 		log.Debug().Msg("GetAllEnvironmentVariables: Trying to fetch secrets using logged in details")
 
 		loggedInUserDetails, err := GetCurrentLoggedInUserDetails()
+		isConnected := ValidateInfisicalAPIConnection()
+
+		if isConnected {
+			log.Debug().Msg("GetAllEnvironmentVariables: Connected to Infisical instance, checking logged in creds")
+		}
+
 		if err != nil {
 			return nil, err
 		}
 
-		if loggedInUserDetails.LoginExpired {
+		if isConnected && loggedInUserDetails.LoginExpired {
 			PrintErrorMessageAndExit("Your login session has expired, please run [infisical login] and try again")
 		}
 
@@ -364,12 +365,12 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 
 		backupSecretsEncryptionKey := []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32]
 		if errorToReturn == nil {
-			WriteBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, backupSecretsEncryptionKey, secretsToReturn)
+			WriteBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, params.SecretsPath, backupSecretsEncryptionKey, secretsToReturn)
 		}
 
 		// only attempt to serve cached secrets if no internet connection and if at least one secret cached
 		if !isConnected {
-			backedSecrets, err := ReadBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, backupSecretsEncryptionKey)
+			backedSecrets, err := ReadBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, params.SecretsPath, backupSecretsEncryptionKey)
 			if len(backedSecrets) > 0 {
 				PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug")
 				secretsToReturn = backedSecrets
@@ -634,8 +635,9 @@ func GetPlainTextSecrets(key []byte, encryptedSecrets []api.EncryptedSecretV3) (
 	return plainTextSecrets, nil
 }
 
-func WriteBackupSecrets(workspace string, environment string, encryptionKey []byte, secrets []models.SingleEnvironmentVariable) error {
-	fileName := fmt.Sprintf("secrets_%s_%s", workspace, environment)
+func WriteBackupSecrets(workspace string, environment string, secretsPath string, encryptionKey []byte, secrets []models.SingleEnvironmentVariable) error {
+	formattedPath := strings.ReplaceAll(secretsPath, "/", "-")
+	fileName := fmt.Sprintf("secrets_%s_%s_%s", workspace, environment, formattedPath)
 	secrets_backup_folder_name := "secrets-backup"
 
 	_, fullConfigFileDirPath, err := GetFullConfigFilePath()
@@ -672,8 +674,9 @@ func WriteBackupSecrets(workspace string, environment string, encryptionKey []by
 	return nil
 }
 
-func ReadBackupSecrets(workspace string, environment string, encryptionKey []byte) ([]models.SingleEnvironmentVariable, error) {
-	fileName := fmt.Sprintf("secrets_%s_%s", workspace, environment)
+func ReadBackupSecrets(workspace string, environment string, secretsPath string, encryptionKey []byte) ([]models.SingleEnvironmentVariable, error) {
+	formattedPath := strings.ReplaceAll(secretsPath, "/", "-")
+	fileName := fmt.Sprintf("secrets_%s_%s_%s", workspace, environment, formattedPath)
 	secrets_backup_folder_name := "secrets-backup"
 
 	_, fullConfigFileDirPath, err := GetFullConfigFilePath()

cli/scripts/export_test_env.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+TEST_ENV_FILE=".test.env"
+
+# Check if the .env file exists
+if [ ! -f "$TEST_ENV_FILE" ]; then
+    echo "$TEST_ENV_FILE does not exist."
+    exit 1
+fi
+
+# Export the variables
+while IFS= read -r line
+do
+    # Skip empty lines and lines starting with #
+    if [[ -z "$line" || "$line" =~ ^\# ]]; then
+        continue
+    fi
+    # Read the key-value pair
+    IFS='=' read -r key value <<< "$line"
+    eval export $key=\$value
+done < "$TEST_ENV_FILE"
+
+echo "Test environment variables set."