Merge pull request #2591 from Infisical/maidul-uhdgwqudy

prevent sync of empty secret in ssm

.env.migration.example

@@ -1 +1,2 @@
 DB_CONNECTION_URI=
+AUDIT_LOGS_DB_CONNECTION_URI=

.github/workflows/build-staging-and-deploy-aws.yml

@@ -127,6 +127,7 @@ jobs:
       - name: Change directory to backend and install dependencies
         env:
           DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
+          AUDIT_LOGS_DB_CONNECTION_URI: ${{ secrets.AUDIT_LOGS_DB_CONNECTION_URI }}
         run: |
           cd backend
           npm install

README.md

@@ -135,9 +135,7 @@ Lean about Infisical's code scanning feature [here](https://infisical.com/docs/c
 
 This repo available under the [MIT expat license](https://github.com/Infisical/infisical/blob/main/LICENSE), with the exception of the `ee` directory which will contain premium enterprise features requiring a Infisical license.
 
-If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://infisical.cal.com/vlad/infisical-demo):
-
-<a href="[https://infisical.cal.com/vlad/infisical-demo](https://infisical.cal.com/vlad/infisical-demo)"><img alt="Schedule a meeting" src="https://cal.com/book-with-cal-dark.svg" /></a>
+If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://infisical.cal.com/vlad/infisical-demo).
 
 ## Security
 
@@ -163,4 +161,3 @@ Not sure where to get started? You can:
 - [Twitter](https://twitter.com/infisical) for fast news
 - [YouTube](https://www.youtube.com/@infisical_os) for videos on secret management
 - [Blog](https://infisical.com/blog) for secret management insights, articles, tutorials, and updates
-- [Roadmap](https://www.notion.so/infisical/be2d2585a6694e40889b03aef96ea36b?v=5b19a8127d1a4060b54769567a8785fa) for planned features

backend/package-lock.json

@@ -21,6 +21,7 @@
         "@fastify/etag": "^5.1.0",
         "@fastify/formbody": "^7.4.0",
         "@fastify/helmet": "^11.1.1",
+        "@fastify/multipart": "8.3.0",
         "@fastify/passport": "^2.4.0",
         "@fastify/rate-limit": "^9.0.0",
         "@fastify/session": "^10.7.0",
@@ -4311,6 +4312,15 @@
         "fast-uri": "^2.0.0"
       }
     },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
+      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@fastify/cookie": {
       "version": "9.3.1",
       "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz",
@@ -4381,6 +4391,20 @@
         "helmet": "^7.0.0"
       }
     },
+    "node_modules/@fastify/multipart": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-8.3.0.tgz",
+      "integrity": "sha512-A8h80TTyqUzaMVH0Cr9Qcm6RxSkVqmhK/MVBYHYeRRSUbUYv08WecjWKSlG2aSnD4aGI841pVxAjC+G1GafUeQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@fastify/busboy": "^2.1.0",
+        "@fastify/deepmerge": "^1.0.0",
+        "@fastify/error": "^3.0.0",
+        "fastify-plugin": "^4.0.0",
+        "secure-json-parse": "^2.4.0",
+        "stream-wormhole": "^1.1.0"
+      }
+    },
     "node_modules/@fastify/passport": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/@fastify/passport/-/passport-2.4.0.tgz",
@@ -16604,6 +16628,15 @@
       "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz",
       "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ=="
     },
+    "node_modules/stream-wormhole": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/stream-wormhole/-/stream-wormhole-1.1.0.tgz",
+      "integrity": "sha512-gHFfL3px0Kctd6Po0M8TzEvt3De/xu6cnRrjlfYNhwbhLPLwigI2t1nc6jrzNuaYg5C4YF78PPFuQPzRiqn9ew==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",

backend/package.json

@@ -45,13 +45,19 @@
     "test:e2e-coverage": "vitest run --coverage -c vitest.e2e.config.ts",
     "generate:component": "tsx ./scripts/create-backend-file.ts",
     "generate:schema": "tsx ./scripts/generate-schema-types.ts",
+    "auditlog-migration:latest": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:latest",
+    "auditlog-migration:up": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:up",
+    "auditlog-migration:down": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:down",
+    "auditlog-migration:list": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:list",
+    "auditlog-migration:status": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:status",
+    "auditlog-migration:rollback": "knex --knexfile ./src/db/auditlog-knexfile.ts migrate:rollback",
     "migration:new": "tsx ./scripts/create-migration.ts",
-    "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up",
-    "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down",
-    "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list",
-    "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest",
-    "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status",
-    "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback",
+    "migration:up": "npm run auditlog-migration:up && knex --knexfile ./src/db/knexfile.ts --client pg migrate:up",
+    "migration:down": "npm run auditlog-migration:down && knex --knexfile ./src/db/knexfile.ts --client pg migrate:down",
+    "migration:list": "npm run auditlog-migration:list && knex --knexfile ./src/db/knexfile.ts --client pg migrate:list",
+    "migration:latest": "npm run auditlog-migration:latest && knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest",
+    "migration:status": "npm run auditlog-migration:status && knex --knexfile ./src/db/knexfile.ts --client pg migrate:status",
+    "migration:rollback": "npm run auditlog-migration:rollback && knex --knexfile ./src/db/knexfile.ts migrate:rollback",
     "seed:new": "tsx ./scripts/create-seed-file.ts",
     "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
     "db:reset": "npm run migration:rollback -- --all && npm run migration:latest"
@@ -119,6 +125,7 @@
     "@fastify/etag": "^5.1.0",
     "@fastify/formbody": "^7.4.0",
     "@fastify/helmet": "^11.1.1",
+    "@fastify/multipart": "8.3.0",
     "@fastify/passport": "^2.4.0",
     "@fastify/rate-limit": "^9.0.0",
     "@fastify/session": "^10.7.0",

backend/scripts/generate-schema-types.ts

@@ -90,7 +90,12 @@ const main = async () => {
       .whereRaw("table_schema =  current_schema()")
       .select<{ tableName: string }[]>("table_name as tableName")
       .orderBy("table_name")
-  ).filter((el) => !el.tableName.includes("_migrations"));
+  ).filter(
+    (el) =>
+      !el.tableName.includes("_migrations") &&
+      !el.tableName.includes("audit_logs_") &&
+      el.tableName !== "intermediate_audit_logs"
+  );
 
   for (let i = 0; i < tables.length; i += 1) {
     const { tableName } = tables[i];

backend/src/db/auditlog-knexfile.ts

@@ -0,0 +1,75 @@
+// eslint-disable-next-line
+import "ts-node/register";
+
+import dotenv from "dotenv";
+import type { Knex } from "knex";
+import path from "path";
+
+// Update with your config settings. .
+dotenv.config({
+  path: path.join(__dirname, "../../../.env.migration")
+});
+dotenv.config({
+  path: path.join(__dirname, "../../../.env")
+});
+
+if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST) {
+  console.info("Dedicated audit log database not found. No further migrations necessary");
+  process.exit(0);
+}
+
+console.info("Executing migration on audit log database...");
+
+export default {
+  development: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    seeds: {
+      directory: "./seeds"
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  },
+  production: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  }
+} as Knex.Config;

backend/src/db/index.ts

@@ -1,2 +1,2 @@
 export type { TDbClient } from "./instance";
-export { initDbConnection } from "./instance";
+export { initAuditLogDbConnection, initDbConnection } from "./instance";

backend/src/db/instance.ts

@@ -70,3 +70,45 @@ export const initDbConnection = ({
 
   return db;
 };
+
+export const initAuditLogDbConnection = ({
+  dbConnectionUri,
+  dbRootCert
+}: {
+  dbConnectionUri: string;
+  dbRootCert?: string;
+}) => {
+  // akhilmhdh: the default Knex is knex.Knex<any, any[]>. but when assigned with knex({<config>}) the value is knex.Knex<any, unknown[]>
+  // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[]
+  // eslint-disable-next-line
+  const db: Knex<any, unknown[]> = knex({
+    client: "pg",
+    connection: {
+      connectionString: dbConnectionUri,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      // @ts-expect-error I have no clue why only for the port there is a type error
+      // eslint-disable-next-line
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: dbRootCert
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(dbRootCert, "base64").toString("ascii")
+          }
+        : false
+    }
+  });
+
+  // we add these overrides so that auditLogDb and the primary DB are interchangeable
+  db.primaryNode = () => {
+    return db;
+  };
+
+  db.replicaNode = () => {
+    return db;
+  };
+
+  return db;
+};

backend/src/db/manual-migrations/partition-audit-logs.ts

@@ -0,0 +1,161 @@
+import kx, { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const INTERMEDIATE_AUDIT_LOG_TABLE = "intermediate_audit_logs";
+
+const formatPartitionDate = (date: Date) => {
+  const year = date.getFullYear();
+  const month = String(date.getMonth() + 1).padStart(2, "0");
+  const day = String(date.getDate()).padStart(2, "0");
+
+  return `${year}-${month}-${day}`;
+};
+
+const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => {
+  const startDateStr = formatPartitionDate(startDate);
+  const endDateStr = formatPartitionDate(endDate);
+
+  const partitionName = `${TableName.AuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace(/-/g, "")}`;
+
+  await knex.schema.raw(
+    `CREATE TABLE ${partitionName} PARTITION OF ${TableName.AuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')`
+  );
+};
+
+const up = async (knex: Knex): Promise<void> => {
+  console.info("Dropping primary key of audit log table...");
+  await knex.schema.alterTable(TableName.AuditLog, (t) => {
+    // remove existing keys
+    t.dropPrimary();
+  });
+
+  // Get all indices of the audit log table and drop them
+  const indexNames: { rows: { indexname: string }[] } = await knex.raw(
+    `
+    SELECT indexname
+    FROM pg_indexes
+    WHERE tablename = '${TableName.AuditLog}'
+  `
+  );
+
+  console.log(
+    "Deleting existing audit log indices:",
+    indexNames.rows.map((e) => e.indexname)
+  );
+
+  for await (const row of indexNames.rows) {
+    await knex.raw(`DROP INDEX IF EXISTS ${row.indexname}`);
+  }
+
+  // renaming audit log to intermediate table
+  console.log("Renaming audit log table to the intermediate name");
+  await knex.schema.renameTable(TableName.AuditLog, INTERMEDIATE_AUDIT_LOG_TABLE);
+
+  if (!(await knex.schema.hasTable(TableName.AuditLog))) {
+    const createTableSql = knex.schema
+      .createTable(TableName.AuditLog, (t) => {
+        t.uuid("id").defaultTo(knex.fn.uuid());
+        t.string("actor").notNullable();
+        t.jsonb("actorMetadata").notNullable();
+        t.string("ipAddress");
+        t.string("eventType").notNullable();
+        t.jsonb("eventMetadata");
+        t.string("userAgent");
+        t.string("userAgentType");
+        t.datetime("expiresAt");
+        t.timestamps(true, true, true);
+        t.uuid("orgId");
+        t.string("projectId");
+        t.string("projectName");
+        t.primary(["id", "createdAt"]);
+      })
+      .toString();
+
+    console.info("Creating partition table...");
+    await knex.schema.raw(`
+        ${createTableSql} PARTITION BY RANGE ("createdAt");
+    `);
+
+    console.log("Adding indices...");
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      t.index(["projectId", "createdAt"]);
+      t.index(["orgId", "createdAt"]);
+      t.index("expiresAt");
+      t.index("orgId");
+      t.index("projectId");
+    });
+
+    console.log("Adding GIN indices...");
+
+    await knex.raw(
+      `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)`
+    );
+    console.log("GIN index for actorMetadata done");
+
+    await knex.raw(
+      `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.AuditLog} USING gin("eventMetadata" jsonb_path_ops)`
+    );
+    console.log("GIN index for eventMetadata done");
+
+    // create default partition
+    console.log("Creating default partition...");
+    await knex.schema.raw(`CREATE TABLE ${TableName.AuditLog}_default PARTITION OF ${TableName.AuditLog} DEFAULT`);
+
+    const nextDate = new Date();
+    nextDate.setDate(nextDate.getDate() + 1);
+    const nextDateStr = formatPartitionDate(nextDate);
+
+    console.log("Attaching existing audit log table as a partition...");
+    await knex.schema.raw(`
+    ALTER TABLE ${INTERMEDIATE_AUDIT_LOG_TABLE} ADD CONSTRAINT audit_log_old
+    CHECK ( "createdAt" < DATE '${nextDateStr}' );
+
+    ALTER TABLE ${TableName.AuditLog} ATTACH PARTITION ${INTERMEDIATE_AUDIT_LOG_TABLE}
+    FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' );
+    `);
+
+    // create partition from now until end of month
+    console.log("Creating audit log partitions ahead of time... next date:", nextDateStr);
+    await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1));
+
+    // create partitions 4 years ahead
+    const partitionMonths = 4 * 12;
+    const partitionPromises: Promise<void>[] = [];
+    for (let x = 1; x <= partitionMonths; x += 1) {
+      partitionPromises.push(
+        createAuditLogPartition(
+          knex,
+          new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1),
+          new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1)
+        )
+      );
+    }
+
+    await Promise.all(partitionPromises);
+    console.log("Partition migration complete");
+  }
+};
+
+export const executeMigration = async (url: string) => {
+  console.log("Executing migration...");
+  const knex = kx({
+    client: "pg",
+    connection: url
+  });
+
+  await knex.transaction(async (tx) => {
+    await up(tx);
+  });
+};
+
+const dbUrl = process.env.AUDIT_LOGS_DB_CONNECTION_URI;
+if (!dbUrl) {
+  console.error("Please provide a DB connection URL to the AUDIT_LOGS_DB_CONNECTION_URI env");
+  process.exit(1);
+}
+
+void executeMigration(dbUrl).then(() => {
+  console.log("Migration: partition-audit-logs DONE");
+  process.exit(0);
+});

backend/src/db/migrations/20241007052025_make-audit-log-independent.ts

@@ -0,0 +1,48 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+    const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+    const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesOrgIdExist) {
+        t.dropForeign("orgId");
+      }
+
+      if (doesProjectIdExist) {
+        t.dropForeign("projectId");
+      }
+
+      // add normalized field
+      if (!doesProjectNameExist) {
+        t.string("projectName");
+      }
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesOrgIdExist) {
+        t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      }
+      if (doesProjectIdExist) {
+        t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      }
+
+      // remove normalized field
+      if (doesProjectNameExist) {
+        t.dropColumn("projectName");
+      }
+    });
+  }
+}

backend/src/db/migrations/20241007202149_default-org-membership-roles.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "@app/db/schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  // org default role
+  if (await knex.schema.hasTable(TableName.Organization)) {
+    const hasDefaultRoleCol = await knex.schema.hasColumn(TableName.Organization, "defaultMembershipRole");
+
+    if (!hasDefaultRoleCol) {
+      await knex.schema.alterTable(TableName.Organization, (tb) => {
+        tb.string("defaultMembershipRole").notNullable().defaultTo("member");
+      });
+    }
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // org default role
+  if (await knex.schema.hasTable(TableName.Organization)) {
+    const hasDefaultRoleCol = await knex.schema.hasColumn(TableName.Organization, "defaultMembershipRole");
+
+    if (hasDefaultRoleCol) {
+      await knex.schema.alterTable(TableName.Organization, (tb) => {
+        tb.dropColumn("defaultMembershipRole");
+      });
+    }
+  }
+}

backend/src/db/migrations/20241015084434_increase-identity-metadata-col-length.ts

@@ -0,0 +1,19 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.IdentityMetadata, "value")) {
+    await knex.schema.alterTable(TableName.IdentityMetadata, (t) => {
+      t.string("value", 1020).alter();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.IdentityMetadata, "value")) {
+    await knex.schema.alterTable(TableName.IdentityMetadata, (t) => {
+      t.string("value", 255).alter();
+    });
+  }
+}

backend/src/db/schemas/audit-logs.ts

@@ -20,7 +20,8 @@ export const AuditLogsSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   orgId: z.string().uuid().nullable().optional(),
-  projectId: z.string().nullable().optional()
+  projectId: z.string().nullable().optional(),
+  projectName: z.string().nullable().optional()
 });
 
 export type TAuditLogs = z.infer<typeof AuditLogsSchema>;

backend/src/db/schemas/organizations.ts

@@ -19,7 +19,8 @@ export const OrganizationsSchema = z.object({
   authEnforced: z.boolean().default(false).nullable().optional(),
   scimEnabled: z.boolean().default(false).nullable().optional(),
   kmsDefaultKeyId: z.string().uuid().nullable().optional(),
-  kmsEncryptedDataKey: zodBuffer.nullable().optional()
+  kmsEncryptedDataKey: zodBuffer.nullable().optional(),
+  defaultMembershipRole: z.string().default("member")
 });
 
 export type TOrganizations = z.infer<typeof OrganizationsSchema>;

backend/src/ee/routes/v1/saml-router.ts

@@ -128,7 +128,10 @@ export const registerSamlRouter = async (server: FastifyZodProvider) => {
             .map((key) => {
               // for the ones like in format: http://schemas.xmlsoap.org/ws/2005/05/identity/claims/email
               const formatedKey = key.startsWith("http") ? key.split("/").at(-1) || "" : key;
-              return { key: formatedKey, value: String((profile.attributes as Record<string, string>)[key]) };
+              return {
+                key: formatedKey,
+                value: String((profile.attributes as Record<string, string>)[key]).substring(0, 1020)
+              };
             })
             .filter((el) => el.key && !["email", "firstName", "lastName"].includes(el.key));
 

backend/src/ee/services/audit-log/audit-log-dal.ts

@@ -1,8 +1,9 @@
-import { Knex } from "knex";
+// weird commonjs-related error in the CI requires us to do the import like this
+import knex from "knex";
 
 import { TDbClient } from "@app/db";
-import { AuditLogsSchema, TableName } from "@app/db/schemas";
-import { DatabaseError } from "@app/lib/errors";
+import { TableName } from "@app/db/schemas";
+import { DatabaseError, GatewayTimeoutError } from "@app/lib/errors";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
 import { logger } from "@app/lib/logger";
 import { QueueName } from "@app/queue";
@@ -46,7 +47,7 @@ export const auditLogDALFactory = (db: TDbClient) => {
       eventType?: EventType[];
       eventMetadata?: Record<string, string>;
     },
-    tx?: Knex
+    tx?: knex.Knex
   ) => {
     if (!orgId && !projectId) {
       throw new Error("Either orgId or projectId must be provided");
@@ -55,11 +56,10 @@ export const auditLogDALFactory = (db: TDbClient) => {
     try {
       // Find statements
       const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog)
-        .leftJoin(TableName.Project, `${TableName.AuditLog}.projectId`, `${TableName.Project}.id`)
         // eslint-disable-next-line func-names
         .where(function () {
           if (orgId) {
-            void this.where(`${TableName.Project}.orgId`, orgId).orWhere(`${TableName.AuditLog}.orgId`, orgId);
+            void this.where(`${TableName.AuditLog}.orgId`, orgId);
           } else if (projectId) {
             void this.where(`${TableName.AuditLog}.projectId`, projectId);
           }
@@ -72,23 +72,19 @@ export const auditLogDALFactory = (db: TDbClient) => {
       // Select statements
       void sqlQuery
         .select(selectAllTableCols(TableName.AuditLog))
-        .select(
-          db.ref("name").withSchema(TableName.Project).as("projectName"),
-          db.ref("slug").withSchema(TableName.Project).as("projectSlug")
-        )
         .limit(limit)
         .offset(offset)
         .orderBy(`${TableName.AuditLog}.createdAt`, "desc");
 
       // Special case: Filter by actor ID
       if (actorId) {
-        void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actorId]);
+        void sqlQuery.whereRaw(`"actorMetadata" @> jsonb_build_object('userId', ?::text)`, [actorId]);
       }
 
       // Special case: Filter by key/value pairs in eventMetadata field
       if (eventMetadata && Object.keys(eventMetadata).length) {
         Object.entries(eventMetadata).forEach(([key, value]) => {
-          void sqlQuery.whereRaw(`"eventMetadata"->>'${key}' = ?`, [value]);
+          void sqlQuery.whereRaw(`"eventMetadata" @> jsonb_build_object(?::text, ?::text)`, [key, value]);
         });
       }
 
@@ -109,30 +105,25 @@ export const auditLogDALFactory = (db: TDbClient) => {
       if (endDate) {
         void sqlQuery.where(`${TableName.AuditLog}.createdAt`, "<=", endDate);
       }
-      const docs = await sqlQuery;
 
-      return docs.map((doc) => {
-        // Our type system refuses to acknowledge that the project name and slug are present in the doc, due to the disjointed query structure above.
-        // This is a quick and dirty way to get around the types.
-        const projectDoc = doc as unknown as { projectName: string; projectSlug: string };
+      // we timeout long running queries to prevent DB resource issues (2 minutes)
+      const docs = await sqlQuery.timeout(1000 * 120);
 
-        return {
-          ...AuditLogsSchema.parse(doc),
-          ...(projectDoc?.projectSlug && {
-            project: {
-              name: projectDoc.projectName,
-              slug: projectDoc.projectSlug
-            }
-          })
-        };
-      });
+      return docs;
     } catch (error) {
+      if (error instanceof knex.KnexTimeoutError) {
+        throw new GatewayTimeoutError({
+          error,
+          message: "Failed to fetch audit logs due to timeout. Add more search filters."
+        });
+      }
+
       throw new DatabaseError({ error });
     }
   };
 
   // delete all audit log that have expired
-  const pruneAuditLog = async (tx?: Knex) => {
+  const pruneAuditLog = async (tx?: knex.Knex) => {
     const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000;
     const MAX_RETRY_ON_FAILURE = 3;
 
@@ -148,6 +139,7 @@ export const auditLogDALFactory = (db: TDbClient) => {
           .where("expiresAt", "<", today)
           .select("id")
           .limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
+
         // eslint-disable-next-line no-await-in-loop
         deletedAuditLogIds = await (tx || db)(TableName.AuditLog)
           .whereIn("id", findExpiredLogSubQuery)

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -74,6 +74,7 @@ export const auditLogQueueServiceFactory = ({
       actorMetadata: actor.metadata,
       userAgent,
       projectId,
+      projectName: project?.name,
       ipAddress,
       orgId,
       eventType: event.type,

backend/src/ee/services/ldap-config/ldap-config-service.ts

@@ -1,14 +1,7 @@
 import { ForbiddenError } from "@casl/ability";
 import jwt from "jsonwebtoken";
 
-import {
-  OrgMembershipRole,
-  OrgMembershipStatus,
-  SecretKeyEncoding,
-  TableName,
-  TLdapConfigsUpdate,
-  TUsers
-} from "@app/db/schemas";
+import { OrgMembershipStatus, SecretKeyEncoding, TableName, TLdapConfigsUpdate, TUsers } from "@app/db/schemas";
 import { TGroupDALFactory } from "@app/ee/services/group/group-dal";
 import { addUsersToGroupByUserIds, removeUsersFromGroupByUserIds } from "@app/ee/services/group/group-fns";
 import { TUserGroupMembershipDALFactory } from "@app/ee/services/group/user-group-membership-dal";
@@ -28,6 +21,7 @@ import { TokenType } from "@app/services/auth-token/auth-token-types";
 import { TGroupProjectDALFactory } from "@app/services/group-project/group-project-dal";
 import { TOrgBotDALFactory } from "@app/services/org/org-bot-dal";
 import { TOrgDALFactory } from "@app/services/org/org-dal";
+import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns";
 import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
@@ -444,11 +438,14 @@ export const ldapConfigServiceFactory = ({
           { tx }
         );
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole);
+
           await orgDAL.createMembership(
             {
               userId: userAlias.userId,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: OrgMembershipStatus.Accepted,
               isActive: true
             },
@@ -529,12 +526,15 @@ export const ldapConfigServiceFactory = ({
         );
 
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole);
+
           await orgMembershipDAL.create(
             {
               userId: newUser.id,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: newUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },

backend/src/ee/services/oidc/oidc-config-service.ts

@@ -3,7 +3,7 @@ import { ForbiddenError } from "@casl/ability";
 import jwt from "jsonwebtoken";
 import { Issuer, Issuer as OpenIdIssuer, Strategy as OpenIdStrategy, TokenSet } from "openid-client";
 
-import { OrgMembershipRole, OrgMembershipStatus, SecretKeyEncoding, TableName, TUsers } from "@app/db/schemas";
+import { OrgMembershipStatus, SecretKeyEncoding, TableName, TUsers } from "@app/db/schemas";
 import { TOidcConfigsUpdate } from "@app/db/schemas/oidc-configs";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
@@ -23,6 +23,7 @@ import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-se
 import { TokenType } from "@app/services/auth-token/auth-token-types";
 import { TOrgBotDALFactory } from "@app/services/org/org-bot-dal";
 import { TOrgDALFactory } from "@app/services/org/org-dal";
+import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns";
 import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
 import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
 import { getServerCfg } from "@app/services/super-admin/super-admin-service";
@@ -187,12 +188,15 @@ export const oidcConfigServiceFactory = ({
           { tx }
         );
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole);
+
           await orgMembershipDAL.create(
             {
               userId: userAlias.userId,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: foundUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },
@@ -261,12 +265,15 @@ export const oidcConfigServiceFactory = ({
         );
 
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole);
+
           await orgMembershipDAL.create(
             {
               userId: newUser.id,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: newUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },

backend/src/ee/services/saml-config/saml-config-service.ts

@@ -2,7 +2,6 @@ import { ForbiddenError } from "@casl/ability";
 import jwt from "jsonwebtoken";
 
 import {
-  OrgMembershipRole,
   OrgMembershipStatus,
   SecretKeyEncoding,
   TableName,
@@ -26,6 +25,7 @@ import { TokenType } from "@app/services/auth-token/auth-token-types";
 import { TIdentityMetadataDALFactory } from "@app/services/identity/identity-metadata-dal";
 import { TOrgBotDALFactory } from "@app/services/org/org-bot-dal";
 import { TOrgDALFactory } from "@app/services/org/org-dal";
+import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns";
 import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
 import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
 import { getServerCfg } from "@app/services/super-admin/super-admin-service";
@@ -369,12 +369,15 @@ export const samlConfigServiceFactory = ({
           { tx }
         );
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole);
+
           await orgMembershipDAL.create(
             {
               userId: userAlias.userId,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: foundUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },
@@ -472,12 +475,15 @@ export const samlConfigServiceFactory = ({
         );
 
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(organization.defaultMembershipRole);
+
           await orgMembershipDAL.create(
             {
               userId: newUser.id,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: newUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },

backend/src/ee/services/scim/scim-service.ts

@@ -16,6 +16,7 @@ import { AuthTokenType } from "@app/services/auth/auth-type";
 import { TGroupProjectDALFactory } from "@app/services/group-project/group-project-dal";
 import { TOrgDALFactory } from "@app/services/org/org-dal";
 import { deleteOrgMembershipFn } from "@app/services/org/org-fns";
+import { getDefaultOrgMembershipRole } from "@app/services/org/org-role-fns";
 import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
@@ -318,12 +319,15 @@ export const scimServiceFactory = ({
         );
 
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(org.defaultMembershipRole);
+
           orgMembership = await orgMembershipDAL.create(
             {
               userId: userAlias.userId,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.NoAccess,
+              role,
+              roleId,
               status: user.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },
@@ -391,12 +395,15 @@ export const scimServiceFactory = ({
         orgMembership = foundOrgMembership;
 
         if (!orgMembership) {
+          const { role, roleId } = await getDefaultOrgMembershipRole(org.defaultMembershipRole);
+
           orgMembership = await orgMembershipDAL.create(
             {
               userId: user.id,
               inviteEmail: email,
               orgId,
-              role: OrgMembershipRole.Member,
+              role,
+              roleId,
               status: user.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited, // if user is fully completed, then set status to accepted, otherwise set it to invited so we can update it later
               isActive: true
             },

backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts

@@ -240,7 +240,8 @@ export const secretSnapshotServiceFactory = ({
             },
             tx
           );
-          const snapshotSecrets = await snapshotSecretV2BridgeDAL.insertMany(
+
+          const snapshotSecrets = await snapshotSecretV2BridgeDAL.batchInsert(
             secretVersions.map(({ id }) => ({
               secretVersionId: id,
               envId: folder.environment.envId,
@@ -248,7 +249,8 @@ export const secretSnapshotServiceFactory = ({
             })),
             tx
           );
-          const snapshotFolders = await snapshotFolderDAL.insertMany(
+
+          const snapshotFolders = await snapshotFolderDAL.batchInsert(
             folderVersions.map(({ id }) => ({
               folderVersionId: id,
               envId: folder.environment.envId,

backend/src/lib/config/env.ts

@@ -34,6 +34,12 @@ const envSchema = z
     DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default(
       `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`
     ),
+    AUDIT_LOGS_DB_CONNECTION_URI: zpStr(
+      z.string().describe("Postgres database connection string for Audit logs").optional()
+    ),
+    AUDIT_LOGS_DB_ROOT_CERT: zpStr(
+      z.string().describe("Postgres database base64-encoded CA cert for Audit logs").optional()
+    ),
     MAX_LEASE_LIMIT: z.coerce.number().default(10000),
     DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()),
     DB_HOST: zpStr(z.string().describe("Postgres database host").optional()),

backend/src/lib/errors/index.ts

@@ -23,6 +23,18 @@ export class InternalServerError extends Error {
   }
 }
 
+export class GatewayTimeoutError extends Error {
+  name: string;
+
+  error: unknown;
+
+  constructor({ name, error, message }: { message?: string; name?: string; error?: unknown }) {
+    super(message || "Timeout error");
+    this.name = name || "GatewayTimeoutError";
+    this.error = error;
+  }
+}
+
 export class UnauthorizedError extends Error {
   name: string;
 

backend/src/lib/fn/array.ts

@@ -70,3 +70,14 @@ export const objectify = <T, Key extends string | number | symbol, Value = T>(
     {} as Record<Key, Value>
   );
 };
+
+/**
+ * Chunks an array into smaller arrays of the given size.
+ */
+export const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    chunks.push(array.slice(i, i + chunkSize));
+  }
+  return chunks;
+};

backend/src/lib/knex/scim.ts

@@ -8,12 +8,14 @@ const appendParentToGroupingOperator = (parentPath: string, filter: Filter) => {
   return filter;
 };
 
-export const generateKnexQueryFromScim = (
+const processDynamicQuery = (
   rootQuery: Knex.QueryBuilder,
-  rootScimFilter: string,
-  getAttributeField: (attr: string) => string | null
+  scimRootFilterAst: Filter,
+  getAttributeField: (attr: string) => string | null,
+  depth = 0
 ) => {
-  const scimRootFilterAst = parse(rootScimFilter);
+  if (depth > 20) return;
+
   const stack = [
     {
       scimFilterAst: scimRootFilterAst,
@@ -75,42 +77,35 @@ export const generateKnexQueryFromScim = (
         break;
       }
       case "and": {
-        void query.andWhere((subQueryBuilder) => {
-          scimFilterAst.filters.forEach((el) => {
-            stack.push({
-              query: subQueryBuilder,
-              scimFilterAst: el
-            });
+        scimFilterAst.filters.forEach((el) => {
+          void query.andWhere((subQueryBuilder) => {
+            processDynamicQuery(subQueryBuilder, el, getAttributeField, depth + 1);
           });
         });
         break;
       }
       case "or": {
-        void query.orWhere((subQueryBuilder) => {
-          scimFilterAst.filters.forEach((el) => {
-            stack.push({
-              query: subQueryBuilder,
-              scimFilterAst: el
-            });
+        scimFilterAst.filters.forEach((el) => {
+          void query.orWhere((subQueryBuilder) => {
+            processDynamicQuery(subQueryBuilder, el, getAttributeField, depth + 1);
           });
         });
         break;
       }
       case "not": {
         void query.whereNot((subQueryBuilder) => {
-          stack.push({
-            query: subQueryBuilder,
-            scimFilterAst: scimFilterAst.filter
-          });
+          processDynamicQuery(subQueryBuilder, scimFilterAst.filter, getAttributeField, depth + 1);
         });
         break;
       }
       case "[]": {
-        void query.whereNot((subQueryBuilder) => {
-          stack.push({
-            query: subQueryBuilder,
-            scimFilterAst: appendParentToGroupingOperator(scimFilterAst.attrPath, scimFilterAst.valFilter)
-          });
+        void query.where((subQueryBuilder) => {
+          processDynamicQuery(
+            subQueryBuilder,
+            appendParentToGroupingOperator(scimFilterAst.attrPath, scimFilterAst.valFilter),
+            getAttributeField,
+            depth + 1
+          );
         });
         break;
       }
@@ -119,3 +114,12 @@ export const generateKnexQueryFromScim = (
     }
   }
 };
+
+export const generateKnexQueryFromScim = (
+  rootQuery: Knex.QueryBuilder,
+  rootScimFilter: string,
+  getAttributeField: (attr: string) => string | null
+) => {
+  const scimRootFilterAst = parse(rootScimFilter);
+  return processDynamicQuery(rootQuery, scimRootFilterAst, getAttributeField);
+};

backend/src/main.ts

@@ -1,7 +1,7 @@
 import dotenv from "dotenv";
 import path from "path";
 
-import { initDbConnection } from "./db";
+import { initAuditLogDbConnection, initDbConnection } from "./db";
 import { keyStoreFactory } from "./keystore/keystore";
 import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env";
 import { isMigrationMode } from "./lib/fn";
@@ -25,6 +25,13 @@ const run = async () => {
     }))
   });
 
+  const auditLogDb = appCfg.AUDIT_LOGS_DB_CONNECTION_URI
+    ? initAuditLogDbConnection({
+        dbConnectionUri: appCfg.AUDIT_LOGS_DB_CONNECTION_URI,
+        dbRootCert: appCfg.AUDIT_LOGS_DB_ROOT_CERT
+      })
+    : undefined;
+
   // Case: App is running in packaged mode (binary), and migration mode is enabled.
   // Run the migrations and exit the process after completion.
   if (IS_PACKAGED && isMigrationMode()) {
@@ -46,7 +53,7 @@ const run = async () => {
   const queue = queueServiceFactory(appCfg.REDIS_URL);
   const keyStore = keyStoreFactory(appCfg.REDIS_URL);
 
-  const server = await main({ db, smtp, logger, queue, keyStore });
+  const server = await main({ db, auditLogDb, smtp, logger, queue, keyStore });
   const bootstrap = await bootstrapCheck({ db });
 
   // eslint-disable-next-line

backend/src/queue/queue-service.ts

@@ -1,7 +1,7 @@
 import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
 import Redis from "ioredis";
 
-import { SecretKeyEncoding } from "@app/db/schemas";
+import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
 import {
   TScanFullRepoEventPayload,
@@ -32,7 +32,8 @@ export enum QueueName {
   SecretReplication = "secret-replication",
   SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
   ProjectV3Migration = "project-v3-migration",
-  AccessTokenStatusUpdate = "access-token-status-update"
+  AccessTokenStatusUpdate = "access-token-status-update",
+  ImportSecretsFromExternalSource = "import-secrets-from-external-source"
 }
 
 export enum QueueJobs {
@@ -56,7 +57,8 @@ export enum QueueJobs {
   SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
   ProjectV3Migration = "project-v3-migration",
   IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
-  ServiceTokenStatusUpdate = "service-token-status-update"
+  ServiceTokenStatusUpdate = "service-token-status-update",
+  ImportSecretsFromExternalSource = "import-secrets-from-external-source"
 }
 
 export type TQueueJobTypes = {
@@ -166,6 +168,19 @@ export type TQueueJobTypes = {
     name: QueueJobs.ProjectV3Migration;
     payload: { projectId: string };
   };
+  [QueueName.ImportSecretsFromExternalSource]: {
+    name: QueueJobs.ImportSecretsFromExternalSource;
+    payload: {
+      actorEmail: string;
+      data: {
+        iv: string;
+        tag: string;
+        ciphertext: string;
+        algorithm: SecretEncryptionAlgo;
+        encoding: SecretKeyEncoding;
+      };
+    };
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/server/app.ts

@@ -30,6 +30,7 @@ import { fastifySwagger } from "./plugins/swagger";
 import { registerRoutes } from "./routes";
 
 type TMain = {
+  auditLogDb?: Knex;
   db: Knex;
   smtp: TSmtpService;
   logger?: Logger;
@@ -38,7 +39,7 @@ type TMain = {
 };
 
 // Run the server!
-export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
+export const main = async ({ db, auditLogDb, smtp, logger, queue, keyStore }: TMain) => {
   const appCfg = getConfig();
   const server = fastify({
     logger: appCfg.NODE_ENV === "test" ? false : logger,
@@ -94,7 +95,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
 
     await server.register(maintenanceMode);
 
-    await server.register(registerRoutes, { smtp, queue, db, keyStore });
+    await server.register(registerRoutes, { smtp, queue, db, auditLogDb, keyStore });
 
     if (appCfg.isProductionMode) {
       await server.register(registerExternalNextjs, {

backend/src/server/plugins/add-errors-to-response-schemas.ts

@@ -3,9 +3,12 @@ import fp from "fastify-plugin";
 
 import { DefaultResponseErrorsSchema } from "../routes/sanitizedSchemas";
 
+const isScimRoutes = (pathname: string) =>
+  pathname.startsWith("/api/v1/scim/Users") || pathname.startsWith("/api/v1/scim/Groups");
+
 export const addErrorsToResponseSchemas = fp(async (server) => {
   server.addHook("onRoute", (routeOptions) => {
-    if (routeOptions.schema && routeOptions.schema.response) {
+    if (routeOptions.schema && routeOptions.schema.response && !isScimRoutes(routeOptions.path)) {
       routeOptions.schema.response = {
         ...DefaultResponseErrorsSchema,
         ...routeOptions.schema.response

backend/src/server/plugins/error-handler.ts

@@ -7,6 +7,7 @@ import {
   BadRequestError,
   DatabaseError,
   ForbiddenRequestError,
+  GatewayTimeoutError,
   InternalServerError,
   NotFoundError,
   ScimRequestError,
@@ -25,7 +26,8 @@ enum HttpStatusCodes {
   Unauthorized = 401,
   Forbidden = 403,
   // eslint-disable-next-line @typescript-eslint/no-shadow
-  InternalServerError = 500
+  InternalServerError = 500,
+  GatewayTimeout = 504
 }
 
 export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider) => {
@@ -47,6 +49,10 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
       void res
         .status(HttpStatusCodes.InternalServerError)
         .send({ statusCode: HttpStatusCodes.InternalServerError, message: "Something went wrong", error: error.name });
+    } else if (error instanceof GatewayTimeoutError) {
+      void res
+        .status(HttpStatusCodes.GatewayTimeout)
+        .send({ statusCode: HttpStatusCodes.GatewayTimeout, message: error.message, error: error.name });
     } else if (error instanceof ZodError) {
       void res
         .status(HttpStatusCodes.Unauthorized)
@@ -91,7 +97,11 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider
         message
       });
     } else {
-      void res.send(error);
+      void res.status(HttpStatusCodes.InternalServerError).send({
+        statusCode: HttpStatusCodes.InternalServerError,
+        error: "InternalServerError",
+        message: "Something went wrong"
+      });
     }
   });
 });

backend/src/server/routes/index.ts

@@ -97,6 +97,7 @@ import { certificateTemplateDALFactory } from "@app/services/certificate-templat
 import { certificateTemplateEstConfigDALFactory } from "@app/services/certificate-template/certificate-template-est-config-dal";
 import { certificateTemplateServiceFactory } from "@app/services/certificate-template/certificate-template-service";
 import { cmekServiceFactory } from "@app/services/cmek/cmek-service";
+import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue";
 import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service";
 import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal";
 import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal";
@@ -214,11 +215,12 @@ import { registerV3Routes } from "./v3";
 export const registerRoutes = async (
   server: FastifyZodProvider,
   {
+    auditLogDb,
     db,
     smtp: smtpService,
     queue: queueService,
     keyStore
-  }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory }
+  }: { auditLogDb?: Knex; db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory }
 ) => {
   const appCfg = getConfig();
   if (!appCfg.DISABLE_SECRET_SCANNING) {
@@ -283,7 +285,7 @@ export const registerRoutes = async (
   const identityOidcAuthDAL = identityOidcAuthDALFactory(db);
   const identityAzureAuthDAL = identityAzureAuthDALFactory(db);
 
-  const auditLogDAL = auditLogDALFactory(db);
+  const auditLogDAL = auditLogDALFactory(auditLogDb ?? db);
   const auditLogStreamDAL = auditLogStreamDALFactory(db);
   const trustedIpDAL = trustedIpDALFactory(db);
   const telemetryDAL = telemetryDALFactory(db);
@@ -491,6 +493,9 @@ export const registerRoutes = async (
     authDAL,
     userDAL
   });
+
+  const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL, projectDAL });
+
   const orgService = orgServiceFactory({
     userAliasDAL,
     identityMetadataDAL,
@@ -513,7 +518,8 @@ export const registerRoutes = async (
     userDAL,
     groupDAL,
     orgBotDAL,
-    oidcConfigDAL
+    oidcConfigDAL,
+    projectBotService
   });
   const signupService = authSignupServiceFactory({
     tokenService,
@@ -531,7 +537,7 @@ export const registerRoutes = async (
     orgService,
     licenseService
   });
-  const orgRoleService = orgRoleServiceFactory({ permissionService, orgRoleDAL });
+  const orgRoleService = orgRoleServiceFactory({ permissionService, orgRoleDAL, orgDAL });
   const superAdminService = superAdminServiceFactory({
     userDAL,
     authService: loginService,
@@ -572,7 +578,6 @@ export const registerRoutes = async (
     secretScanningDAL,
     secretScanningQueue
   });
-  const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL, projectDAL });
 
   const projectMembershipService = projectMembershipServiceFactory({
     projectMembershipDAL,
@@ -836,7 +841,10 @@ export const registerRoutes = async (
     integrationAuthDAL,
     snapshotDAL,
     snapshotSecretV2BridgeDAL,
-    secretApprovalRequestDAL
+    secretApprovalRequestDAL,
+    projectKeyDAL,
+    projectUserMembershipRoleDAL,
+    orgService
   });
   const secretImportService = secretImportServiceFactory({
     licenseService,
@@ -1201,12 +1209,26 @@ export const registerRoutes = async (
     permissionService
   });
 
-  const migrationService = externalMigrationServiceFactory({
-    projectService,
-    orgService,
+  const externalMigrationQueue = externalMigrationQueueFactory({
     projectEnvService,
-    permissionService,
-    secretService
+    projectDAL,
+    projectService,
+    smtpService,
+    kmsService,
+    projectEnvDAL,
+    secretVersionDAL: secretVersionV2BridgeDAL,
+    secretTagDAL,
+    secretVersionTagDAL: secretVersionTagV2BridgeDAL,
+    folderDAL,
+    secretDAL: secretV2BridgeDAL,
+    queueService,
+    secretV2BridgeService
+  });
+
+  const migrationService = externalMigrationServiceFactory({
+    externalMigrationQueue,
+    userDAL,
+    permissionService
   });
 
   await superAdminService.initServerCfg();

backend/src/server/routes/v1/integration-router.ts

@@ -52,7 +52,13 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
       }),
       response: {
         200: z.object({
-          integration: IntegrationsSchema
+          integration: IntegrationsSchema.extend({
+            environment: z.object({
+              slug: z.string().trim(),
+              name: z.string().trim(),
+              id: z.string().trim()
+            })
+          })
         })
       }
     },
@@ -138,7 +144,13 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
       }),
       response: {
         200: z.object({
-          integration: IntegrationsSchema
+          integration: IntegrationsSchema.extend({
+            environment: z.object({
+              slug: z.string().trim(),
+              name: z.string().trim(),
+              id: z.string().trim()
+            })
+          })
         })
       }
     },

backend/src/server/routes/v1/organization-router.ts

@@ -1,3 +1,4 @@
+import slugify from "@sindresorhus/slugify";
 import { z } from "zod";
 
 import {
@@ -11,8 +12,6 @@ import {
 } from "@app/db/schemas";
 import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types";
 import { AUDIT_LOGS, ORGANIZATIONS } from "@app/lib/api-docs";
-import { getConfig } from "@app/lib/config/env";
-import { BadRequestError } from "@app/lib/errors";
 import { getLastMidnightDateISO } from "@app/lib/fn";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
@@ -125,12 +124,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
           })
             .merge(
               z.object({
-                project: z
-                  .object({
-                    name: z.string(),
-                    slug: z.string()
-                  })
-                  .optional(),
                 event: z.object({
                   type: z.string(),
                   metadata: z.any()
@@ -145,13 +138,8 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
         })
       }
     },
-    onRequest: verifyAuth([AuthMode.JWT]),
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
-      const appCfg = getConfig();
-      if (appCfg.isCloud) {
-        throw new BadRequestError({ message: "Infisical cloud audit log is in maintenance mode." });
-      }
-
       const auditLogs = await server.services.auditLog.listAuditLogs({
         filter: {
           ...req.query,
@@ -168,6 +156,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
         actorAuthMethod: req.permission.authMethod,
         actor: req.permission.type
       });
+
       return { auditLogs };
     }
   });
@@ -229,7 +218,15 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
           .regex(/^[a-zA-Z0-9-]+$/, "Slug must only contain alphanumeric characters or hyphens")
           .optional(),
         authEnforced: z.boolean().optional(),
-        scimEnabled: z.boolean().optional()
+        scimEnabled: z.boolean().optional(),
+        defaultMembershipRoleSlug: z
+          .string()
+          .min(1)
+          .trim()
+          .refine((v) => slugify(v) === v, {
+            message: "Membership role must be a valid slug"
+          })
+          .optional()
       }),
       response: {
         200: z.object({

backend/src/server/routes/v1/password-router.ts

@@ -65,7 +65,7 @@ export const registerPasswordRouter = async (server: FastifyZodProvider) => {
       const appCfg = getConfig();
       await server.services.password.changePassword({ ...req.body, userId: req.permission.id });
 
-      void res.cookie("jid", appCfg.COOKIE_SECRET_SIGN_KEY, {
+      void res.cookie("jid", "", {
         httpOnly: true,
         path: "/",
         sameSite: "strict",

backend/src/server/routes/v3/external-migration-router.ts

@@ -1,30 +1,50 @@
-import { z } from "zod";
+import fastifyMultipart from "@fastify/multipart";
 
+import { BadRequestError } from "@app/lib/errors";
 import { readLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
 
+const MB25_IN_BYTES = 26214400;
+
 export const registerExternalMigrationRouter = async (server: FastifyZodProvider) => {
+  await server.register(fastifyMultipart);
+
   server.route({
     method: "POST",
+    bodyLimit: MB25_IN_BYTES,
     url: "/env-key",
     config: {
       rateLimit: readLimit
     },
-    schema: {
-      body: z.object({
-        decryptionKey: z.string().trim().min(1),
-        encryptedJson: z.object({
-          nonce: z.string().trim().min(1),
-          data: z.string().trim().min(1)
-        })
-      })
-    },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
+      const data = await req.file({
+        limits: {
+          fileSize: MB25_IN_BYTES
+        }
+      });
+
+      if (!data) {
+        throw new BadRequestError({ message: "No file provided" });
+      }
+
+      const fullFile = Buffer.from(await data.toBuffer()).toString("utf8");
+      const parsedJsonFile = JSON.parse(fullFile) as { nonce: string; data: string };
+
+      const decryptionKey = (data.fields.decryptionKey as { value: string }).value;
+
+      if (!parsedJsonFile.nonce || !parsedJsonFile.data) {
+        throw new BadRequestError({ message: "Invalid file format. Nonce or data missing." });
+      }
+
+      if (!decryptionKey) {
+        throw new BadRequestError({ message: "Decryption key is required" });
+      }
+
       await server.services.migration.importEnvKeyData({
-        decryptionKey: req.body.decryptionKey,
-        encryptedJson: req.body.encryptedJson,
+        decryptionKey,
+        encryptedJson: parsedJsonFile,
         actorId: req.permission.id,
         actor: req.permission.type,
         actorOrgId: req.permission.orgId,

backend/src/services/external-migration/external-migration-fns.ts

@@ -4,22 +4,41 @@ import sjcl from "sjcl";
 import tweetnacl from "tweetnacl";
 import tweetnaclUtil from "tweetnacl-util";
 
-import { OrgMembershipRole, ProjectMembershipRole, SecretType } from "@app/db/schemas";
-import { BadRequestError } from "@app/lib/errors";
+import { SecretType } from "@app/db/schemas";
+import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { chunkArray } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 
-import { TOrgServiceFactory } from "../org/org-service";
+import { TKmsServiceFactory } from "../kms/kms-service";
+import { KmsDataKey } from "../kms/kms-types";
+import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
-import { TSecretServiceFactory } from "../secret/secret-service";
+import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
+import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
+import { fnSecretBulkInsert, getAllNestedSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns";
+import type { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
+import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
 import { InfisicalImportData, TEnvKeyExportJSON, TImportInfisicalDataCreate } from "./external-migration-types";
 
 export type TImportDataIntoInfisicalDTO = {
-  projectService: TProjectServiceFactory;
-  orgService: TOrgServiceFactory;
-  projectEnvService: TProjectEnvServiceFactory;
-  secretService: TSecretServiceFactory;
+  projectDAL: Pick<TProjectDALFactory, "transaction">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys">;
+  secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create">;
+  secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
+
+  folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath">;
+  projectService: Pick<TProjectServiceFactory, "createProject">;
+  projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
+  secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
 
   input: TImportInfisicalDataCreate;
 };
@@ -46,13 +65,13 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
   const parsedJson: TEnvKeyExportJSON = JSON.parse(decryptedJson) as TEnvKeyExportJSON;
 
   const infisicalImportData: InfisicalImportData = {
-    projects: new Map<string, { name: string; id: string }>(),
-    environments: new Map<string, { name: string; id: string; projectId: string }>(),
-    secrets: new Map<string, { name: string; id: string; projectId: string; environmentId: string; value: string }>()
+    projects: [],
+    environments: [],
+    secrets: []
   };
 
   parsedJson.apps.forEach((app: { name: string; id: string }) => {
-    infisicalImportData.projects.set(app.id, { name: app.name, id: app.id });
+    infisicalImportData.projects.push({ name: app.name, id: app.id });
   });
 
   // string to string map for env templates
@@ -63,7 +82,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
 
   // environments
   for (const env of parsedJson.baseEnvironments) {
-    infisicalImportData.environments?.set(env.id, {
+    infisicalImportData.environments.push({
       id: env.id,
       name: envTemplates.get(env.environmentRoleId)!,
       projectId: env.envParentId
@@ -75,9 +94,8 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
     if (!env.includes("|")) {
       const envData = parsedJson.envs[env];
       for (const secret of Object.keys(envData.variables)) {
-        const id = randomUUID();
-        infisicalImportData.secrets?.set(id, {
-          id,
+        infisicalImportData.secrets.push({
+          id: randomUUID(),
           name: secret,
           environmentId: env,
           value: envData.variables[secret].val
@@ -91,9 +109,14 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
 
 export const importDataIntoInfisicalFn = async ({
   projectService,
-  orgService,
-  projectEnvService,
-  secretService,
+  projectEnvDAL,
+  projectDAL,
+  secretDAL,
+  kmsService,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL,
   input: { data, actor, actorId, actorOrgId, actorAuthMethod }
 }: TImportDataIntoInfisicalDTO) => {
   // Import data to infisical
@@ -103,95 +126,145 @@ export const importDataIntoInfisicalFn = async ({
 
   const originalToNewProjectId = new Map<string, string>();
   const originalToNewEnvironmentId = new Map<string, string>();
+  const projectsNotImported: string[] = [];
 
-  for await (const [id, project] of data.projects) {
-    const newProject = await projectService
-      .createProject({
-        actor,
-        actorId,
-        actorOrgId,
-        actorAuthMethod,
-        workspaceName: project.name,
-        createDefaultEnvs: false
-      })
-      .catch(() => {
-        throw new BadRequestError({ message: `Failed to import to project [name:${project.name}] [id:${id}]` });
-      });
-
-    originalToNewProjectId.set(project.id, newProject.id);
-  }
-
-  // Invite user importing projects
-  const invites = await orgService.inviteUserToOrganization({
-    actorAuthMethod,
-    actorId,
-    actorOrgId,
-    actor,
-    inviteeEmails: [],
-    orgId: actorOrgId,
-    organizationRoleSlug: OrgMembershipRole.NoAccess,
-    projects: Array.from(originalToNewProjectId.values()).map((project) => ({
-      id: project,
-      projectRoleSlug: [ProjectMembershipRole.Member]
-    }))
-  });
-  if (!invites) {
-    throw new BadRequestError({ message: `Failed to invite user to projects: [userId:${actorId}]` });
-  }
-
-  // Import environments
-  if (data.environments) {
-    for await (const [id, environment] of data.environments) {
-      try {
-        const newEnvironment = await projectEnvService.createEnvironment({
+  await projectDAL.transaction(async (tx) => {
+    for await (const project of data.projects) {
+      const newProject = await projectService
+        .createProject({
           actor,
           actorId,
           actorOrgId,
           actorAuthMethod,
-          name: environment.name,
-          projectId: originalToNewProjectId.get(environment.projectId)!,
-          slug: slugify(`${environment.name}-${alphaNumericNanoId(4)}`)
+          workspaceName: project.name,
+          createDefaultEnvs: false,
+          tx
+        })
+        .catch((e) => {
+          logger.error(e, `Failed to import to project [name:${project.name}]`);
+          throw new BadRequestError({ message: `Failed to import to project [name:${project.name}]` });
         });
+      originalToNewProjectId.set(project.id, newProject.id);
+    }
 
-        if (!newEnvironment) {
-          logger.error(`Failed to import environment: [name:${environment.name}] [id:${id}]`);
+    // Import environments
+    if (data.environments) {
+      for await (const environment of data.environments) {
+        const projectId = originalToNewProjectId.get(environment.projectId);
+        const slug = slugify(`${environment.name}-${alphaNumericNanoId(4)}`);
+
+        if (!projectId) {
+          projectsNotImported.push(environment.projectId);
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        const existingEnv = await projectEnvDAL.findOne({ projectId, slug }, tx);
+
+        if (existingEnv) {
           throw new BadRequestError({
-            message: `Failed to import environment: [name:${environment.name}] [id:${id}]`
+            message: `Environment with slug '${slug}' already exist`,
+            name: "CreateEnvironment"
           });
         }
-        originalToNewEnvironmentId.set(id, newEnvironment.slug);
-      } catch (error) {
-        throw new BadRequestError({
-          message: `Failed to import environment: ${environment.name}]`,
-          name: "EnvKeyMigrationImportEnvironment"
+
+        const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
+        const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
+        await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
+
+        originalToNewEnvironmentId.set(environment.id, doc.slug);
+      }
+    }
+
+    if (data.secrets && data.secrets.length > 0) {
+      const mappedToEnvironmentId = new Map<
+        string,
+        {
+          secretKey: string;
+          secretValue: string;
+        }[]
+      >();
+
+      for (const secret of data.secrets) {
+        if (!originalToNewEnvironmentId.get(secret.environmentId)) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        if (!mappedToEnvironmentId.has(secret.environmentId)) {
+          mappedToEnvironmentId.set(secret.environmentId, []);
+        }
+        mappedToEnvironmentId.get(secret.environmentId)!.push({
+          secretKey: secret.name,
+          secretValue: secret.value || ""
         });
       }
-    }
-  }
 
-  // Import secrets
-  if (data.secrets) {
-    for await (const [id, secret] of data.secrets) {
-      const dataProjectId = data.environments?.get(secret.environmentId)?.projectId;
-      if (!dataProjectId) {
-        throw new BadRequestError({ message: `Failed to import secret "${secret.name}", project not found` });
-      }
-      const projectId = originalToNewProjectId.get(dataProjectId);
-      const newSecret = await secretService.createSecretRaw({
-        actorId,
-        actor,
-        actorOrgId,
-        environment: originalToNewEnvironmentId.get(secret.environmentId)!,
-        actorAuthMethod,
-        projectId: projectId!,
-        secretPath: "/",
-        secretName: secret.name,
-        type: SecretType.Shared,
-        secretValue: secret.value
-      });
-      if (!newSecret) {
-        throw new BadRequestError({ message: `Failed to import secret: [name:${secret.name}] [id:${id}]` });
+      // for each of the mappedEnvironmentId
+      for await (const [envId, secrets] of mappedToEnvironmentId) {
+        const environment = data.environments.find((env) => env.id === envId);
+        const projectId = originalToNewProjectId.get(environment?.projectId as string)!;
+
+        if (!projectId) {
+          throw new BadRequestError({ message: `Failed to import secret, project not found` });
+        }
+
+        const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
+          {
+            type: KmsDataKey.SecretManager,
+            projectId
+          },
+          tx
+        );
+
+        const envSlug = originalToNewEnvironmentId.get(envId)!;
+        const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx);
+        if (!folder)
+          throw new NotFoundError({
+            message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`,
+            name: "Create secret"
+          });
+
+        const secretBatches = chunkArray(secrets, 2500);
+        for await (const secretBatch of secretBatches) {
+          const secretsByKeys = await secretDAL.findBySecretKeys(
+            folder.id,
+            secretBatch.map((el) => ({
+              key: el.secretKey,
+              type: SecretType.Shared
+            })),
+            tx
+          );
+          if (secretsByKeys.length) {
+            throw new BadRequestError({
+              message: `Secret already exist: ${secretsByKeys.map((el) => el.key).join(",")}`
+            });
+          }
+          await fnSecretBulkInsert({
+            inputSecrets: secretBatch.map((el) => {
+              const references = getAllNestedSecretReferences(el.secretValue);
+
+              return {
+                version: 1,
+                encryptedValue: el.secretValue
+                  ? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
+                  : undefined,
+                key: el.secretKey,
+                references,
+                type: SecretType.Shared
+              };
+            }),
+            folderId: folder.id,
+            secretDAL,
+            secretVersionDAL,
+            secretTagDAL,
+            secretVersionTagDAL,
+            tx
+          });
+        }
       }
     }
-  }
+  });
+
+  return { projectsNotImported };
 };

backend/src/services/external-migration/external-migration-queue.ts

@@ -0,0 +1,152 @@
+import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
+import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption";
+import { logger } from "@app/lib/logger";
+import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+
+import { TKmsServiceFactory } from "../kms/kms-service";
+import { TProjectDALFactory } from "../project/project-dal";
+import { TProjectServiceFactory } from "../project/project-service";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
+import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
+import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
+import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
+import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
+import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
+import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
+import { importDataIntoInfisicalFn } from "./external-migration-fns";
+import { ExternalPlatforms, TImportInfisicalDataCreate } from "./external-migration-types";
+
+export type TExternalMigrationQueueFactoryDep = {
+  smtpService: TSmtpService;
+  queueService: TQueueServiceFactory;
+
+  projectDAL: Pick<TProjectDALFactory, "transaction">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys">;
+  secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create">;
+  secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
+
+  folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath">;
+  projectService: Pick<TProjectServiceFactory, "createProject">;
+  projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
+  secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
+};
+
+export type TExternalMigrationQueueFactory = ReturnType<typeof externalMigrationQueueFactory>;
+
+export const externalMigrationQueueFactory = ({
+  queueService,
+  projectService,
+  smtpService,
+  projectDAL,
+  projectEnvService,
+  secretV2BridgeService,
+  kmsService,
+  projectEnvDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TExternalMigrationQueueFactoryDep) => {
+  const startImport = async (dto: {
+    actorEmail: string;
+    data: {
+      iv: string;
+      tag: string;
+      ciphertext: string;
+      algorithm: SecretEncryptionAlgo;
+      encoding: SecretKeyEncoding;
+    };
+  }) => {
+    await queueService.queue(
+      QueueName.ImportSecretsFromExternalSource,
+      QueueJobs.ImportSecretsFromExternalSource,
+      dto,
+      {
+        removeOnComplete: true,
+        removeOnFail: true
+      }
+    );
+  };
+
+  queueService.start(QueueName.ImportSecretsFromExternalSource, async (job) => {
+    try {
+      const { data, actorEmail } = job.data;
+
+      await smtpService.sendMail({
+        recipients: [actorEmail],
+        subjectLine: "Infisical import started",
+        substitutions: {
+          provider: ExternalPlatforms.EnvKey
+        },
+        template: SmtpTemplates.ExternalImportStarted
+      });
+
+      const decrypted = infisicalSymmetricDecrypt({
+        ciphertext: data.ciphertext,
+        iv: data.iv,
+        keyEncoding: data.encoding,
+        tag: data.tag
+      });
+
+      const decryptedJson = JSON.parse(decrypted) as TImportInfisicalDataCreate;
+
+      const { projectsNotImported } = await importDataIntoInfisicalFn({
+        input: decryptedJson,
+        projectDAL,
+        projectEnvDAL,
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
+        folderDAL,
+        kmsService,
+        projectService,
+        projectEnvService,
+        secretV2BridgeService
+      });
+
+      if (projectsNotImported.length) {
+        logger.info(
+          {
+            actorEmail,
+            actorOrgId: decryptedJson.actorOrgId,
+            projectsNotImported
+          },
+          "One or more projects were not imported during import from external source"
+        );
+      }
+
+      await smtpService.sendMail({
+        recipients: [actorEmail],
+        subjectLine: "Infisical import successful",
+        substitutions: {
+          provider: ExternalPlatforms.EnvKey
+        },
+        template: SmtpTemplates.ExternalImportSuccessful
+      });
+    } catch (err) {
+      await smtpService.sendMail({
+        recipients: [job.data.actorEmail],
+        subjectLine: "Infisical import failed",
+        substitutions: {
+          provider: ExternalPlatforms.EnvKey,
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
+          error: (err as any)?.message || "Unknown error"
+        },
+        template: SmtpTemplates.ExternalImportFailed
+      });
+
+      logger.error(err, "Failed to import data from external source");
+    }
+  });
+  return {
+    startImport
+  };
+};

backend/src/services/external-migration/external-migration-service.ts

@@ -1,30 +1,25 @@
 import { OrgMembershipRole } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
+import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
 import { ForbiddenRequestError } from "@app/lib/errors";
 
-import { TOrgServiceFactory } from "../org/org-service";
-import { TProjectServiceFactory } from "../project/project-service";
-import { TProjectEnvServiceFactory } from "../project-env/project-env-service";
-import { TSecretServiceFactory } from "../secret/secret-service";
-import { decryptEnvKeyDataFn, importDataIntoInfisicalFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import { TUserDALFactory } from "../user/user-dal";
+import { decryptEnvKeyDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import { TExternalMigrationQueueFactory } from "./external-migration-queue";
 import { TImportEnvKeyDataCreate } from "./external-migration-types";
 
 type TExternalMigrationServiceFactoryDep = {
-  projectService: TProjectServiceFactory;
-  orgService: TOrgServiceFactory;
-  projectEnvService: TProjectEnvServiceFactory;
-  secretService: TSecretServiceFactory;
   permissionService: TPermissionServiceFactory;
+  externalMigrationQueue: TExternalMigrationQueueFactory;
+  userDAL: Pick<TUserDALFactory, "findById">;
 };
 
 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
 
 export const externalMigrationServiceFactory = ({
-  projectService,
-  orgService,
-  projectEnvService,
   permissionService,
-  secretService
+  externalMigrationQueue,
+  userDAL
 }: TExternalMigrationServiceFactoryDep) => {
   const importEnvKeyData = async ({
     decryptionKey,
@@ -41,21 +36,28 @@ export const externalMigrationServiceFactory = ({
       actorAuthMethod,
       actorOrgId
     );
-
     if (membership.role !== OrgMembershipRole.Admin) {
       throw new ForbiddenRequestError({ message: "Only admins can import data" });
     }
 
+    const user = await userDAL.findById(actorId);
     const json = await decryptEnvKeyDataFn(decryptionKey, encryptedJson);
     const envKeyData = await parseEnvKeyDataFn(json);
-    const response = await importDataIntoInfisicalFn({
-      input: { data: envKeyData, actor, actorId, actorOrgId, actorAuthMethod },
-      projectService,
-      orgService,
-      projectEnvService,
-      secretService
+
+    const stringifiedJson = JSON.stringify({
+      data: envKeyData,
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod
+    });
+
+    const encrypted = infisicalSymmetricEncypt(stringifiedJson);
+
+    await externalMigrationQueue.startImport({
+      actorEmail: user.email!,
+      data: encrypted
     });
-    return response;
   };
 
   return {

backend/src/services/external-migration/external-migration-types.ts

@@ -1,26 +1,9 @@
 import { ActorAuthMethod, ActorType } from "../auth/auth-type";
 
 export type InfisicalImportData = {
-  projects: Map<string, { name: string; id: string }>;
-
-  environments?: Map<
-    string,
-    {
-      name: string;
-      id: string;
-      projectId: string;
-    }
-  >;
-
-  secrets?: Map<
-    string,
-    {
-      name: string;
-      id: string;
-      environmentId: string;
-      value: string;
-    }
-  >;
+  projects: Array<{ name: string; id: string }>;
+  environments: Array<{ name: string; id: string; projectId: string }>;
+  secrets: Array<{ name: string; id: string; environmentId: string; value: string }>;
 };
 
 export type TImportEnvKeyDataCreate = {
@@ -104,3 +87,7 @@ export type TEnvKeyExportJSON = {
     }
   >;
 };
+
+export enum ExternalPlatforms {
+  EnvKey = "EnvKey"
+}

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -9,6 +9,7 @@
 
 import {
   CreateSecretCommand,
+  DeleteSecretCommand,
   DescribeSecretCommand,
   GetSecretValueCommand,
   ResourceNotFoundException,
@@ -727,7 +728,7 @@ const syncSecretsAWSParameterStore = async ({
           awsParameterStoreSecretsObj[key].KeyId !== metadata.kmsKeyId;
 
         // we ensure that the KMS key configured in the integration is applied for ALL parameters on AWS
-        if (shouldUpdateKms || awsParameterStoreSecretsObj[key].Value !== secrets[key].value) {
+        if (secrets[key].value && (shouldUpdateKms || awsParameterStoreSecretsObj[key].Value !== secrets[key].value)) {
           await ssm
             .putParameter({
               Name: `${integration.path}${key}`,
@@ -788,7 +789,7 @@ const syncSecretsAWSParameterStore = async ({
         logger.info(
           `getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}]  [secretPath=${integration.secretPath}] [step=2]`
         );
-        if (!(key in secrets)) {
+        if (!(key in secrets) || !secrets[key].value) {
           logger.info(
             `getIntegrationSecrets: inside of shouldDisableDelete AWS SSM [projectId=${projectId}] [environment=${integration.environment.slug}]  [secretPath=${integration.secretPath}] [step=3]`
           );
@@ -899,12 +900,21 @@ const syncSecretsAWSSecretManager = async ({
       }
 
       if (!isEqual(secretToCompare, secretValue)) {
-        await secretsManager.send(
-          new UpdateSecretCommand({
-            SecretId: secretId,
-            SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue)
-          })
-        );
+        if (secretValue) {
+          await secretsManager.send(
+            new UpdateSecretCommand({
+              SecretId: secretId,
+              SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue)
+            })
+          );
+          // delete it
+        } else {
+          await secretsManager.send(
+            new DeleteSecretCommand({
+              SecretId: secretId
+            })
+          );
+        }
       }
 
       const secretAWSTag = metadata.secretAWSTag as { key: string; value: string }[] | undefined;
@@ -989,16 +999,21 @@ const syncSecretsAWSSecretManager = async ({
     } catch (err) {
       // case 1: when AWS manager can't find the specified secret
       if (err instanceof ResourceNotFoundException && secretsManager) {
-        await secretsManager.send(
-          new CreateSecretCommand({
-            Name: secretId,
-            SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue),
-            ...(metadata.kmsKeyId && { KmsKeyId: metadata.kmsKeyId }),
-            Tags: metadata.secretAWSTag
-              ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({ Key: tag.key, Value: tag.value }))
-              : []
-          })
-        );
+        if (secretValue) {
+          await secretsManager.send(
+            new CreateSecretCommand({
+              Name: secretId,
+              SecretString: typeof secretValue === "string" ? secretValue : JSON.stringify(secretValue),
+              ...(metadata.kmsKeyId && { KmsKeyId: metadata.kmsKeyId }),
+              Tags: metadata.secretAWSTag
+                ? metadata.secretAWSTag.map((tag: { key: string; value: string }) => ({
+                    Key: tag.key,
+                    Value: tag.value
+                  }))
+                : []
+            })
+          );
+        }
         // case 2: something unexpected went wrong, so we'll throw the error to reflect the error in the integration sync status
       } else {
         throw err;

backend/src/services/integration/integration-service.ts

@@ -120,7 +120,13 @@ export const integrationServiceFactory = ({
       secretPath,
       projectId: integrationAuth.projectId
     });
-    return { integration, integrationAuth };
+    return {
+      integration: {
+        ...integration,
+        environment: folder.environment
+      },
+      integrationAuth
+    };
   };
 
   const updateIntegration = async ({
@@ -183,7 +189,10 @@ export const integrationServiceFactory = ({
       projectId: folder.projectId
     });
 
-    return updatedIntegration;
+    return {
+      ...updatedIntegration,
+      environment: folder.environment
+    };
   };
 
   const getIntegration = async ({ id, actor, actorAuthMethod, actorId, actorOrgId }: TGetIntegrationDTO) => {

backend/src/services/kms/kms-service.ts

@@ -160,8 +160,8 @@ export const kmsServiceFactory = ({
    * In mean time the rest of the request will wait until creation is finished followed by getting the created on
    * In real time this would be milliseconds
    */
-  const getOrgKmsKeyId = async (orgId: string) => {
-    let org = await orgDAL.findById(orgId);
+  const getOrgKmsKeyId = async (orgId: string, trx?: Knex) => {
+    let org = await orgDAL.findById(orgId, trx);
 
     if (!org) {
       throw new NotFoundError({ message: "Org not found" });
@@ -180,9 +180,9 @@ export const kmsServiceFactory = ({
             waitingCb: () => logger.info("KMS. Waiting for org key to be created")
           });
 
-          org = await orgDAL.findById(orgId);
+          org = await orgDAL.findById(orgId, trx);
         } else {
-          const keyId = await orgDAL.transaction(async (tx) => {
+          const keyId = await (trx || orgDAL).transaction(async (tx) => {
             org = await orgDAL.findById(orgId, tx);
             if (org.kmsDefaultKeyId) {
               return org.kmsDefaultKeyId;
@@ -240,11 +240,12 @@ export const kmsServiceFactory = ({
 
   const decryptWithKmsKey = async ({
     kmsId,
-    depth = 0
-  }: Omit<TDecryptWithKmsDTO, "cipherTextBlob"> & { depth?: number }) => {
+    depth = 0,
+    tx
+  }: Omit<TDecryptWithKmsDTO, "cipherTextBlob"> & { depth?: number; tx?: Knex }) => {
     if (depth > 2) throw new BadRequestError({ message: "KMS depth max limit" });
 
-    const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
+    const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx);
     if (!kmsDoc) {
       throw new NotFoundError({ message: "KMS ID not found" });
     }
@@ -261,7 +262,8 @@ export const kmsServiceFactory = ({
       // we put a limit of depth to avoid too many cycles
       const orgKmsDecryptor = await decryptWithKmsKey({
         kmsId: kmsDoc.orgKms.id,
-        depth: depth + 1
+        depth: depth + 1,
+        tx
       });
 
       const orgKmsDataKey = await orgKmsDecryptor({
@@ -375,9 +377,9 @@ export const kmsServiceFactory = ({
     };
   };
 
-  const $getOrgKmsDataKey = async (orgId: string) => {
-    const kmsKeyId = await getOrgKmsKeyId(orgId);
-    let org = await orgDAL.findById(orgId);
+  const $getOrgKmsDataKey = async (orgId: string, trx?: Knex) => {
+    const kmsKeyId = await getOrgKmsKeyId(orgId, trx);
+    let org = await orgDAL.findById(orgId, trx);
 
     if (!org) {
       throw new NotFoundError({ message: "Org not found" });
@@ -396,9 +398,9 @@ export const kmsServiceFactory = ({
             waitingCb: () => logger.info("KMS. Waiting for org data key to be created")
           });
 
-          org = await orgDAL.findById(orgId);
+          org = await orgDAL.findById(orgId, trx);
         } else {
-          const orgDataKey = await orgDAL.transaction(async (tx) => {
+          const orgDataKey = await (trx || orgDAL).transaction(async (tx) => {
             org = await orgDAL.findById(orgId, tx);
             if (org.kmsEncryptedDataKey) {
               return;
@@ -455,8 +457,8 @@ export const kmsServiceFactory = ({
     });
   };
 
-  const getProjectSecretManagerKmsKeyId = async (projectId: string) => {
-    let project = await projectDAL.findById(projectId);
+  const getProjectSecretManagerKmsKeyId = async (projectId: string, trx?: Knex) => {
+    let project = await projectDAL.findById(projectId, trx);
     if (!project) {
       throw new NotFoundError({ message: "Project not found" });
     }
@@ -477,7 +479,7 @@ export const kmsServiceFactory = ({
 
           project = await projectDAL.findById(projectId);
         } else {
-          const kmsKeyId = await projectDAL.transaction(async (tx) => {
+          const kmsKeyId = await (trx || projectDAL).transaction(async (tx) => {
             project = await projectDAL.findById(projectId, tx);
             if (project.kmsSecretManagerKeyId) {
               return project.kmsSecretManagerKeyId;
@@ -520,9 +522,9 @@ export const kmsServiceFactory = ({
     return project.kmsSecretManagerKeyId;
   };
 
-  const $getProjectSecretManagerKmsDataKey = async (projectId: string) => {
-    const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId);
-    let project = await projectDAL.findById(projectId);
+  const $getProjectSecretManagerKmsDataKey = async (projectId: string, trx?: Knex) => {
+    const kmsKeyId = await getProjectSecretManagerKmsKeyId(projectId, trx);
+    let project = await projectDAL.findById(projectId, trx);
 
     if (!project.kmsSecretManagerEncryptedDataKey) {
       const lock = await keyStore
@@ -538,18 +540,21 @@ export const kmsServiceFactory = ({
             delay: 500
           });
 
-          project = await projectDAL.findById(projectId);
+          project = await projectDAL.findById(projectId, trx);
         } else {
-          const projectDataKey = await projectDAL.transaction(async (tx) => {
+          const projectDataKey = await (trx || projectDAL).transaction(async (tx) => {
             project = await projectDAL.findById(projectId, tx);
             if (project.kmsSecretManagerEncryptedDataKey) {
               return;
             }
 
             const dataKey = randomSecureBytes();
-            const kmsEncryptor = await encryptWithKmsKey({
-              kmsId: kmsKeyId
-            });
+            const kmsEncryptor = await encryptWithKmsKey(
+              {
+                kmsId: kmsKeyId
+              },
+              tx
+            );
 
             const { cipherTextBlob } = await kmsEncryptor({
               plainText: dataKey
@@ -585,7 +590,8 @@ export const kmsServiceFactory = ({
     }
 
     const kmsDecryptor = await decryptWithKmsKey({
-      kmsId: kmsKeyId
+      kmsId: kmsKeyId,
+      tx: trx
     });
 
     return kmsDecryptor({
@@ -593,13 +599,13 @@ export const kmsServiceFactory = ({
     });
   };
 
-  const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO) => {
+  const $getDataKey = async (dto: TEncryptWithKmsDataKeyDTO, trx?: Knex) => {
     switch (dto.type) {
       case KmsDataKey.SecretManager: {
-        return $getProjectSecretManagerKmsDataKey(dto.projectId);
+        return $getProjectSecretManagerKmsDataKey(dto.projectId, trx);
       }
       default: {
-        return $getOrgKmsDataKey(dto.orgId);
+        return $getOrgKmsDataKey(dto.orgId, trx);
       }
     }
   };
@@ -607,8 +613,9 @@ export const kmsServiceFactory = ({
   // by keeping the decrypted data key in inner scope
   // none of the entities outside can interact directly or expose the data key
   // NOTICE: If changing here update migrations/utils/kms
-  const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO) => {
-    const dataKey = await $getDataKey(encryptionContext);
+  const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO, trx?: Knex) => {
+    const dataKey = await $getDataKey(encryptionContext, trx);
+
     const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
 
     return {

backend/src/services/org/org-dal.ts

@@ -108,7 +108,9 @@ export const orgDALFactory = (db: TDbClient) => {
           db.ref("id").withSchema(TableName.Users).as("userId"),
           db.ref("publicKey").withSchema(TableName.UserEncryptionKey)
         )
-        .where({ isGhost: false }); // MAKE SURE USER IS NOT A GHOST USER
+        .where({ isGhost: false }) // MAKE SURE USER IS NOT A GHOST USER
+        .orderBy("firstName")
+        .orderBy("lastName");
 
       return members.map(({ email, isEmailVerified, username, firstName, lastName, userId, publicKey, ...data }) => ({
         ...data,
@@ -370,6 +372,7 @@ export const orgDALFactory = (db: TDbClient) => {
           db.ref("firstName").withSchema(TableName.Users),
           db.ref("lastName").withSchema(TableName.Users),
           db.ref("scimEnabled").withSchema(TableName.Organization),
+          db.ref("defaultMembershipRole").withSchema(TableName.Organization),
           db.ref("externalId").withSchema(TableName.UserAliases)
         )
         .where({ isGhost: false });

backend/src/services/org/org-role-fns.ts

@@ -0,0 +1,54 @@
+import { OrgMembershipRole } from "@app/db/schemas";
+import { TFeatureSet } from "@app/ee/services/license/license-types";
+import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { TOrgRoleDALFactory } from "@app/services/org/org-role-dal";
+
+const RESERVED_ORG_ROLE_SLUGS = Object.values(OrgMembershipRole).filter((role) => role !== "custom");
+
+// this is only for updating an org
+export const getDefaultOrgMembershipRoleForUpdateOrg = async ({
+  membershipRoleSlug,
+  orgRoleDAL,
+  plan,
+  orgId
+}: {
+  orgId: string;
+  membershipRoleSlug: string;
+  orgRoleDAL: TOrgRoleDALFactory;
+  plan: TFeatureSet;
+}) => {
+  const isCustomRole = !RESERVED_ORG_ROLE_SLUGS.includes(membershipRoleSlug as OrgMembershipRole);
+
+  if (isCustomRole) {
+    if (!plan?.rbac)
+      throw new BadRequestError({
+        message:
+          "Failed to set custom default role due to plan RBAC restriction. Upgrade plan to set custom default org membership role."
+      });
+
+    const customRole = await orgRoleDAL.findOne({ slug: membershipRoleSlug, orgId });
+    if (!customRole) throw new NotFoundError({ name: "UpdateOrg", message: "Organization role not found" });
+
+    // use ID for default role
+    return customRole.id;
+  }
+
+  // not custom, use reserved slug
+  return membershipRoleSlug;
+};
+
+// this is only for creating an org membership
+export const getDefaultOrgMembershipRole = async (
+  defaultOrgMembershipRole: string // can either be ID or reserved slug
+) => {
+  const isCustomRole = !RESERVED_ORG_ROLE_SLUGS.includes(defaultOrgMembershipRole as OrgMembershipRole);
+
+  if (isCustomRole)
+    return {
+      roleId: defaultOrgMembershipRole,
+      role: OrgMembershipRole.Custom
+    };
+
+  // will be reserved slug
+  return { roleId: undefined, role: defaultOrgMembershipRole as OrgMembershipRole };
+};

backend/src/services/org/org-role-service.ts

@@ -11,6 +11,7 @@ import {
 } from "@app/ee/services/permission/org-permission";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { TOrgDALFactory } from "@app/services/org/org-dal";
 
 import { ActorAuthMethod } from "../auth/auth-type";
 import { TOrgRoleDALFactory } from "./org-role-dal";
@@ -18,11 +19,12 @@ import { TOrgRoleDALFactory } from "./org-role-dal";
 type TOrgRoleServiceFactoryDep = {
   orgRoleDAL: TOrgRoleDALFactory;
   permissionService: TPermissionServiceFactory;
+  orgDAL: TOrgDALFactory;
 };
 
 export type TOrgRoleServiceFactory = ReturnType<typeof orgRoleServiceFactory>;
 
-export const orgRoleServiceFactory = ({ orgRoleDAL, permissionService }: TOrgRoleServiceFactoryDep) => {
+export const orgRoleServiceFactory = ({ orgRoleDAL, orgDAL, permissionService }: TOrgRoleServiceFactoryDep) => {
   const createRole = async (
     userId: string,
     orgId: string,
@@ -129,6 +131,19 @@ export const orgRoleServiceFactory = ({ orgRoleDAL, permissionService }: TOrgRol
   ) => {
     const { permission } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId);
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Delete, OrgPermissionSubjects.Role);
+
+    const org = await orgDAL.findOrgById(orgId);
+
+    if (!org)
+      throw new NotFoundError({
+        message: "Failed to find organization"
+      });
+
+    if (org.defaultMembershipRole === roleId)
+      throw new BadRequestError({
+        message: "Cannot delete default org membership role. Please re-assign and try again."
+      });
+
     const [deletedRole] = await orgRoleDAL.delete({ id: roleId, orgId });
     if (!deletedRole) throw new NotFoundError({ message: "Organization role not found", name: "Update role" });
 

backend/src/services/org/org-service.ts

@@ -32,6 +32,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro
 import { groupBy } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { isDisposableEmail } from "@app/lib/validator";
+import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns";
 import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
 import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal";
 
@@ -40,8 +41,9 @@ import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service";
 import { TokenType } from "../auth-token/auth-token-types";
 import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal";
 import { TProjectDALFactory } from "../project/project-dal";
-import { assignWorkspaceKeysToMembers } from "../project/project-fns";
+import { assignWorkspaceKeysToMembers, createProjectKey } from "../project/project-fns";
 import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
+import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
 import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
 import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
@@ -79,7 +81,7 @@ type TOrgServiceFactoryDep = {
     TProjectMembershipDALFactory,
     "findProjectMembershipsByUserId" | "delete" | "create" | "find" | "insertMany" | "transaction"
   >;
-  projectKeyDAL: Pick<TProjectKeyDALFactory, "find" | "delete" | "insertMany" | "findLatestProjectKey">;
+  projectKeyDAL: Pick<TProjectKeyDALFactory, "find" | "delete" | "insertMany" | "findLatestProjectKey" | "create">;
   orgMembershipDAL: Pick<TOrgMembershipDALFactory, "findOrgMembershipById" | "findOne" | "findById">;
   incidentContactDAL: TIncidentContactsDALFactory;
   samlConfigDAL: Pick<TSamlConfigDALFactory, "findOne" | "findEnforceableSamlCfg">;
@@ -93,8 +95,9 @@ type TOrgServiceFactoryDep = {
   >;
   projectUserAdditionalPrivilegeDAL: Pick<TProjectUserAdditionalPrivilegeDALFactory, "delete">;
   projectRoleDAL: Pick<TProjectRoleDALFactory, "find">;
-  projectBotDAL: Pick<TProjectBotDALFactory, "findOne">;
-  projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "insertMany">;
+  projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "updateById">;
+  projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "insertMany" | "create">;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
 };
 
 export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
@@ -121,7 +124,8 @@ export const orgServiceFactory = ({
   oidcConfigDAL,
   projectBotDAL,
   projectUserMembershipRoleDAL,
-  identityMetadataDAL
+  identityMetadataDAL,
+  projectBotService
 }: TOrgServiceFactoryDep) => {
   /*
    * Get organization details by the organization id
@@ -264,7 +268,7 @@ export const orgServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     orgId,
-    data: { name, slug, authEnforced, scimEnabled }
+    data: { name, slug, authEnforced, scimEnabled, defaultMembershipRoleSlug }
   }: TUpdateOrgDTO) => {
     const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings);
@@ -298,11 +302,22 @@ export const orgServiceFactory = ({
         });
     }
 
+    let defaultMembershipRole: string | undefined;
+    if (defaultMembershipRoleSlug) {
+      defaultMembershipRole = await getDefaultOrgMembershipRoleForUpdateOrg({
+        membershipRoleSlug: defaultMembershipRoleSlug,
+        orgId,
+        orgRoleDAL,
+        plan
+      });
+    }
+
     const org = await orgDAL.updateById(orgId, {
       name,
       slug: slug ? slugify(slug) : undefined,
       authEnforced,
-      scimEnabled
+      scimEnabled,
+      defaultMembershipRole
     });
     if (!org) throw new NotFoundError({ message: "Organization not found" });
     return org;
@@ -706,20 +721,67 @@ export const orgServiceFactory = ({
 
         const customRolesGroupBySlug = groupBy(customRoles, ({ slug }) => slug);
 
-        const ghostUser = await projectDAL.findProjectGhostUser(projectId, tx);
-        if (!ghostUser) {
-          throw new NotFoundError({
-            name: "InviteUser",
-            message: "Failed to find project owner"
-          });
-        }
+        // this will auto generate bot
+        const { botKey, bot: autoGeneratedBot } = await projectBotService.getBotKey(projectId, true);
 
-        const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.id, projectId, tx);
-        if (!ghostUserLatestKey) {
-          throw new NotFoundError({
-            name: "InviteUser",
-            message: "Failed to find project owner's latest key"
+        const ghostUser = await projectDAL.findProjectGhostUser(projectId, tx);
+        let ghostUserId = ghostUser?.id;
+
+        //  backfill missing ghost user
+        if (!ghostUserId) {
+          const newGhostUser = await addGhostUser(project.orgId, tx);
+          const projectMembership = await projectMembershipDAL.create(
+            {
+              userId: newGhostUser.user.id,
+              projectId: project.id
+            },
+            tx
+          );
+          await projectUserMembershipRoleDAL.create(
+            { projectMembershipId: projectMembership.id, role: ProjectMembershipRole.Admin },
+            tx
+          );
+
+          const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({
+            publicKey: newGhostUser.keys.publicKey,
+            privateKey: newGhostUser.keys.plainPrivateKey,
+            plainProjectKey: botKey
           });
+
+          // 4. Save the project key for the ghost user.
+          await projectKeyDAL.create(
+            {
+              projectId: project.id,
+              receiverId: newGhostUser.user.id,
+              encryptedKey: encryptedProjectKey,
+              nonce: encryptedProjectKeyIv,
+              senderId: newGhostUser.user.id
+            },
+            tx
+          );
+
+          const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(
+            newGhostUser.keys.plainPrivateKey
+          );
+          if (autoGeneratedBot) {
+            await projectBotDAL.updateById(
+              autoGeneratedBot.id,
+              {
+                tag,
+                iv,
+                encryptedProjectKey,
+                encryptedProjectKeyNonce: encryptedProjectKeyIv,
+                encryptedPrivateKey: ciphertext,
+                isActive: true,
+                publicKey: newGhostUser.keys.publicKey,
+                senderId: newGhostUser.user.id,
+                algorithm,
+                keyEncoding: encoding
+              },
+              tx
+            );
+          }
+          ghostUserId = newGhostUser.user.id;
         }
 
         const bot = await projectBotDAL.findOne({ projectId }, tx);
@@ -730,6 +792,14 @@ export const orgServiceFactory = ({
           });
         }
 
+        const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUserId, projectId, tx);
+        if (!ghostUserLatestKey) {
+          throw new NotFoundError({
+            name: "InviteUser",
+            message: "Failed to find project owner's latest key"
+          });
+        }
+
         const botPrivateKey = infisicalSymmetricDecrypt({
           keyEncoding: bot.keyEncoding as SecretKeyEncoding,
           iv: bot.iv,
@@ -773,7 +843,7 @@ export const orgServiceFactory = ({
           newWsMembers.map((el) => ({
             encryptedKey: el.workspaceEncryptedKey,
             nonce: el.workspaceEncryptedNonce,
-            senderId: ghostUser.id,
+            senderId: ghostUserId,
             receiverId: el.orgMembershipId,
             projectId
           })),

backend/src/services/org/org-types.ts

@@ -26,18 +26,13 @@ export type TDeleteOrgMembershipDTO = {
 };
 
 export type TInviteUserToOrgDTO = {
-  actorId: string;
-  actor: ActorType;
-  orgId: string;
-  actorOrgId: string | undefined;
-  actorAuthMethod: ActorAuthMethod;
   inviteeEmails: string[];
   organizationRoleSlug: string;
   projects?: {
     id: string;
     projectRoleSlug?: string[];
   }[];
-};
+} & TOrgPermission;
 
 export type TVerifyUserToOrgDTO = {
   email: string;
@@ -63,7 +58,13 @@ export type TFindAllWorkspacesDTO = {
 };
 
 export type TUpdateOrgDTO = {
-  data: Partial<{ name: string; slug: string; authEnforced: boolean; scimEnabled: boolean }>;
+  data: Partial<{
+    name: string;
+    slug: string;
+    authEnforced: boolean;
+    scimEnabled: boolean;
+    defaultMembershipRoleSlug: string;
+  }>;
 } & TOrgPermission;
 
 export type TGetOrgGroupsDTO = TOrgPermission;

backend/src/services/project-bot/project-bot-fns.ts

@@ -24,14 +24,14 @@ export const getBotKeyFnFactory = (
   projectBotDAL: TProjectBotDALFactory,
   projectDAL: Pick<TProjectDALFactory, "findById">
 ) => {
-  const getBotKeyFn = async (projectId: string) => {
+  const getBotKeyFn = async (projectId: string, shouldGetBotKey?: boolean) => {
     const project = await projectDAL.findById(projectId);
     if (!project)
       throw new NotFoundError({
         message: "Project not found during bot lookup. Are you sure you are using the correct project ID?"
       });
 
-    if (project.version === 3) {
+    if (project.version === 3 && !shouldGetBotKey) {
       return { project, shouldUseSecretV2Bridge: true };
     }
 
@@ -65,8 +65,9 @@ export const getBotKeyFnFactory = (
       const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(botKey.privateKey);
       const encryptedWorkspaceKey = encryptAsymmetric(workspaceKey, botKey.publicKey, userPrivateKey);
 
+      let botId;
       if (!bot) {
-        await projectBotDAL.create({
+        const newBot = await projectBotDAL.create({
           name: "Infisical Bot (Ghost)",
           projectId,
           isActive: true,
@@ -80,8 +81,9 @@ export const getBotKeyFnFactory = (
           encryptedProjectKeyNonce: encryptedWorkspaceKey.nonce,
           senderId: projectV1Keys.userId
         });
+        botId = newBot.id;
       } else {
-        await projectBotDAL.updateById(bot.id, {
+        const updatedBot = await projectBotDAL.updateById(bot.id, {
           isActive: true,
           tag,
           iv,
@@ -93,8 +95,10 @@ export const getBotKeyFnFactory = (
           encryptedProjectKeyNonce: encryptedWorkspaceKey.nonce,
           senderId: projectV1Keys.userId
         });
+        botId = updatedBot.id;
       }
-      return { botKey: workspaceKey, project, shouldUseSecretV2Bridge: false };
+
+      return { botKey: workspaceKey, project, shouldUseSecretV2Bridge: false, bot: { id: botId } };
     }
 
     const botPrivateKey = getBotPrivateKey({ bot });
@@ -104,7 +108,7 @@ export const getBotKeyFnFactory = (
       nonce: bot.encryptedProjectKeyNonce,
       publicKey: bot.sender.publicKey
     });
-    return { botKey, project, shouldUseSecretV2Bridge: false };
+    return { botKey, project, shouldUseSecretV2Bridge: false, bot: { id: bot.id } };
   };
 
   return getBotKeyFn;

backend/src/services/project-bot/project-bot-service.ts

@@ -27,8 +27,8 @@ export const projectBotServiceFactory = ({
 }: TProjectBotServiceFactoryDep) => {
   const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL);
 
-  const getBotKey = async (projectId: string) => {
-    return getBotKeyFn(projectId);
+  const getBotKey = async (projectId: string, shouldGetBotKey?: boolean) => {
+    return getBotKeyFn(projectId, shouldGetBotKey);
   };
 
   const findBotByProjectId = async ({

backend/src/services/project/project-service.ts

@@ -147,6 +147,7 @@ export const projectServiceFactory = ({
     workspaceName,
     slug: projectSlug,
     kmsKeyId,
+    tx: trx,
     createDefaultEnvs = true
   }: TCreateProjectDTO) => {
     const organization = await orgDAL.findOne({ id: actorOrgId });
@@ -169,7 +170,7 @@ export const projectServiceFactory = ({
       });
     }
 
-    const results = await projectDAL.transaction(async (tx) => {
+    const results = await (trx || projectDAL).transaction(async (tx) => {
       const ghostUser = await orgService.addGhostUser(organization.id, tx);
 
       if (kmsKeyId) {

backend/src/services/project/project-types.ts

@@ -1,3 +1,5 @@
+import { Knex } from "knex";
+
 import { TProjectKeys } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
 
@@ -30,6 +32,7 @@ export type TCreateProjectDTO = {
   slug?: string;
   kmsKeyId?: string;
   createDefaultEnvs?: boolean;
+  tx?: Knex;
 };
 
 export type TDeleteProjectBySlugDTO = {

backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts

@@ -82,7 +82,10 @@ export const fnSecretBulkInsert = async ({
     })
   );
 
-  const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId })));
+  const newSecrets = await secretDAL.insertMany(
+    sanitizedInputSecrets.map((el) => ({ ...el, folderId })),
+    tx
+  );
   const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key);
   const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) =>
     secretTags.map((tag) => ({

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -193,14 +193,16 @@ export const secretV2BridgeServiceFactory = ({
       })
     );
 
-    await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({
-      secretPath,
-      actorId,
-      actor,
-      projectId,
-      environmentSlug: folder.environment.slug
-    });
+    if (inputSecret.type === SecretType.Shared) {
+      await snapshotService.performSnapshot(folderId);
+      await secretQueueService.syncSecrets({
+        secretPath,
+        actorId,
+        actor,
+        projectId,
+        environmentSlug: folder.environment.slug
+      });
+    }
 
     return reshapeBridgeSecret(projectId, environment, secretPath, {
       ...secret[0],
@@ -349,14 +351,17 @@ export const secretV2BridgeServiceFactory = ({
       projectId
     });
 
-    await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({
-      actor,
-      actorId,
-      secretPath,
-      projectId,
-      environmentSlug: folder.environment.slug
-    });
+    if (inputSecret.type === SecretType.Shared) {
+      await snapshotService.performSnapshot(folderId);
+      await secretQueueService.syncSecrets({
+        secretPath,
+        actorId,
+        actor,
+        projectId,
+        environmentSlug: folder.environment.slug
+      });
+    }
+
     return reshapeBridgeSecret(projectId, environment, secretPath, {
       ...updatedSecret[0],
       value: inputSecret.secretValue || "",
@@ -427,14 +432,16 @@ export const secretV2BridgeServiceFactory = ({
       })
     );
 
-    await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({
-      actor,
-      actorId,
-      secretPath,
-      projectId,
-      environmentSlug: folder.environment.slug
-    });
+    if (inputSecret.type === SecretType.Shared) {
+      await snapshotService.performSnapshot(folderId);
+      await secretQueueService.syncSecrets({
+        secretPath,
+        actorId,
+        actor,
+        projectId,
+        environmentSlug: folder.environment.slug
+      });
+    }
 
     const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.SecretManager,

backend/src/services/secret/secret-queue.ts

@@ -17,6 +17,7 @@ import { TSnapshotSecretV2DALFactory } from "@app/ee/services/secret-snapshot/sn
 import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
 import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
 import { getTimeDifferenceInSeconds, groupBy, isSamePath, unique } from "@app/lib/fn";
@@ -37,10 +38,14 @@ import { syncIntegrationSecrets } from "../integration-auth/integration-sync-sec
 import { TKmsServiceFactory } from "../kms/kms-service";
 import { KmsDataKey } from "../kms/kms-types";
 import { TOrgDALFactory } from "../org/org-dal";
+import { TOrgServiceFactory } from "../org/org-service";
 import { TProjectDALFactory } from "../project/project-dal";
+import { createProjectKey } from "../project/project-fns";
 import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
+import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
 import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
+import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
@@ -77,7 +82,8 @@ type TSecretQueueFactoryDep = {
   projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "find">;
   projectDAL: TProjectDALFactory;
   projectBotDAL: TProjectBotDALFactory;
-  projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
+  projectKeyDAL: Pick<TProjectKeyDALFactory, "create">;
+  projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers" | "create">;
   smtpService: TSmtpService;
   orgDAL: Pick<TOrgDALFactory, "findOrgByProjectId">;
   secretVersionDAL: TSecretVersionDALFactory;
@@ -85,7 +91,7 @@ type TSecretQueueFactoryDep = {
   secretTagDAL: TSecretTagDALFactory;
   userDAL: Pick<TUserDALFactory, "findById">;
   secretVersionTagDAL: TSecretVersionTagDALFactory;
-  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+  kmsService: TKmsServiceFactory;
   secretV2BridgeDAL: TSecretV2BridgeDALFactory;
   secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "batchInsert" | "insertMany" | "findLatestVersionMany">;
   secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "batchInsert">;
@@ -95,6 +101,8 @@ type TSecretQueueFactoryDep = {
   snapshotSecretV2BridgeDAL: Pick<TSnapshotSecretV2DALFactory, "insertMany" | "batchInsert">;
   keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
   auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
+  orgService: Pick<TOrgServiceFactory, "addGhostUser">;
+  projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "create">;
 };
 
 export type TGetSecrets = {
@@ -111,6 +119,8 @@ type TIntegrationSecret = Record<
   string,
   { value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined }
 >;
+
+// TODO(akhilmhdh): split this into multiple queue
 export const secretQueueFactory = ({
   queueService,
   integrationDAL,
@@ -141,7 +151,10 @@ export const secretQueueFactory = ({
   snapshotSecretV2BridgeDAL,
   secretApprovalRequestDAL,
   keyStore,
-  auditLogService
+  auditLogService,
+  orgService,
+  projectUserMembershipRoleDAL,
+  projectKeyDAL
 }: TSecretQueueFactoryDep) => {
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
     const appCfg = getConfig();
@@ -1028,11 +1041,13 @@ export const secretQueueFactory = ({
     const {
       botKey,
       shouldUseSecretV2Bridge: isProjectUpgradedToV3,
-      project
+      project,
+      bot
     } = await projectBotService.getBotKey(projectId);
     if (isProjectUpgradedToV3 || project.upgradeStatus === ProjectUpgradeStatus.InProgress) {
       return;
     }
+
     if (!botKey) throw new NotFoundError({ message: "Project bot not found" });
     await projectDAL.updateById(projectId, { upgradeStatus: ProjectUpgradeStatus.InProgress });
 
@@ -1044,6 +1059,57 @@ export const secretQueueFactory = ({
     const folders = await folderDAL.findByProjectId(projectId);
     // except secret version and snapshot migrate rest of everything first in a transaction
     await secretDAL.transaction(async (tx) => {
+      // if project v1 create the project ghost user
+      if (project.version === ProjectVersion.V1) {
+        const ghostUser = await orgService.addGhostUser(project.orgId, tx);
+        const projectMembership = await projectMembershipDAL.create(
+          {
+            userId: ghostUser.user.id,
+            projectId: project.id
+          },
+          tx
+        );
+        await projectUserMembershipRoleDAL.create(
+          { projectMembershipId: projectMembership.id, role: ProjectMembershipRole.Admin },
+          tx
+        );
+
+        const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({
+          publicKey: ghostUser.keys.publicKey,
+          privateKey: ghostUser.keys.plainPrivateKey,
+          plainProjectKey: botKey
+        });
+
+        // 4. Save the project key for the ghost user.
+        await projectKeyDAL.create(
+          {
+            projectId: project.id,
+            receiverId: ghostUser.user.id,
+            encryptedKey: encryptedProjectKey,
+            nonce: encryptedProjectKeyIv,
+            senderId: ghostUser.user.id
+          },
+          tx
+        );
+        const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey);
+        await projectBotDAL.updateById(
+          bot.id,
+          {
+            tag,
+            iv,
+            encryptedProjectKey,
+            encryptedProjectKeyNonce: encryptedProjectKeyIv,
+            encryptedPrivateKey: ciphertext,
+            isActive: true,
+            publicKey: ghostUser.keys.publicKey,
+            senderId: ghostUser.user.id,
+            algorithm,
+            keyEncoding: encoding
+          },
+          tx
+        );
+      }
+
       for (const folder of folders) {
         const folderId = folder.id;
         /*

backend/src/services/secret/secret-service.ts

@@ -264,14 +264,16 @@ export const secretServiceFactory = ({
       })
     );
 
-    await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({
-      secretPath: path,
-      actorId,
-      actor,
-      projectId,
-      environmentSlug: folder.environment.slug
-    });
+    if (inputSecret.type === SecretType.Shared) {
+      await snapshotService.performSnapshot(folderId);
+      await secretQueueService.syncSecrets({
+        secretPath: path,
+        actorId,
+        actor,
+        projectId,
+        environmentSlug: folder.environment.slug
+      });
+    }
     return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
   };
 
@@ -399,14 +401,16 @@ export const secretServiceFactory = ({
       })
     );
 
-    await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({
-      actor,
-      actorId,
-      secretPath: path,
-      projectId,
-      environmentSlug: folder.environment.slug
-    });
+    if (inputSecret.type === SecretType.Shared) {
+      await snapshotService.performSnapshot(folderId);
+      await secretQueueService.syncSecrets({
+        secretPath: path,
+        actorId,
+        actor,
+        projectId,
+        environmentSlug: folder.environment.slug
+      });
+    }
     return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
   };
 
@@ -474,15 +478,17 @@ export const secretServiceFactory = ({
       })
     );
 
-    await snapshotService.performSnapshot(folderId);
-    await secretQueueService.syncSecrets({
-      actor,
-      actorId,
-      secretPath: path,
-      projectId,
-      environmentSlug: folder.environment.slug
-    });
-    // TODO(akhilmhdh-pg): license check, posthog service and snapshot
+    if (inputSecret.type === SecretType.Shared) {
+      await snapshotService.performSnapshot(folderId);
+      await secretQueueService.syncSecrets({
+        secretPath: path,
+        actorId,
+        actor,
+        projectId,
+        environmentSlug: folder.environment.slug
+      });
+    }
+
     return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
   };
 

backend/src/services/smtp/smtp-service.ts

@@ -34,7 +34,10 @@ export enum SmtpTemplates {
   WorkspaceInvite = "workspaceInvitation.handlebars",
   ScimUserProvisioned = "scimUserProvisioned.handlebars",
   PkiExpirationAlert = "pkiExpirationAlert.handlebars",
-  IntegrationSyncFailed = "integrationSyncFailed.handlebars"
+  IntegrationSyncFailed = "integrationSyncFailed.handlebars",
+  ExternalImportSuccessful = "externalImportSuccessful.handlebars",
+  ExternalImportFailed = "externalImportFailed.handlebars",
+  ExternalImportStarted = "externalImportStarted.handlebars"
 }
 
 export enum SmtpHost {

backend/src/services/smtp/templates/externalImportFailed.handlebars

@@ -0,0 +1,21 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Import failed</title>
+  </head>
+
+  <body>
+    <h2>An import from {{provider}} to Infisical has failed</h2>
+    <p>An import from
+      {{provider}}
+      to Infisical has failed due to unforeseen circumstances. Please re-try your import, and if the issue persists, you
+      can contact the Infisical team at team@infisical.com.
+    </p>
+
+    <p>Error: {{error}}</p>
+
+  </body>
+
+</html>

backend/src/services/smtp/templates/externalImportStarted.handlebars

@@ -0,0 +1,17 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Import in progress</title>
+  </head>
+
+  <body>
+    <h2>An import from {{provider}} to Infisical is in progress</h2>
+    <p>An import from
+      {{provider}}
+      to Infisical is in progress. The import process may take up to 30 minutes, and you will receive once the import
+      has finished or if it fails.</p>
+  </body>
+
+</html>

backend/src/services/smtp/templates/externalImportSuccessful.handlebars

@@ -0,0 +1,14 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Import successful</title>
+  </head>
+
+  <body>
+    <h2>An import from {{provider}} to Infisical was successful</h2>
+    <p>An import from {{provider}} was successful. Your data is now available in Infisical.</p>
+  </body>
+
+</html>

cli/packages/api/api.go

@@ -415,6 +415,10 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques
 		req.SetQueryParam("recursive", "true")
 	}
 
+	if request.ExpandSecretReferences {
+		req.SetQueryParam("expandSecretReferences", "true")
+	}
+
 	response, err := req.Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL))
 
 	if err != nil {

cli/packages/api/model.go

@@ -569,12 +569,13 @@ type CreateDynamicSecretLeaseV1Response struct {
 }
 
 type GetRawSecretsV3Request struct {
-	Environment   string `json:"environment"`
-	WorkspaceId   string `json:"workspaceId"`
-	SecretPath    string `json:"secretPath"`
-	IncludeImport bool   `json:"include_imports"`
-	Recursive     bool   `json:"recursive"`
-	TagSlugs      string `json:"tagSlugs,omitempty"`
+	Environment            string `json:"environment"`
+	WorkspaceId            string `json:"workspaceId"`
+	SecretPath             string `json:"secretPath"`
+	IncludeImport          bool   `json:"include_imports"`
+	Recursive              bool   `json:"recursive"`
+	TagSlugs               string `json:"tagSlugs,omitempty"`
+	ExpandSecretReferences bool   `json:"expandSecretReferences,omitempty"`
 }
 
 type GetRawSecretsV3Response struct {
@@ -587,6 +588,7 @@ type GetRawSecretsV3Response struct {
 		SecretKey     string `json:"secretKey"`
 		SecretValue   string `json:"secretValue"`
 		SecretComment string `json:"secretComment"`
+		SecretPath    string `json:"secretPath"`
 	} `json:"secrets"`
 	Imports []ImportedRawSecretV3 `json:"imports"`
 	ETag    string
@@ -610,6 +612,7 @@ type GetRawSecretV3ByNameResponse struct {
 		SecretKey     string `json:"secretKey"`
 		SecretValue   string `json:"secretValue"`
 		SecretComment string `json:"secretComment"`
+		SecretPath    string `json:"secretPath"`
 	} `json:"secret"`
 	ETag string
 }

cli/packages/cmd/agent.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -311,9 +312,34 @@ func ParseAgentConfig(configFile []byte) (*Config, error) {
 	return config, nil
 }
 
-func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) {
-	return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) {
-		res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, false, "")
+type secretArguments struct {
+	IsRecursive                  bool  `json:"recursive"`
+	ShouldExpandSecretReferences *bool `json:"expandSecretReferences,omitempty"`
+}
+
+func (s *secretArguments) SetDefaults() {
+	if s.ShouldExpandSecretReferences == nil {
+		var bool = true
+		s.ShouldExpandSecretReferences = &bool
+	}
+}
+
+func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string, ...string) ([]models.SingleEnvironmentVariable, error) {
+	// ...string is because golang doesn't have optional arguments.
+	// thus we make it slice and pick it only first element
+	return func(projectID, envSlug, secretPath string, args ...string) ([]models.SingleEnvironmentVariable, error) {
+		var parsedArguments secretArguments
+		// to make it optional
+		if len(args) > 0 {
+			err := json.Unmarshal([]byte(args[0]), &parsedArguments)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		parsedArguments.SetDefaults()
+
+		res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, parsedArguments.IsRecursive, "", *parsedArguments.ShouldExpandSecretReferences)
 		if err != nil {
 			return nil, err
 		}
@@ -322,9 +348,7 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag
 			*currentEtag = res.Etag
 		}
 
-		expandedSecrets := util.ExpandSecrets(res.Secrets, models.ExpandSecretsAuthentication{UniversalAuthAccessToken: accessToken}, "")
-
-		return expandedSecrets, nil
+		return res.Secrets, nil
 	}
 }
 
@@ -456,7 +480,6 @@ func ProcessLiteralTemplate(templateId int, templateString string, data interfac
 	return &buf, nil
 }
 
-
 type AgentManager struct {
 	accessToken              string
 	accessTokenTTL           time.Duration

cli/packages/cmd/export.go

@@ -87,11 +87,12 @@ var exportCmd = &cobra.Command{
 		}
 
 		request := models.GetAllSecretsParameters{
-			Environment:   environmentName,
-			TagSlugs:      tagSlugs,
-			WorkspaceId:   projectId,
-			SecretsPath:   secretsPath,
-			IncludeImport: includeImports,
+			Environment:            environmentName,
+			TagSlugs:               tagSlugs,
+			WorkspaceId:            projectId,
+			SecretsPath:            secretsPath,
+			IncludeImport:          includeImports,
+			ExpandSecretReferences: shouldExpandSecrets,
 		}
 
 		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
@@ -137,18 +138,6 @@ var exportCmd = &cobra.Command{
 		}
 
 		var output string
-		if shouldExpandSecrets {
-
-			authParams := models.ExpandSecretsAuthentication{}
-
-			if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-				authParams.InfisicalToken = token.Token
-			} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-				authParams.UniversalAuthAccessToken = token.Token
-			}
-
-			secrets = util.ExpandSecrets(secrets, authParams, "")
-		}
 		secrets = util.FilterSecretsByTag(secrets, tagSlugs)
 		secrets = util.SortSecretsByKeys(secrets)
 

cli/packages/cmd/run.go

@@ -137,15 +137,16 @@ var runCmd = &cobra.Command{
 		}
 
 		request := models.GetAllSecretsParameters{
-			Environment:   environmentName,
-			WorkspaceId:   projectId,
-			TagSlugs:      tagSlugs,
-			SecretsPath:   secretsPath,
-			IncludeImport: includeImports,
-			Recursive:     recursive,
+			Environment:            environmentName,
+			WorkspaceId:            projectId,
+			TagSlugs:               tagSlugs,
+			SecretsPath:            secretsPath,
+			IncludeImport:          includeImports,
+			Recursive:              recursive,
+			ExpandSecretReferences: shouldExpandSecrets,
 		}
 
-		injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, shouldExpandSecrets, token)
+		injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token)
 		if err != nil {
 			util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid")
 		}
@@ -153,7 +154,7 @@ var runCmd = &cobra.Command{
 		log.Debug().Msgf("injecting the following environment variables into shell: %v", injectableEnvironment.Variables)
 
 		if watchMode {
-			executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, shouldExpandSecrets, secretOverriding, token)
+			executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, secretOverriding, token)
 		} else {
 			if cmd.Flags().Changed("command") {
 				command := cmd.Flag("command").Value.String()
@@ -306,7 +307,7 @@ func waitForExitCommand(cmd *exec.Cmd) (int, error) {
 	return waitStatus.ExitStatus(), nil
 }
 
-func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, expandSecrets bool, secretOverriding bool, token *models.TokenDetails) {
+func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) {
 
 	var cmd *exec.Cmd
 	var err error
@@ -420,7 +421,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt
 		<-recheckSecretsChannel
 		watchMutex.Lock()
 
-		newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, expandSecrets, token)
+		newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token)
 		if err != nil {
 			log.Error().Err(err).Msg("[HOT RELOAD] Failed to fetch secrets")
 			continue
@@ -437,7 +438,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt
 	}
 }
 
-func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, shouldExpandSecrets bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) {
+func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) {
 
 	if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
 		request.InfisicalToken = token.Token
@@ -457,19 +458,6 @@ func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, proje
 		secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
 	}
 
-	if shouldExpandSecrets {
-
-		authParams := models.ExpandSecretsAuthentication{}
-
-		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-			authParams.InfisicalToken = token.Token
-		} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-			authParams.UniversalAuthAccessToken = token.Token
-		}
-
-		secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir)
-	}
-
 	secretsByKey := getSecretsByKeys(secrets)
 	environmentVariables := make(map[string]string)
 

cli/packages/cmd/secrets.go

@@ -79,12 +79,13 @@ var secretsCmd = &cobra.Command{
 		}
 
 		request := models.GetAllSecretsParameters{
-			Environment:   environmentName,
-			WorkspaceId:   projectId,
-			TagSlugs:      tagSlugs,
-			SecretsPath:   secretsPath,
-			IncludeImport: includeImports,
-			Recursive:     recursive,
+			Environment:            environmentName,
+			WorkspaceId:            projectId,
+			TagSlugs:               tagSlugs,
+			SecretsPath:            secretsPath,
+			IncludeImport:          includeImports,
+			Recursive:              recursive,
+			ExpandSecretReferences: shouldExpandSecrets,
 		}
 
 		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
@@ -104,17 +105,6 @@ var secretsCmd = &cobra.Command{
 			secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
 		}
 
-		if shouldExpandSecrets {
-			authParams := models.ExpandSecretsAuthentication{}
-			if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-				authParams.InfisicalToken = token.Token
-			} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-				authParams.UniversalAuthAccessToken = token.Token
-			}
-
-			secrets = util.ExpandSecrets(secrets, authParams, "")
-		}
-
 		// Sort the secrets by key so we can create a consistent output
 		secrets = util.SortSecretsByKeys(secrets)
 
@@ -382,12 +372,13 @@ func getSecretsByNames(cmd *cobra.Command, args []string) {
 	}
 
 	request := models.GetAllSecretsParameters{
-		Environment:   environmentName,
-		WorkspaceId:   projectId,
-		TagSlugs:      tagSlugs,
-		SecretsPath:   secretsPath,
-		IncludeImport: includeImports,
-		Recursive:     recursive,
+		Environment:            environmentName,
+		WorkspaceId:            projectId,
+		TagSlugs:               tagSlugs,
+		SecretsPath:            secretsPath,
+		IncludeImport:          includeImports,
+		Recursive:              recursive,
+		ExpandSecretReferences: shouldExpand,
 	}
 
 	if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
@@ -407,17 +398,6 @@ func getSecretsByNames(cmd *cobra.Command, args []string) {
 		secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
 	}
 
-	if shouldExpand {
-		authParams := models.ExpandSecretsAuthentication{}
-		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-			authParams.InfisicalToken = token.Token
-		} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-			authParams.UniversalAuthAccessToken = token.Token
-		}
-
-		secrets = util.ExpandSecrets(secrets, authParams, "")
-	}
-
 	requestedSecrets := []models.SingleEnvironmentVariable{}
 
 	secretsMap := getSecretsByKeys(secrets)

cli/packages/models/cli.go

@@ -30,6 +30,7 @@ type SingleEnvironmentVariable struct {
 	Value       string `json:"value"`
 	Type        string `json:"type"`
 	ID          string `json:"_id"`
+	SecretPath  string `json:"secretPath"`
 	Tags        []struct {
 		ID        string `json:"_id"`
 		Name      string `json:"name"`
@@ -103,6 +104,7 @@ type GetAllSecretsParameters struct {
 	SecretsPath              string
 	IncludeImport            bool
 	Recursive                bool
+	ExpandSecretReferences   bool
 }
 
 type InjectableEnvironmentResult struct {

cli/packages/util/secrets.go

@@ -8,8 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
-	"regexp"
 	"strings"
 	"unicode"
 
@@ -21,7 +19,7 @@ import (
 	"github.com/zalando/go-keyring"
 )
 
-func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string) ([]models.SingleEnvironmentVariable, error) {
+func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) ([]models.SingleEnvironmentVariable, error) {
 	serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4)
 	if len(serviceTokenParts) < 4 {
 		return nil, fmt.Errorf("invalid service token entered. Please double check your service token and try again")
@@ -49,12 +47,13 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str
 	}
 
 	rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{
-		WorkspaceId:   serviceTokenDetails.Workspace,
-		Environment:   environment,
-		SecretPath:    secretPath,
-		IncludeImport: includeImports,
-		Recursive:     recursive,
-		TagSlugs:      tagSlugs,
+		WorkspaceId:            serviceTokenDetails.Workspace,
+		Environment:            environment,
+		SecretPath:             secretPath,
+		IncludeImport:          includeImports,
+		Recursive:              recursive,
+		TagSlugs:               tagSlugs,
+		ExpandSecretReferences: expandSecretReferences,
 	})
 
 	if err != nil {
@@ -78,17 +77,18 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str
 
 }
 
-func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string) (models.PlaintextSecretResult, error) {
+func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) (models.PlaintextSecretResult, error) {
 	httpClient := resty.New()
 	httpClient.SetAuthToken(accessToken).
 		SetHeader("Accept", "application/json")
 
 	getSecretsRequest := api.GetRawSecretsV3Request{
-		WorkspaceId:   workspaceId,
-		Environment:   environmentName,
-		IncludeImport: includeImports,
-		Recursive:     recursive,
-		TagSlugs:      tagSlugs,
+		WorkspaceId:            workspaceId,
+		Environment:            environmentName,
+		IncludeImport:          includeImports,
+		Recursive:              recursive,
+		TagSlugs:               tagSlugs,
+		ExpandSecretReferences: expandSecretReferences,
 	}
 
 	if secretsPath != "" {
@@ -104,7 +104,7 @@ func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentNa
 	plainTextSecrets := []models.SingleEnvironmentVariable{}
 
 	for _, secret := range rawSecrets.Secrets {
-		plainTextSecrets = append(plainTextSecrets, models.SingleEnvironmentVariable{Key: secret.SecretKey, Value: secret.SecretValue, Type: secret.Type, WorkspaceId: secret.Workspace})
+		plainTextSecrets = append(plainTextSecrets, models.SingleEnvironmentVariable{Key: secret.SecretKey, Value: secret.SecretValue, Type: secret.Type, WorkspaceId: secret.Workspace, SecretPath: secret.SecretPath})
 	}
 
 	if includeImports {
@@ -145,6 +145,7 @@ func GetSinglePlainTextSecretByNameV3(accessToken string, workspaceId string, en
 		Type:        rawSecret.Secret.Type,
 		ID:          rawSecret.Secret.ID,
 		Comment:     rawSecret.Secret.SecretComment,
+		SecretPath:  rawSecret.Secret.SecretPath,
 	}
 
 	return formattedSecrets, rawSecret.ETag, nil
@@ -283,7 +284,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 		}
 
 		res, err := GetPlainTextSecretsV3(loggedInUserDetails.UserCredentials.JTWToken, infisicalDotJson.WorkspaceId,
-			params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs)
+			params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, true)
 		log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", err)
 
 		if err == nil {
@@ -312,7 +313,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 	} else {
 		if params.InfisicalToken != "" {
 			log.Debug().Msg("Trying to fetch secrets using service token")
-			secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs)
+			secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences)
 		} else if params.UniversalAuthAccessToken != "" {
 
 			if params.WorkspaceId == "" {
@@ -320,7 +321,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 			}
 
 			log.Debug().Msg("Trying to fetch secrets using universal auth")
-			res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs)
+			res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences)
 
 			errorToReturn = err
 			secretsToReturn = res.Secrets
@@ -330,44 +331,6 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 	return secretsToReturn, errorToReturn
 }
 
-var secRefRegex = regexp.MustCompile(`\${([^\}]*)}`)
-
-func recursivelyExpandSecret(expandedSecs map[string]string, interpolatedSecs map[string]string, crossSecRefFetch func(env string, path []string, key string) string, key string) string {
-	if v, ok := expandedSecs[key]; ok {
-		return v
-	}
-
-	interpolatedVal, ok := interpolatedSecs[key]
-	if !ok {
-		HandleError(fmt.Errorf("could not find refered secret -  %s", key), "Kindly check whether its provided")
-	}
-
-	refs := secRefRegex.FindAllStringSubmatch(interpolatedVal, -1)
-	for _, val := range refs {
-		// key: "${something}" val: [${something},something]
-		interpolatedExp, interpolationKey := val[0], val[1]
-		ref := strings.Split(interpolationKey, ".")
-
-		// ${KEY1} => [key1]
-		if len(ref) == 1 {
-			val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey)
-			interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val)
-			continue
-		}
-
-		// cross board reference ${env.folder.key1} => [env folder key1]
-		if len(ref) > 1 {
-			secEnv, tmpSecPath, secKey := ref[0], ref[1:len(ref)-1], ref[len(ref)-1]
-			interpolatedSecs[interpolationKey] = crossSecRefFetch(secEnv, tmpSecPath, secKey) // get the reference value
-			val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey)
-			interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val)
-		}
-
-	}
-	expandedSecs[key] = interpolatedVal
-	return interpolatedVal
-}
-
 func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]models.SingleEnvironmentVariable {
 	secretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets))
 
@@ -378,70 +341,6 @@ func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]mod
 	return secretMapByName
 }
 
-func ExpandSecrets(secrets []models.SingleEnvironmentVariable, auth models.ExpandSecretsAuthentication, projectConfigPathDir string) []models.SingleEnvironmentVariable {
-	expandedSecs := make(map[string]string)
-	interpolatedSecs := make(map[string]string)
-	// map[env.secret-path][keyname]Secret
-	crossEnvRefSecs := make(map[string]map[string]models.SingleEnvironmentVariable) // a cache to hold all cross board reference secrets
-
-	for _, sec := range secrets {
-		// get all references in a secret
-		refs := secRefRegex.FindAllStringSubmatch(sec.Value, -1)
-		// nil means its a secret without reference
-		if refs == nil {
-			expandedSecs[sec.Key] = sec.Value // atomic secrets without any interpolation
-		} else {
-			interpolatedSecs[sec.Key] = sec.Value
-		}
-	}
-
-	for i, sec := range secrets {
-		// already present pick that up
-		if expandedVal, ok := expandedSecs[sec.Key]; ok {
-			secrets[i].Value = expandedVal
-			continue
-		}
-
-		expandedVal := recursivelyExpandSecret(expandedSecs, interpolatedSecs, func(env string, secPaths []string, secKey string) string {
-			secPaths = append([]string{"/"}, secPaths...)
-			secPath := path.Join(secPaths...)
-
-			secPathDot := strings.Join(secPaths, ".")
-			uniqKey := fmt.Sprintf("%s.%s", env, secPathDot)
-
-			if crossRefSec, ok := crossEnvRefSecs[uniqKey]; !ok {
-
-				var refSecs []models.SingleEnvironmentVariable
-				var err error
-
-				// if not in cross reference cache, fetch it from server
-				if auth.InfisicalToken != "" {
-					refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, InfisicalToken: auth.InfisicalToken, SecretsPath: secPath}, projectConfigPathDir)
-				} else if auth.UniversalAuthAccessToken != "" {
-					refSecs, err = GetAllEnvironmentVariables((models.GetAllSecretsParameters{Environment: env, UniversalAuthAccessToken: auth.UniversalAuthAccessToken, SecretsPath: secPath, WorkspaceId: sec.WorkspaceId}), projectConfigPathDir)
-				} else if IsLoggedIn() {
-					refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, SecretsPath: secPath}, projectConfigPathDir)
-				} else {
-					HandleError(errors.New("no authentication provided"), "Please provide authentication to fetch secrets")
-				}
-				if err != nil {
-					HandleError(err, fmt.Sprintf("Could not fetch secrets in environment: %s secret-path: %s", env, secPath), "If you are using a service token to fetch secrets, please ensure it is valid")
-				}
-				refSecsByKey := getSecretsByKeys(refSecs)
-				// save it to avoid calling api again for same environment and folder path
-				crossEnvRefSecs[uniqKey] = refSecsByKey
-				return refSecsByKey[secKey].Value
-
-			} else {
-				return crossRefSec[secKey].Value
-			}
-		}, sec.Key)
-
-		secrets[i].Value = expandedVal
-	}
-	return secrets
-}
-
 func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType string) []models.SingleEnvironmentVariable {
 	personalSecrets := make(map[string]models.SingleEnvironmentVariable)
 	sharedSecrets := make(map[string]models.SingleEnvironmentVariable)

cli/secret-render-template

@@ -1,5 +1,5 @@
-{{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }}
+{{- with secret "8fac9f01-4a81-44d7-8ff0-3d7be684f56f" "staging" "/" `{"recursive":true, "expandSecretReferences": false}` }}
 {{- range . }}
 {{ .Key }}={{ .Value }}
 {{- end }}
-{{- end }}
+{{- end }}

docs/changelog/overview.mdx

@@ -4,6 +4,27 @@ title: "Changelog"
 
 The changelog below reflects new product developments and updates on a monthly basis.
 
+## September 2024
+- Improved paginations for identities and secrets. 
+- Significant improvements to the [Infisical Terraform Provider](https://registry.terraform.io/providers/Infisical/infisical/latest/docs). 
+- Created [Slack Integration](https://infisical.com/docs/documentation/platform/workflow-integrations/slack-integration#slack-integration) for Access Requests and Approval Workflows. 
+- Added Dynamic Secrets for [Elaticsearch](https://infisical.com/docs/documentation/platform/dynamic-secrets/elastic-search) and [MongoDB](https://infisical.com/docs/documentation/platform/dynamic-secrets/mongo-db). 
+- More authentication methods are now supported by Infisical SDKs and Agent. 
+- Integrations now have dedicated audit logs and an overview screen. 
+- Added support for secret referencing in the Terraform Provider. 
+- Released support for [older versions of .NET](https://www.nuget.org/packages/Infisical.Sdk#supportedframeworks-body-tab) via SDK.
+- Released Infisical PKI Issuer which works alongside `cert-manager` to manage certificates in Kubernetes.
+
+## August 2024
+- Added [Azure DevOps integration](https://infisical.com/docs/integrations/cloud/azure-devops).
+- Released ability to hot-reload variables in CLI ([--watch flag](https://infisical.com/docs/cli/commands/run#infisical-run:watch)).
+- Added Dynamic Secrets for [Redis](https://infisical.com/docs/documentation/platform/dynamic-secrets/redis). 
+- Added [Alerting](https://infisical.com/docs/documentation/platform/pki/alerting) for Certificate Management. 
+- You can now specify roles and project memberships when adding new users. 
+- Approval workflows now have email notifications. 
+- Access requests are now integrated with User Groups. 
+- Released ability to use IAM Roles for AWS Integrations. 
+
 ## July 2024
 - Released the official [Ruby SDK](https://infisical.com/docs/sdks/languages/ruby). 
 - Increased the speed and efficiency of secret operations.

docs/documentation/platform/admin-panel/org-admin-console.mdx

@@ -0,0 +1,32 @@
+---
+title: "Organization Admin Console"
+description: "Manage your Infisical organization from our organization admin console."
+---
+
+The Organization Admin Console provides a user-friendly interface for Infisical organization admins to manage organization-related configurations.
+
+## Accessing the Organization Admin Console
+
+Only organization admins have access to the Organization Admin Console. 
+
+![Access Organization Admin Panel](/images/platform/admin-panels/access-org-admin-console.png)
+
+1. Click on the profile icon in the left sidebar.
+2. From the dropdown menu, select `Organization Admin Console`.
+
+## Projects Section
+
+![Projects Section](/images/platform/admin-panels/org-admin-console-projects.png)
+
+The Projects Section lists all projects created within your organization, including those you do not have membership in. You can easily search for a project by name using the search bar.
+
+### Accessing a Project in Your Organization
+
+If you want to access a project in which you are not a member but are an organization admin, follow these steps:
+
+![Access project](/images/platform/admin-panels/org-admin-console-access.png)
+
+1. Click on the three-dot icon next to the project you wish to access.
+2. Click on the **Access** button.
+
+This will grant you admin permissions for the selected project and generate an audit log of your access, ensuring transparency regarding admin privileges.

docs/documentation/platform/admin-panel/overview.mdx

@@ -0,0 +1,25 @@
+---
+description: "Learn about Infisical's Admin Panel."
+---
+
+The Infisical Admin Panel allows you to configure and manage various resources within your organization and server.
+
+<CardGroup cols={2}>
+  <Card
+    title="Server Admin Panel"
+    href="./server-admin"
+    icon="user-tie"
+    color="#000000"
+  >
+    Configure and manage your server settings effectively.
+  </Card>
+  
+  <Card
+    title="Organization Admin Console"
+    href="./org-admin-console"
+    icon="sitemap"
+    color="#000000"
+  >
+    Manage settings specific to your organization.
+  </Card>
+</CardGroup>

docs/documentation/platform/admin-panel/server-admin.mdx

@@ -0,0 +1,70 @@
+---
+title: "Server Admin Panel"
+description: "Manage your Infisical server from the Server Admin Panel."
+---
+
+The Server Admin Panel provides a user interface for Infisical server administrators to configure various parameters as needed. This includes configuring rate limits, managing allowed signups, and more.
+
+## Accessing the Server Admin Panel
+
+The first user who created the account in Infisical is designated as the server administrator. You can access the admin panel by navigating as follows:
+
+![Access Server Admin Panel](/images/platform/admin-panels/access-server-admin-panel.png)
+
+1. Click on the profile icon in the left sidebar.
+2. From the dropdown menu, select `Server Admin Panel`.
+
+## General Section
+
+![General Settings](/images/platform/admin-panels/admin-panel-general.png)
+
+### Allow User Signups
+
+This setting controls whether users can sign up for your Infisical instance. The options are:
+
+1. **Anyone**: Any user with access to your instance can sign up.
+2. **Disabled**: No one will be able to sign up.
+
+### Restrict Signup Domain
+
+This setting allows only users with specific email domains (such as your organization's domain) to sign up.
+
+### Default Organization
+
+Use this setting if you want all users accessing your Infisical instance to log in through your configured SAML/LDAP provider. This prevents users from manually entering their organization slug during authentication and redirects them to the SAML/LDAP authentication page.
+
+### Trust Emails
+
+By default, Infisical does not trust emails logged in via SAML/LDAP/OIDC due to the potential for email spoofing. Users must verify their email addresses before proceeding. You can disable this validation if you are running an Infisical instance within your organization and trust incoming emails from your members.
+
+## Authentication Section
+
+![Authentication Settings](/images/platform/admin-panels/admin-panel-auths.png)
+
+This section allows you to configure various login and signup methods for your instance.
+
+## Rate Limit Section
+
+![Rate Limit Settings](/images/platform/admin-panels/admin-panel-rate-limits.png)
+
+Configure the rate limits for your Infisical instance across various endpoints. You do not need to redeploy when making changes to rate limits; they will be automatically synchronized to all instances.
+
+<Info>
+  Note that rate limit configuration is a paid feature. Please contact sales@infisical.com to purchase a license for its use.
+</Info>
+
+## User Management Section
+
+![User Management](/images/platform/admin-panels/admin-panel-users.png)
+
+The User Management section lists all users who have signed up for your instance. You can search for users using the search bar. 
+
+To delete a user from Infisical:
+
+1. Search for the user.
+2. Click the cross button next to the user.
+3. Confirm the warning popup.
+
+<Info>
+  Note that user management configuration is a paid feature. Please contact sales@infisical.com to purchase a license for its use.
+</Info>

docs/integrations/platforms/infisical-agent.mdx

@@ -312,21 +312,37 @@ infisical agent --config example-agent-config-file.yaml
 
 <Accordion title="listSecrets">
   ```bash
-    listSecrets "<project-id>" "environment-slug" "<secret-path>"
+    listSecrets "<project-id>" "environment-slug" "<secret-path>" "<optional-modifier>"
   ```
-    ```bash example-template-usage
-  {{- with listSecrets "6553ccb2b7da580d7f6e7260" "dev" "/" }}
+    ```bash example-template-usage-1
+  {{- with listSecrets "6553ccb2b7da580d7f6e7260" "dev" "/" `{"recursive": false, "expandSecretReferences": true}` }}
   {{- range . }}
   {{ .Key }}={{ .Value }}
   {{- end }}
   {{- end }}
   ```
+  ```bash example-template-usage-2
+{{- with secret "da8056c8-01e2-4d24-b39f-cb4e004b8d44" "staging" "/" `{"recursive": true, "expandSecretReferences": true}` }}
+{{- range . }}
+{{- if eq .SecretPath "/"}}
+{{ .Key }}={{ .Value }}
+{{- else}}
+{{ .SecretPath }}/{{ .Key }}={{ .Value }}
+{{- end}}
+{{- end }}
+{{- end }}
+  ```
+
+  
 
 **Function name**: listSecrets
 
-**Description**: This function can be used to render the full list of secrets within a given project, environment and secret path.
+**Description**: This function can be used to render the full list of secrets within a given project, environment and secret path. 
 
-**Returns**: A single secret object with the following keys `Key, WorkspaceId, Value, Type, ID, and Comment`
+An optional JSON argument is also available. It includes the properties `recursive`, which defaults to false, and `expandSecretReferences`, which defaults to true and expands the returned secrets.
+
+
+**Returns**: A single secret object with the following keys `Key, WorkspaceId, Value, SecretPath, Type, ID, and Comment`
 
 </Accordion>
 

docs/mint.json

@@ -187,6 +187,14 @@
             "documentation/platform/workflow-integrations/slack-integration"
           ]
         },
+        {
+          "group": "Admin Panel",
+          "pages": [
+            "documentation/platform/admin-panel/overview",
+            "documentation/platform/admin-panel/server-admin",
+            "documentation/platform/admin-panel/org-admin-console"
+          ]
+        },
         "documentation/platform/secret-sharing"
       ]
     },
@@ -858,5 +866,166 @@
     "koala": {
       "publicApiKey": "pk_b50d7184e0e39ddd5cdb43cf6abeadd9b97d"
     }
+  },
+  "footer": {
+    "socials": {
+      "x": "https://www.twitter.com/infisical/",
+      "linkedin": "https://www.linkedin.com/company/infisical/",
+      "github": "https://github.com/Infisical/infisical-cli",
+      "slack": "https://infisical.com/slack"
+    },
+    "links": [
+      {
+        "title": "PRODUCT",
+        "links": [
+          { "label": "Secret Management", "url": "https://infisical.com/" },
+          { "label": "Secret Scanning", "url": "https://infisical.com/radar" },
+          {
+            "label": "Share Secrets",
+            "url": "https://app.infisical.com/share-secret"
+          },
+          { "label": "Pricing", "url": "https://infisical.com/pricing" },
+          {
+            "label": "Security",
+            "url": "https://infisical.com/docs/internals/security"
+          },
+          {
+            "label": "Blog",
+            "url": "https://infisical.com/blog"
+          },
+          {
+            "label": "Infisical vs Vault",
+            "url": "https://infisical.com/infisical-vs-hashicorp-vault"
+          },
+          {
+            "label": "Forum",
+            "url": "https://questions.infisical.com/"
+          }
+        ]
+      },
+      {
+        "title": "USE CASES",
+        "links": [
+          {
+            "label": "Infisical Agent",
+            "url": "https://infisical.com/docs/documentation/getting-started/introduction"
+          },
+          {
+            "label": "Kubernetes",
+            "url": "https://infisical.com/docs/integrations/platforms/kubernetes"
+          },
+          {
+            "label": "Dynamic Secrets",
+            "url": "https://infisical.com/docs/documentation/platform/dynamic-secrets/overview"
+          },
+          {
+            "label": "Terraform",
+            "url": "https://infisical.com/docs/integrations/frameworks/terraform"
+          },
+          {
+            "label": "Ansible",
+            "url": "https://infisical.com/docs/integrations/platforms/ansible"
+          },
+          {
+            "label": "Jenkins",
+            "url": "https://infisical.com/docs/integrations/cicd/jenkins"
+          },
+          {
+            "label": "Docker",
+            "url": "https://infisical.com/docs/integrations/platforms/docker-intro"
+          },
+          {
+            "label": "AWS ECS",
+            "url": "https://infisical.com/docs/integrations/platforms/ecs-with-agent"
+          },
+          {
+            "label": "GitLab",
+            "url": "https://infisical.com/docs/integrations/cicd/gitlab"
+          },
+          {
+            "label": "GitHub",
+            "url": "https://infisical.com/docs/integrations/cicd/githubactions"
+          },
+          {
+            "label": "SDK",
+            "url": "https://infisical.com/docs/sdks/overview"
+          }
+        ]
+      },
+      {
+        "title": "DEVELOPERS",
+        "links": [
+          {
+            "label": "Changelog",
+            "url": "https://www.infisical.com/docs/changelog"
+          },
+          {
+            "label": "Status",
+            "url": "https://status.infisical.com/"
+          },
+          {
+            "label": "Feedback & Requests",
+            "url": "https://github.com/Infisical/infisical/issues"
+          },
+          {
+            "label": "Trust of Center",
+            "url": "https://app.vanta.com/infisical.com/trust/hoop8cr78cuarxo9sztvs"
+          },
+          {
+            "label": "Open Source Friends",
+            "url": "https://infisical.com/infisical-friends"
+          },
+          {
+            "label": "How to contribute",
+            "url": "https://www.infisical.com/infisical-heroes"
+          }
+        ]
+      },
+      {
+        "title": "OTHERS",
+        "links": [
+          {
+            "label": "Customers",
+            "url": "https://infisical.com/customers/traba"
+          },
+          {
+            "label": "Company Handbook",
+            "url": "https://infisical.com/wiki/handbook/overview"
+          },
+          {
+            "label": "Careers",
+            "url": "https://infisical.com/careers"
+          },
+          {
+            "label": "Terms of Service",
+            "url": "https://infisical.com/terms"
+          },
+          {
+            "label": "Privacy Policy",
+            "url": "https://infisical.com/privacy"
+          },
+          {
+            "label": "Subprocessors",
+            "url": "https://infisical.com/subprocessors"
+          },
+          {
+            "label": "SLA",
+            "url": "https://infisical.com/sla"
+          },
+          {
+            "label": "Team Email",
+            "url": "mailto:team@infisical.com"
+          },
+          {
+            "label": "Sales",
+            "url": "mailto:sales@infisical.com"
+          },
+          {
+            "label": "Support",
+            "url": "https://infisical.com/slack"
+          }
+        ]
+      }
+    ]
   }
 }

docs/self-hosting/configuration/requirements.mdx

@@ -59,6 +59,7 @@ Redis requirements:
 - Use Redis versions 6.x or 7.x. We advise upgrading to at least Redis 6.2.
 - Redis Cluster mode is currently not supported; use Redis Standalone, with or without High Availability (HA).
 - Redis storage needs are minimal: a setup with 2 vCPU, 4 GB RAM, and 30GB SSD will be sufficient for small deployments.
+- Set cache eviction policy to `noeviction`.
 
 ## Supported Web Browsers
 

frontend/package-lock.json

@@ -88,6 +88,7 @@
         "react-mailchimp-subscribe": "^2.1.3",
         "react-markdown": "^8.0.3",
         "react-redux": "^8.0.2",
+        "react-select": "^5.8.1",
         "react-table": "^7.8.0",
         "react-toastify": "^9.1.3",
         "sanitize-html": "^2.12.1",
@@ -2505,15 +2506,16 @@
       }
     },
     "node_modules/@emotion/babel-plugin": {
-      "version": "11.11.0",
-      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz",
-      "integrity": "sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==",
+      "version": "11.12.0",
+      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.12.0.tgz",
+      "integrity": "sha512-y2WQb+oP8Jqvvclh8Q55gLUyb7UFvgv7eJfsj7td5TToBrIUtPay2kMrZi4xjq9qw2vD0ZR5fSho0yqoFgX7Rw==",
+      "license": "MIT",
       "dependencies": {
         "@babel/helper-module-imports": "^7.16.7",
         "@babel/runtime": "^7.18.3",
-        "@emotion/hash": "^0.9.1",
-        "@emotion/memoize": "^0.8.1",
-        "@emotion/serialize": "^1.1.2",
+        "@emotion/hash": "^0.9.2",
+        "@emotion/memoize": "^0.9.0",
+        "@emotion/serialize": "^1.2.0",
         "babel-plugin-macros": "^3.1.0",
         "convert-source-map": "^1.5.0",
         "escape-string-regexp": "^4.0.0",
@@ -2522,18 +2524,31 @@
         "stylis": "4.2.0"
       }
     },
+    "node_modules/@emotion/babel-plugin/node_modules/@emotion/memoize": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz",
+      "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
+      "license": "MIT"
+    },
     "node_modules/@emotion/cache": {
-      "version": "11.11.0",
-      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz",
-      "integrity": "sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ==",
+      "version": "11.13.1",
+      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.13.1.tgz",
+      "integrity": "sha512-iqouYkuEblRcXmylXIwwOodiEK5Ifl7JcX7o6V4jI3iW4mLXX3dmt5xwBtIkJiQEXFAI+pC8X0i67yiPkH9Ucw==",
+      "license": "MIT",
       "dependencies": {
-        "@emotion/memoize": "^0.8.1",
-        "@emotion/sheet": "^1.2.2",
-        "@emotion/utils": "^1.2.1",
-        "@emotion/weak-memoize": "^0.3.1",
+        "@emotion/memoize": "^0.9.0",
+        "@emotion/sheet": "^1.4.0",
+        "@emotion/utils": "^1.4.0",
+        "@emotion/weak-memoize": "^0.4.0",
         "stylis": "4.2.0"
       }
     },
+    "node_modules/@emotion/cache/node_modules/@emotion/memoize": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz",
+      "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
+      "license": "MIT"
+    },
     "node_modules/@emotion/css": {
       "version": "11.11.2",
       "resolved": "https://registry.npmjs.org/@emotion/css/-/css-11.11.2.tgz",
@@ -2547,9 +2562,10 @@
       }
     },
     "node_modules/@emotion/hash": {
-      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz",
-      "integrity": "sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ=="
+      "version": "0.9.2",
+      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz",
+      "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==",
+      "license": "MIT"
     },
     "node_modules/@emotion/is-prop-valid": {
       "version": "0.8.8",
@@ -2571,18 +2587,49 @@
       "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz",
       "integrity": "sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA=="
     },
-    "node_modules/@emotion/serialize": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.3.tgz",
-      "integrity": "sha512-iD4D6QVZFDhcbH0RAG1uVu1CwVLMWUkCvAqqlewO/rxf8+87yIBAlt4+AxMiiKPLs5hFc0owNk/sLLAOROw3cA==",
+    "node_modules/@emotion/react": {
+      "version": "11.13.3",
+      "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.13.3.tgz",
+      "integrity": "sha512-lIsdU6JNrmYfJ5EbUCf4xW1ovy5wKQ2CkPRM4xogziOxH1nXxBSjpC9YqbFAP7circxMfYp+6x676BqWcEiixg==",
+      "license": "MIT",
       "dependencies": {
-        "@emotion/hash": "^0.9.1",
-        "@emotion/memoize": "^0.8.1",
-        "@emotion/unitless": "^0.8.1",
-        "@emotion/utils": "^1.2.1",
+        "@babel/runtime": "^7.18.3",
+        "@emotion/babel-plugin": "^11.12.0",
+        "@emotion/cache": "^11.13.0",
+        "@emotion/serialize": "^1.3.1",
+        "@emotion/use-insertion-effect-with-fallbacks": "^1.1.0",
+        "@emotion/utils": "^1.4.0",
+        "@emotion/weak-memoize": "^0.4.0",
+        "hoist-non-react-statics": "^3.3.1"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@emotion/serialize": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.3.2.tgz",
+      "integrity": "sha512-grVnMvVPK9yUVE6rkKfAJlYZgo0cu3l9iMC77V7DW6E1DUIrU68pSEXRmFZFOFB1QFo57TncmOcvcbMDWsL4yA==",
+      "license": "MIT",
+      "dependencies": {
+        "@emotion/hash": "^0.9.2",
+        "@emotion/memoize": "^0.9.0",
+        "@emotion/unitless": "^0.10.0",
+        "@emotion/utils": "^1.4.1",
         "csstype": "^3.0.2"
       }
     },
+    "node_modules/@emotion/serialize/node_modules/@emotion/memoize": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz",
+      "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
+      "license": "MIT"
+    },
     "node_modules/@emotion/server": {
       "version": "11.11.0",
       "resolved": "https://registry.npmjs.org/@emotion/server/-/server-11.11.0.tgz",
@@ -2603,9 +2650,10 @@
       }
     },
     "node_modules/@emotion/sheet": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz",
-      "integrity": "sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA=="
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz",
+      "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==",
+      "license": "MIT"
     },
     "node_modules/@emotion/stylis": {
       "version": "0.8.5",
@@ -2613,28 +2661,31 @@
       "integrity": "sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ=="
     },
     "node_modules/@emotion/unitless": {
-      "version": "0.8.1",
-      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz",
-      "integrity": "sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ=="
+      "version": "0.10.0",
+      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.10.0.tgz",
+      "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==",
+      "license": "MIT"
     },
     "node_modules/@emotion/use-insertion-effect-with-fallbacks": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz",
-      "integrity": "sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==",
-      "dev": true,
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.1.0.tgz",
+      "integrity": "sha512-+wBOcIV5snwGgI2ya3u99D7/FJquOIniQT1IKyDsBmEgwvpxMNeS65Oib7OnE2d2aY+3BU4OiH+0Wchf8yk3Hw==",
+      "license": "MIT",
       "peerDependencies": {
         "react": ">=16.8.0"
       }
     },
     "node_modules/@emotion/utils": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz",
-      "integrity": "sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg=="
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.4.1.tgz",
+      "integrity": "sha512-BymCXzCG3r72VKJxaYVwOXATqXIZ85cuvg0YOUDxMGNrKc1DJRZk8MgV5wyXRyEayIMd4FuXJIUgTBXvDNW5cA==",
+      "license": "MIT"
     },
     "node_modules/@emotion/weak-memoize": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz",
-      "integrity": "sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww=="
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz",
+      "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==",
+      "license": "MIT"
     },
     "node_modules/@esbuild/android-arm": {
       "version": "0.18.20",
@@ -8844,6 +8895,15 @@
         "redux": "^4.0.0"
       }
     },
+    "node_modules/@types/react-transition-group": {
+      "version": "4.4.11",
+      "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.11.tgz",
+      "integrity": "sha512-RM05tAniPZ5DZPzzNFP+DmrcOdD0efDUxMy3145oljWSl3x9ZV5vhme98gTxFrj2lhXvmGNnUiuDyJgY9IKkNA==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/react": "*"
+      }
+    },
     "node_modules/@types/resolve": {
       "version": "1.20.6",
       "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.6.tgz",
@@ -12668,6 +12728,16 @@
         "utila": "~0.4"
       }
     },
+    "node_modules/dom-helpers": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz",
+      "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.8.7",
+        "csstype": "^3.0.2"
+      }
+    },
     "node_modules/dom-serializer": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
@@ -21113,6 +21183,33 @@
         "react": ">= 16.3"
       }
     },
+    "node_modules/react-select": {
+      "version": "5.8.1",
+      "resolved": "https://registry.npmjs.org/react-select/-/react-select-5.8.1.tgz",
+      "integrity": "sha512-RT1CJmuc+ejqm5MPgzyZujqDskdvB9a9ZqrdnVLsvAHjJ3Tj0hELnLeVPQlmYdVKCdCpxanepl6z7R5KhXhWzg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.12.0",
+        "@emotion/cache": "^11.4.0",
+        "@emotion/react": "^11.8.1",
+        "@floating-ui/dom": "^1.0.1",
+        "@types/react-transition-group": "^4.4.0",
+        "memoize-one": "^6.0.0",
+        "prop-types": "^15.6.0",
+        "react-transition-group": "^4.3.0",
+        "use-isomorphic-layout-effect": "^1.1.2"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
+    "node_modules/react-select/node_modules/memoize-one": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz",
+      "integrity": "sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==",
+      "license": "MIT"
+    },
     "node_modules/react-style-singleton": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
@@ -21167,6 +21264,22 @@
         "node": ">=6"
       }
     },
+    "node_modules/react-transition-group": {
+      "version": "4.4.5",
+      "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
+      "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@babel/runtime": "^7.5.5",
+        "dom-helpers": "^5.0.1",
+        "loose-envify": "^1.4.0",
+        "prop-types": "^15.6.2"
+      },
+      "peerDependencies": {
+        "react": ">=16.6.0",
+        "react-dom": ">=16.6.0"
+      }
+    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
@@ -24242,6 +24355,20 @@
         }
       }
     },
+    "node_modules/use-isomorphic-layout-effect": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz",
+      "integrity": "sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==",
+      "license": "MIT",
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/use-memo-one": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/use-memo-one/-/use-memo-one-1.1.3.tgz",

frontend/package.json

@@ -101,6 +101,7 @@
     "react-mailchimp-subscribe": "^2.1.3",
     "react-markdown": "^8.0.3",
     "react-redux": "^8.0.2",
+    "react-select": "^5.8.1",
     "react-table": "^7.8.0",
     "react-toastify": "^9.1.3",
     "sanitize-html": "^2.12.1",

frontend/src/components/v2/MultiSelect/MultiSelect.tsx

@@ -0,0 +1,104 @@
+import Select, {
+  ClearIndicatorProps,
+  components,
+  DropdownIndicatorProps,
+  MultiValueRemoveProps,
+  OptionProps,
+  Props
+} from "react-select";
+import { faCheckCircle, faCircleXmark } from "@fortawesome/free-regular-svg-icons";
+import { faChevronDown, faX } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { twMerge } from "tailwind-merge";
+
+const DropdownIndicator = (props: DropdownIndicatorProps) => {
+  return (
+    <components.DropdownIndicator {...props}>
+      <FontAwesomeIcon icon={faChevronDown} size="xs" />
+    </components.DropdownIndicator>
+  );
+};
+
+const ClearIndicator = (props: ClearIndicatorProps) => {
+  return (
+    <components.ClearIndicator {...props}>
+      <FontAwesomeIcon icon={faCircleXmark} />
+    </components.ClearIndicator>
+  );
+};
+
+const MultiValueRemove = (props: MultiValueRemoveProps) => {
+  return (
+    <components.MultiValueRemove {...props}>
+      <FontAwesomeIcon icon={faX} size="xs" />
+    </components.MultiValueRemove>
+  );
+};
+
+const Option = ({ isSelected, children, ...props }: OptionProps) => {
+  return (
+    <components.Option isSelected={isSelected} {...props}>
+      {children}
+      {isSelected && (
+        <FontAwesomeIcon className="ml-2 text-primary" icon={faCheckCircle} size="sm" />
+      )}
+    </components.Option>
+  );
+};
+
+export const MultiSelect = (props: Props) => (
+  <Select
+    isMulti
+    closeMenuOnSelect={false}
+    hideSelectedOptions={false}
+    unstyled
+    styles={{
+      input: (base) => ({
+        ...base,
+        "input:focus": {
+          boxShadow: "none"
+        }
+      }),
+      multiValueLabel: (base) => ({
+        ...base,
+        whiteSpace: "normal",
+        overflow: "visible"
+      }),
+      control: (base) => ({
+        ...base,
+        transition: "none"
+      })
+    }}
+    components={{ DropdownIndicator, ClearIndicator, MultiValueRemove, Option }}
+    classNames={{
+      container: () => "w-full font-inter",
+      control: ({ isFocused }) =>
+        twMerge(
+          isFocused ? "border-primary-400/50" : "border-mineshaft-600 hover:border-gray-400",
+          "border w-full p-0.5 rounded-md font-inter bg-mineshaft-900 hover:cursor-pointer"
+        ),
+      placeholder: () => "text-mineshaft-400 text-sm pl-1 py-0.5",
+      input: () => "pl-1 py-0.5",
+      valueContainer: () => "p-1 max-h-[14rem] !overflow-y-scroll gap-1",
+      singleValue: () => "leading-7 ml-1",
+      multiValue: () => "bg-mineshaft-600 rounded items-center py-0.5 px-2 gap-1.5",
+      multiValueLabel: () => "leading-6 text-sm",
+      multiValueRemove: () => "hover:text-red text-bunker-400",
+      indicatorsContainer: () => "p-1 gap-1",
+      clearIndicator: () => "p-1 hover:text-red text-bunker-400",
+      indicatorSeparator: () => "bg-bunker-400",
+      dropdownIndicator: () => "text-bunker-200 p-1",
+      menu: () =>
+        "mt-2 border text-sm text-mineshaft-200 bg-mineshaft-900 border-mineshaft-600 rounded-md",
+      groupHeading: () => "ml-3 mt-2 mb-1 text-mineshaft-400 text-sm",
+      option: ({ isFocused, isSelected }) =>
+        twMerge(
+          isFocused && "bg-mineshaft-700 active:bg-mineshaft-600",
+          isSelected && "text-mineshaft-400",
+          "hover:cursor-pointer text-xs px-3 py-2"
+        ),
+      noOptionsMessage: () => "text-mineshaft-400 p-2 rounded-md"
+    }}
+    {...props}
+  />
+);

frontend/src/components/v2/MultiSelect/index.tsx

@@ -0,0 +1 @@
+export * from "./MultiSelect";

frontend/src/components/v2/index.tsx

@@ -18,6 +18,7 @@ export * from "./IconButton";
 export * from "./Input";
 export * from "./Menu";
 export * from "./Modal";
+export * from "./MultiSelect";
 export * from "./NoticeBanner";
 export * from "./Pagination";
 export * from "./Popoverv2";

frontend/src/hooks/api/auditLogs/queries.tsx

@@ -1,5 +1,7 @@
 import { useInfiniteQuery, UseInfiniteQueryOptions, useQuery } from "@tanstack/react-query";
+import { AxiosError } from "axios";
 
+import { createNotification } from "@app/components/notifications";
 import { apiRequest } from "@app/config/request";
 
 import { Actor, AuditLog, TGetAuditLogsFilter } from "./types";
@@ -28,27 +30,37 @@ export const useGetAuditLogs = (
   return useInfiniteQuery({
     queryKey: auditLogKeys.getAuditLogs(projectId, filters),
     queryFn: async ({ pageParam }) => {
-      const { data } = await apiRequest.get<{ auditLogs: AuditLog[] }>(
-        "/api/v1/organization/audit-logs",
-        {
-          params: {
-            ...filters,
-            offset: pageParam,
-            startDate: filters?.startDate?.toISOString(),
-            endDate: filters?.endDate?.toISOString(),
-            ...(filters.eventMetadata && Object.keys(filters.eventMetadata).length
-              ? {
-                  eventMetadata: Object.entries(filters.eventMetadata)
-                    .map(([key, value]) => `${key}=${value}`)
-                    .join(",")
-                }
-              : {}),
-            ...(filters.eventType?.length ? { eventType: filters.eventType.join(",") } : {}),
-            ...(projectId ? { projectId } : {})
+      try {
+        const { data } = await apiRequest.get<{ auditLogs: AuditLog[] }>(
+          "/api/v1/organization/audit-logs",
+          {
+            params: {
+              ...filters,
+              offset: pageParam,
+              startDate: filters?.startDate?.toISOString(),
+              endDate: filters?.endDate?.toISOString(),
+              ...(filters.eventMetadata && Object.keys(filters.eventMetadata).length
+                ? {
+                    eventMetadata: Object.entries(filters.eventMetadata)
+                      .map(([key, value]) => `${key}=${value}`)
+                      .join(",")
+                  }
+                : {}),
+              ...(filters.eventType?.length ? { eventType: filters.eventType.join(",") } : {}),
+              ...(projectId ? { projectId } : {})
+            }
           }
+        );
+        return data.auditLogs;
+      } catch (error) {
+        if (error instanceof AxiosError) {
+          createNotification({
+            type: "error",
+            text: error.response?.data.message
+          });
         }
-      );
-      return data.auditLogs;
+        return [];
+      }
     },
     getNextPageParam: (lastPage, pages) =>
       lastPage.length !== 0 ? pages.length * filters.limit : undefined,

frontend/src/hooks/api/auditLogs/types.tsx

@@ -886,8 +886,6 @@ export type AuditLog = {
   userAgentType: UserAgentType;
   createdAt: string;
   updatedAt: string;
-  project?: {
-    name: string;
-    slug: string;
-  };
+  projectName?: string;
+  projectId?: string;
 };

frontend/src/hooks/api/migration/mutations.tsx

@@ -8,21 +8,27 @@ export const useImportEnvKey = () => {
   const queryClient = useQueryClient();
 
   return useMutation({
-    mutationFn: async ({
-      encryptedJson,
-      decryptionKey
-    }: {
-      encryptedJson: {
-        nonce: string;
-        data: string;
-      };
-      decryptionKey: string;
-    }) => {
-      const { data } = await apiRequest.post("/api/v3/migrate/env-key/", {
-        encryptedJson,
-        decryptionKey
-      });
-      return data;
+    mutationFn: async ({ file, decryptionKey }: { file: File; decryptionKey: string }) => {
+      const formData = new FormData();
+
+      formData.append("decryptionKey", decryptionKey);
+      formData.append("file", file);
+
+      try {
+        const response = await apiRequest.post("/api/v3/migrate/env-key/", formData, {
+          headers: {
+            "Content-Type": "multipart/form-data"
+          },
+          onUploadProgress: (progressEvent) => {
+            const percentCompleted = Math.round((progressEvent.loaded * 100) / progressEvent.total);
+            console.log(`Upload Progress: ${percentCompleted}%`);
+          }
+        });
+
+        console.log("Upload successful:", response.data);
+      } catch (error) {
+        console.error("Upload failed:", error);
+      }
     },
     onSuccess: () => {
       queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);

frontend/src/hooks/api/organization/queries.tsx

@@ -82,12 +82,13 @@ export const useCreateOrg = (options: { invalidate: boolean } = { invalidate: tr
 export const useUpdateOrg = () => {
   const queryClient = useQueryClient();
   return useMutation<{}, {}, UpdateOrgDTO>({
-    mutationFn: ({ name, authEnforced, scimEnabled, slug, orgId }) => {
+    mutationFn: ({ name, authEnforced, scimEnabled, slug, orgId, defaultMembershipRoleSlug }) => {
       return apiRequest.patch(`/api/v1/organization/${orgId}`, {
         name,
         authEnforced,
         scimEnabled,
-        slug
+        slug,
+        defaultMembershipRoleSlug
       });
     },
     onSuccess: () => {

frontend/src/hooks/api/organization/types.ts

@@ -10,6 +10,7 @@ export type Organization = {
   orgAuthMethod: string;
   scimEnabled: boolean;
   slug: string;
+  defaultMembershipRole: string;
 };
 
 export type UpdateOrgDTO = {
@@ -18,6 +19,7 @@ export type UpdateOrgDTO = {
   authEnforced?: boolean;
   scimEnabled?: boolean;
   slug?: string;
+  defaultMembershipRoleSlug?: string;
 };
 
 export type BillingDetails = {

frontend/src/views/Org/AuditLogsPage/AuditLogsPage.tsx

@@ -1,4 +1,3 @@
-import { NoticeBanner } from "@app/components/v2";
 import { OrgPermissionActions, OrgPermissionSubjects } from "@app/context";
 import { withPermission } from "@app/hoc";
 
@@ -11,15 +10,9 @@ export const AuditLogsPage = withPermission(
         <div className="w-full max-w-7xl px-6">
           <div className="bg-bunker-800 py-6">
             <p className="text-3xl font-semibold text-gray-200">Audit Logs</p>
-            {(window.location.origin.includes("https://app.infisical.com") ||
-              window.location.origin.includes("https://gamma.infisical.com")) && (
-              <NoticeBanner title="The audit logs page is in maintenance" className="mt-4">
-                We are currently working on improving the performance of audit log queries. During this time, querying logs is temporarily disabled. However, audit logs are still being generated as usual, so there is no disruption to log collection.
-              </NoticeBanner>
-            )}
             <div />
           </div>
-          {!window.location.origin.includes("https://app.infisical.com") && <LogsSection filterClassName="static p-2" showFilters isOrgAuditLogs />}
+          <LogsSection filterClassName="static p-2" showFilters isOrgAuditLogs />
         </div>
       </div>
     );

frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx

@@ -573,7 +573,7 @@ export const LogsTableRow = ({ auditLog, isOrgAuditLogs, showActorColumn }: Prop
     <Tr className={`log-${auditLog.id} h-10 border-x-0 border-b border-t-0`}>
       <Td>{formatDate(auditLog.createdAt)}</Td>
       <Td>{`${eventToNameMap[auditLog.event.type]}`}</Td>
-      {isOrgAuditLogs && <Td>{auditLog?.project?.name ?? "N/A"}</Td>}
+      {isOrgAuditLogs && <Td>{auditLog?.projectName ?? auditLog?.projectId ?? "N/A"}</Td>}
       {showActorColumn && renderActor(auditLog.actor)}
       {renderSource()}
       {renderMetadata(auditLog.event)}

frontend/src/views/Org/MembersPage/components/OrgMembersTab/components/OrgMembersSection/AddOrgMemberModal.tsx

@@ -1,3 +1,4 @@
+import { useEffect } from "react";
 import { Controller, useForm } from "react-hook-form";
 import {
   faCheckCircle,
@@ -33,6 +34,7 @@ import {
 import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
 import { ProjectVersion } from "@app/hooks/api/workspace/types";
 import { UsePopUpState } from "@app/hooks/usePopUp";
+import { isCustomOrgRole } from "@app/views/Org/MembersPage/components/OrgRoleTabSection/OrgRoleTable";
 
 import { OrgInviteLink } from "./OrgInviteLink";
 
@@ -78,7 +80,20 @@ export const AddOrgMemberModal = ({
     watch,
     reset,
     formState: { isSubmitting }
-  } = useForm<TAddMemberForm>({ resolver: zodResolver(addMemberFormSchema) });
+  } = useForm<TAddMemberForm>({
+    resolver: zodResolver(addMemberFormSchema)
+  });
+
+  // set initial form role based off org default role
+  useEffect(() => {
+    if (organizationRoles) {
+      reset({
+        organizationRoleSlug: isCustomOrgRole(currentOrg?.defaultMembershipRole!)
+          ? organizationRoles?.find((role) => role.id === currentOrg?.defaultMembershipRole)?.slug!
+          : currentOrg?.defaultMembershipRole
+      });
+    }
+  }, [organizationRoles]);
 
   const selectedProjectIds = watch("projectIds", []);
 
@@ -207,7 +222,6 @@ export const AddOrgMemberModal = ({
                   <div>
                     <Select
                       className="w-full"
-                      defaultValue={DEFAULT_ORG_AND_PROJECT_MEMBER_ROLE_SLUG}
                       {...field}
                       onValueChange={(val) => field.onChange(val)}
                     >

frontend/src/views/Org/MembersPage/components/OrgRoleTabSection/OrgRoleTable.tsx

@@ -6,6 +6,7 @@ import { twMerge } from "tailwind-merge";
 import { createNotification } from "@app/components/notifications";
 import { OrgPermissionCan } from "@app/components/permissions";
 import {
+  Badge,
   Button,
   DeleteActionModal,
   DropdownMenu,
@@ -19,14 +20,30 @@ import {
   Td,
   Th,
   THead,
-  Tr
+  Tooltip,
+  Tr,
+  UpgradePlanModal
 } from "@app/components/v2";
-import { OrgPermissionActions, OrgPermissionSubjects, useOrganization } from "@app/context";
+import {
+  OrgPermissionActions,
+  OrgPermissionSubjects,
+  useOrganization,
+  useSubscription
+} from "@app/context";
 import { usePopUp } from "@app/hooks";
-import { useDeleteOrgRole, useGetOrgRoles } from "@app/hooks/api";
+import { useDeleteOrgRole, useGetOrgRoles, useUpdateOrg } from "@app/hooks/api";
 import { TOrgRole } from "@app/hooks/api/roles/types";
 import { RoleModal } from "@app/views/Org/RolePage/components";
 
+enum OrgMembershipRole {
+  Admin = "admin",
+  Member = "member",
+  NoAccess = "no-access"
+}
+
+export const isCustomOrgRole = (slug: string) =>
+  !Object.values(OrgMembershipRole).includes(slug as OrgMembershipRole);
+
 export const OrgRoleTable = () => {
   const router = useRouter();
   const { currentOrg } = useOrganization();
@@ -34,12 +51,14 @@ export const OrgRoleTable = () => {
 
   const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([
     "role",
-    "deleteRole"
+    "deleteRole",
+    "upgradePlan"
   ] as const);
 
   const { data: roles, isLoading: isRolesLoading } = useGetOrgRoles(orgId);
-
   const { mutateAsync: deleteRole } = useDeleteOrgRole();
+  const { mutateAsync: updateOrg } = useUpdateOrg();
+  const { subscription } = useSubscription();
 
   const handleRoleDelete = async () => {
     const { id } = popUp?.deleteRole?.data as TOrgRole;
@@ -56,6 +75,30 @@ export const OrgRoleTable = () => {
     }
   };
 
+  const handleSetRoleAsDefault = async (defaultMembershipRoleSlug: string) => {
+    const isCustomRole = isCustomOrgRole(defaultMembershipRoleSlug);
+
+    if (isCustomRole && subscription && !subscription?.rbac) {
+      handlePopUpOpen("upgradePlan", {
+        description:
+          "You can set the default org role to a custom role if you upgrade your Infisical plan."
+      });
+      return;
+    }
+
+    try {
+      await updateOrg({
+        orgId,
+        defaultMembershipRoleSlug
+      });
+      createNotification({ type: "success", text: "Successfully updated default membership role" });
+      handlePopUpClose("deleteRole");
+    } catch (err) {
+      console.log(err);
+      createNotification({ type: "error", text: "Failed to update default membership role" });
+    }
+  };
+
   return (
     <div className="rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
       <div className="mb-4 flex justify-between">
@@ -90,14 +133,30 @@ export const OrgRoleTable = () => {
             {roles?.map((role) => {
               const { id, name, slug } = role;
               const isNonMutatable = ["owner", "admin", "member", "no-access"].includes(slug);
+              const isDefaultOrgRole = isCustomOrgRole(slug)
+                ? id === currentOrg?.defaultMembershipRole
+                : slug === currentOrg?.defaultMembershipRole;
               return (
                 <Tr
                   key={`role-list-${id}`}
                   className="h-10 cursor-pointer transition-colors duration-100 hover:bg-mineshaft-700"
                   onClick={() => router.push(`/org/${orgId}/roles/${id}`)}
                 >
-                  <Td className="max-w-md overflow-hidden text-ellipsis whitespace-nowrap">
-                    {name}
+                  <Td className="max-w-md">
+                    <div className="flex">
+                      <p className="overflow-hidden text-ellipsis whitespace-nowrap">{name}</p>
+                      {isDefaultOrgRole && (
+                        <Tooltip
+                          content={`Members joining your organization will be assigned the ${name} role unless otherwise specified.`}
+                        >
+                          <div>
+                            <Badge variant="success" className="ml-1">
+                              Default
+                            </Badge>
+                          </div>
+                        </Tooltip>
+                      )}
+                    </div>
                   </Td>
                   <Td className="max-w-md overflow-hidden text-ellipsis whitespace-nowrap">
                     {slug}
@@ -129,29 +188,61 @@ export const OrgRoleTable = () => {
                             </DropdownMenuItem>
                           )}
                         </OrgPermissionCan>
-                        {!isNonMutatable && (
+                        {!isDefaultOrgRole && (
                           <OrgPermissionCan
-                            I={OrgPermissionActions.Delete}
-                            a={OrgPermissionSubjects.Role}
+                            I={OrgPermissionActions.Edit}
+                            a={OrgPermissionSubjects.Settings}
                           >
                             {(isAllowed) => (
                               <DropdownMenuItem
                                 className={twMerge(
-                                  isAllowed
-                                    ? "hover:!bg-red-500 hover:!text-white"
-                                    : "pointer-events-none cursor-not-allowed opacity-50"
+                                  !isAllowed && "pointer-events-none cursor-not-allowed opacity-50"
                                 )}
+                                disabled={!isAllowed}
                                 onClick={(e) => {
                                   e.stopPropagation();
-                                  handlePopUpOpen("deleteRole", role);
+                                  handleSetRoleAsDefault(slug);
                                 }}
-                                disabled={!isAllowed}
                               >
-                                Delete Role
+                                Set as Default Role
                               </DropdownMenuItem>
                             )}
                           </OrgPermissionCan>
                         )}
+                        {!isNonMutatable && (
+                          <Tooltip
+                            position="left"
+                            content={
+                              isDefaultOrgRole
+                                ? "Cannot delete default organization membership role. Re-assign default to allow deletion."
+                                : ""
+                            }
+                          >
+                            <div>
+                              <OrgPermissionCan
+                                I={OrgPermissionActions.Delete}
+                                a={OrgPermissionSubjects.Role}
+                              >
+                                {(isAllowed) => (
+                                  <DropdownMenuItem
+                                    className={twMerge(
+                                      isAllowed && !isDefaultOrgRole
+                                        ? "hover:!bg-red-500 hover:!text-white"
+                                        : "pointer-events-none cursor-not-allowed opacity-50"
+                                    )}
+                                    onClick={(e) => {
+                                      e.stopPropagation();
+                                      handlePopUpOpen("deleteRole", role);
+                                    }}
+                                    disabled={!isAllowed || isDefaultOrgRole}
+                                  >
+                                    Delete Role
+                                  </DropdownMenuItem>
+                                )}
+                              </OrgPermissionCan>
+                            </div>
+                          </Tooltip>
+                        )}
                       </DropdownMenuContent>
                     </DropdownMenu>
                   </Td>
@@ -172,6 +263,11 @@ export const OrgRoleTable = () => {
         onClose={() => handlePopUpClose("deleteRole")}
         onDeleteApproved={handleRoleDelete}
       />
+      <UpgradePlanModal
+        isOpen={popUp.upgradePlan.isOpen}
+        onOpenChange={(isOpen) => handlePopUpToggle("upgradePlan", isOpen)}
+        text={(popUp.upgradePlan?.data as { description: string })?.description}
+      />
     </div>
   );
 };