Merge pull request #1510 from akhilmhdh/fix/audit-log-queue

fix(server): auditlog won't push if retention period is zero

.env.example

@@ -19,10 +19,6 @@ POSTGRES_DB=infisical
 # Redis
 REDIS_URL=redis://redis:6379
 
-# Optional credentials for MongoDB container instance and Mongo-Express
-MONGO_USERNAME=root
-MONGO_PASSWORD=example
-
 # Website URL
 # Required
 SITE_URL=http://localhost:8080

.github/values.yaml

@@ -13,11 +13,10 @@ fullnameOverride: ""
 ##
 
 infisical:
-  ## @param backend.enabled Enable backend
-  ##
+  autoDatabaseSchemaMigration: false
+  
   enabled: false
-  ## @param backend.name Backend name
-  ##
+
   name: infisical
   replicaCount: 3
   image:
@@ -50,3 +49,9 @@ ingress:
     - secretName: letsencrypt-prod
       hosts:
         - gamma.infisical.com
+
+postgresql:
+  enabled: false
+
+redis:
+  enabled: false

.github/workflows/check-api-for-breaking-changes.yml

@@ -72,4 +72,4 @@ jobs:
         run: |
           docker-compose -f "docker-compose.dev.yml" down
           docker stop infisical-api
-          docker remove infisical-api
+          docker remove infisical-api

.github/workflows/check-fe-ts-and-lint.yml

@@ -1,4 +1,4 @@
-name: Check Frontend Pull Request
+name: Check Frontend Type and Lint check
 
 on:
   pull_request:
@@ -10,8 +10,8 @@ on:
       - "frontend/.eslintrc.js"
 
 jobs:
-  check-fe-pr:
-    name: Check
+  check-fe-ts-lint:
+    name: Check Frontend Type and Lint check
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
@@ -25,12 +25,11 @@ jobs:
           cache: "npm"
           cache-dependency-path: frontend/package-lock.json
       - name: 📦 Install dependencies
-        run: npm ci --only-production --ignore-scripts
+        run: npm install
         working-directory: frontend
-      # -
-      #   name: 🧪 Run tests
-      #   run: npm run test:ci
-      #   working-directory: frontend
-      - name: 🏗️ Run build
-        run: npm run build
+      - name: 🏗️ Run Type check
+        run: npm run type:check 
+        working-directory: frontend
+      - name: 🏗️ Run Link check
+        run: npm run lint:fix 
         working-directory: frontend

.github/workflows/run-backend-tests.yml

@@ -44,4 +44,4 @@ jobs:
           ENCRYPTION_KEY: 4bnfe4e407b8921c104518903515b218
       - name: cleanup
         run: |
-          docker-compose -f "docker-compose.dev.yml" down
+          docker-compose -f "docker-compose.dev.yml" down

.gitignore

@@ -63,3 +63,5 @@ yarn-error.log*
 .vscode/*
 
 frontend-build
+
+*.tgz

CONTRIBUTING.md

@@ -2,6 +2,6 @@
 
 Thanks for taking the time to contribute! 😃 🚀
 
-Please refer to our [Contributing Guide](https://infisical.com/docs/contributing/overview) for instructions on how to contribute.
+Please refer to our [Contributing Guide](https://infisical.com/docs/contributing/getting-started/overview) for instructions on how to contribute.
 
 We also have some 🔥amazing🔥 merch for our contributors. Please reach out to tony@infisical.com for more info 👀

Makefile

@@ -11,4 +11,4 @@ up-prod:
 	docker-compose -f docker-compose.prod.yml up --build
 
 down:
-	docker-compose down
+	docker compose -f docker-compose.dev.yml down

backend/e2e-test/mocks/keystore.ts

@@ -0,0 +1,30 @@
+import { TKeyStoreFactory } from "@app/keystore/keystore";
+
+export const mockKeyStore = (): TKeyStoreFactory => {
+  const store: Record<string, string | number | Buffer> = {};
+
+  return {
+    setItem: async (key, value) => {
+      store[key] = value;
+      return "OK";
+    },
+    setItemWithExpiry: async (key, value) => {
+      store[key] = value;
+      return "OK";
+    },
+    deleteItem: async (key) => {
+      delete store[key];
+      return 1;
+    },
+    getItem: async (key) => {
+      const value = store[key];
+      if (typeof value === "string") {
+        return value;
+      }
+      return null;
+    },
+    incrementBy: async () => {
+      return 1;
+    }
+  };
+};

backend/e2e-test/vitest-environment-knex.ts

@@ -14,6 +14,7 @@ import { AuthTokenType } from "@app/services/auth/auth-type";
 
 import { mockQueue } from "./mocks/queue";
 import { mockSmtpServer } from "./mocks/smtp";
+import { mockKeyStore } from "./mocks/keystore";
 
 dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true });
 export default {
@@ -41,7 +42,8 @@ export default {
       await db.seed.run();
       const smtp = mockSmtpServer();
       const queue = mockQueue();
-      const server = await main({ db, smtp, logger, queue });
+      const keyStore = mockKeyStore();
+      const server = await main({ db, smtp, logger, queue, keyStore });
       // @ts-expect-error type
       globalThis.testServer = server;
       // @ts-expect-error type
@@ -56,6 +58,7 @@ export default {
         { expiresIn: cfg.JWT_AUTH_LIFETIME }
       );
     } catch (error) {
+      console.log("[TEST] Error setting up environment", error);
       await db.destroy();
       throw error;
     }

backend/package-lock.json

@@ -9,17 +9,17 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
-        "@aws-sdk/client-secrets-manager": "^3.502.0",
+        "@aws-sdk/client-secrets-manager": "^3.504.0",
         "@casl/ability": "^6.5.0",
-        "@fastify/cookie": "^9.2.0",
-        "@fastify/cors": "^8.4.1",
+        "@fastify/cookie": "^9.3.1",
+        "@fastify/cors": "^8.5.0",
         "@fastify/etag": "^5.1.0",
         "@fastify/formbody": "^7.4.0",
         "@fastify/helmet": "^11.1.1",
         "@fastify/passport": "^2.4.0",
         "@fastify/rate-limit": "^9.0.0",
         "@fastify/session": "^10.7.0",
-        "@fastify/swagger": "^8.12.0",
+        "@fastify/swagger": "^8.14.0",
         "@fastify/swagger-ui": "^2.1.0",
         "@node-saml/passport-saml": "^4.0.4",
         "@octokit/rest": "^20.0.2",
@@ -29,13 +29,13 @@
         "@ucast/mongo2js": "^1.3.4",
         "ajv": "^8.12.0",
         "argon2": "^0.31.2",
-        "aws-sdk": "^2.1545.0",
-        "axios": "^1.6.4",
+        "aws-sdk": "^2.1553.0",
+        "axios": "^1.6.7",
         "axios-retry": "^4.0.0",
         "bcrypt": "^5.1.1",
-        "bullmq": "^5.1.1",
-        "dotenv": "^16.3.1",
-        "fastify": "^4.24.3",
+        "bullmq": "^5.3.3",
+        "dotenv": "^16.4.1",
+        "fastify": "^4.26.0",
         "fastify-plugin": "^4.5.1",
         "handlebars": "^4.7.8",
         "ioredis": "^5.3.2",
@@ -45,7 +45,7 @@
         "knex": "^3.0.1",
         "libsodium-wrappers": "^0.7.13",
         "lodash.isequal": "^4.5.0",
-        "mysql2": "^3.6.5",
+        "mysql2": "^3.9.1",
         "nanoid": "^5.0.4",
         "node-cache": "^5.1.2",
         "nodemailer": "^6.9.9",
@@ -56,14 +56,14 @@
         "pg": "^8.11.3",
         "picomatch": "^3.0.1",
         "pino": "^8.16.2",
-        "posthog-node": "^3.6.0",
+        "posthog-node": "^3.6.2",
         "probot": "^13.0.0",
         "smee-client": "^2.0.0",
         "tweetnacl": "^1.0.3",
         "tweetnacl-util": "^0.15.1",
         "uuid": "^9.0.1",
         "zod": "^3.22.4",
-        "zod-to-json-schema": "^3.22.0"
+        "zod-to-json-schema": "^3.22.4"
       },
       "devDependencies": {
         "@types/bcrypt": "^5.0.2",
@@ -661,15 +661,15 @@
       }
     },
     "node_modules/@aws-sdk/client-secrets-manager": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.502.0.tgz",
-      "integrity": "sha512-ICU084A/EbYMqca6NVFqeMtHh+KCdn0H7UjARUy5ur1yOlXXvxqAJGtKZDYFjuEO08F30zbv7+4HCOy6yjOJ0Q==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.504.0.tgz",
+      "integrity": "sha512-JPwsYfQMjs5t74JmA4r1AjpiOG/LEw74d4a8vEdSy3pe2lhl/sSsxSdQtbI30wlJJramngtLNZjxn2+BGDphbg==",
       "dependencies": {
         "@aws-crypto/sha256-browser": "3.0.0",
         "@aws-crypto/sha256-js": "3.0.0",
-        "@aws-sdk/client-sts": "3.502.0",
+        "@aws-sdk/client-sts": "3.504.0",
         "@aws-sdk/core": "3.496.0",
-        "@aws-sdk/credential-provider-node": "3.502.0",
+        "@aws-sdk/credential-provider-node": "3.504.0",
         "@aws-sdk/middleware-host-header": "3.502.0",
         "@aws-sdk/middleware-logger": "3.502.0",
         "@aws-sdk/middleware-recursion-detection": "3.502.0",
@@ -767,13 +767,13 @@
       }
     },
     "node_modules/@aws-sdk/client-sso-oidc": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.502.0.tgz",
-      "integrity": "sha512-Yc9tZqTOMWtdgpkrdjKShgWb9oKNsFQrItfoiN1xWDllaFFRPi2KTiZiR0AbSTrNasJy13d210DOxrIdte+kWQ==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.504.0.tgz",
+      "integrity": "sha512-ODA33/nm2srhV08EW0KZAP577UgV0qjyr7Xp2yEo8MXWL4ZqQZprk1c+QKBhjr4Djesrm0VPmSD/np0mtYP68A==",
       "dependencies": {
         "@aws-crypto/sha256-browser": "3.0.0",
         "@aws-crypto/sha256-js": "3.0.0",
-        "@aws-sdk/client-sts": "3.502.0",
+        "@aws-sdk/client-sts": "3.504.0",
         "@aws-sdk/core": "3.496.0",
         "@aws-sdk/middleware-host-header": "3.502.0",
         "@aws-sdk/middleware-logger": "3.502.0",
@@ -815,13 +815,13 @@
         "node": ">=14.0.0"
       },
       "peerDependencies": {
-        "@aws-sdk/credential-provider-node": "*"
+        "@aws-sdk/credential-provider-node": "^3.504.0"
       }
     },
     "node_modules/@aws-sdk/client-sts": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.502.0.tgz",
-      "integrity": "sha512-0q08gsvn6nuRqjK+i/e30PT/t7vvYwmGJS0PhJikZWv5yRDNSUxSYG0uDwKSbLDzmc2UX5+mLeyjPHlL4hbGlA==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.504.0.tgz",
+      "integrity": "sha512-IESs8FkL7B/uY+ml4wgoRkrr6xYo4PizcNw6JX17eveq1gRBCPKeGMjE6HTDOcIYZZ8rqz/UeuH3JD4UhrMOnA==",
       "dependencies": {
         "@aws-crypto/sha256-browser": "3.0.0",
         "@aws-crypto/sha256-js": "3.0.0",
@@ -867,7 +867,7 @@
         "node": ">=14.0.0"
       },
       "peerDependencies": {
-        "@aws-sdk/credential-provider-node": "*"
+        "@aws-sdk/credential-provider-node": "^3.504.0"
       }
     },
     "node_modules/@aws-sdk/core": {
@@ -900,16 +900,35 @@
         "node": ">=14.0.0"
       }
     },
-    "node_modules/@aws-sdk/credential-provider-ini": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.502.0.tgz",
-      "integrity": "sha512-1wB/escbspUY6uRDEMp9AMMyypUSyuQ0AMO1yQNtXviV8cPf+CuRbqP/UVnimHO1RuX0n5BmjDVVjUIEU6kuGA==",
+    "node_modules/@aws-sdk/credential-provider-http": {
+      "version": "3.503.1",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.503.1.tgz",
+      "integrity": "sha512-rTdlFFGoPPFMF2YjtlfRuSgKI+XsF49u7d98255hySwhsbwd3Xp+utTTPquxP+CwDxMHbDlI7NxDzFiFdsoZug==",
       "dependencies": {
-        "@aws-sdk/client-sts": "3.502.0",
+        "@aws-sdk/types": "3.502.0",
+        "@smithy/fetch-http-handler": "^2.4.1",
+        "@smithy/node-http-handler": "^2.3.1",
+        "@smithy/property-provider": "^2.1.1",
+        "@smithy/protocol-http": "^3.1.1",
+        "@smithy/smithy-client": "^2.3.1",
+        "@smithy/types": "^2.9.1",
+        "@smithy/util-stream": "^2.1.1",
+        "tslib": "^2.5.0"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@aws-sdk/credential-provider-ini": {
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.504.0.tgz",
+      "integrity": "sha512-ODICLXfr8xTUd3wweprH32Ge41yuBa+u3j0JUcLdTUO1N9ldczSMdo8zOPlP0z4doqD3xbnqMkjNQWgN/Q+5oQ==",
+      "dependencies": {
+        "@aws-sdk/client-sts": "3.504.0",
         "@aws-sdk/credential-provider-env": "3.502.0",
         "@aws-sdk/credential-provider-process": "3.502.0",
-        "@aws-sdk/credential-provider-sso": "3.502.0",
-        "@aws-sdk/credential-provider-web-identity": "3.502.0",
+        "@aws-sdk/credential-provider-sso": "3.504.0",
+        "@aws-sdk/credential-provider-web-identity": "3.504.0",
         "@aws-sdk/types": "3.502.0",
         "@smithy/credential-provider-imds": "^2.2.1",
         "@smithy/property-provider": "^2.1.1",
@@ -922,15 +941,16 @@
       }
     },
     "node_modules/@aws-sdk/credential-provider-node": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.502.0.tgz",
-      "integrity": "sha512-qg71UpYeFrjhu5hD+vdRqZ+EYFB11BeszsbfEJGaHhOMHmmTHNBaDAexW+bUnJSXcJL0a8vniCvca+rElbcAHQ==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.504.0.tgz",
+      "integrity": "sha512-6+V5hIh+tILmUjf2ZQWQINR3atxQVgH/bFrGdSR/sHSp/tEgw3m0xWL3IRslWU1e4/GtXrfg1iYnMknXy68Ikw==",
       "dependencies": {
         "@aws-sdk/credential-provider-env": "3.502.0",
-        "@aws-sdk/credential-provider-ini": "3.502.0",
+        "@aws-sdk/credential-provider-http": "3.503.1",
+        "@aws-sdk/credential-provider-ini": "3.504.0",
         "@aws-sdk/credential-provider-process": "3.502.0",
-        "@aws-sdk/credential-provider-sso": "3.502.0",
-        "@aws-sdk/credential-provider-web-identity": "3.502.0",
+        "@aws-sdk/credential-provider-sso": "3.504.0",
+        "@aws-sdk/credential-provider-web-identity": "3.504.0",
         "@aws-sdk/types": "3.502.0",
         "@smithy/credential-provider-imds": "^2.2.1",
         "@smithy/property-provider": "^2.1.1",
@@ -958,12 +978,12 @@
       }
     },
     "node_modules/@aws-sdk/credential-provider-sso": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.502.0.tgz",
-      "integrity": "sha512-/2Nyvo+cWQpH283lmZBimTJ9JDhES9FzQUkhUXZgxQo3Ez4sguLVi2V9xoFFyG0cMff5fuNivdKHfj4FeMGjZw==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.504.0.tgz",
+      "integrity": "sha512-4MgH2or2SjPzaxM08DCW+BjaX4DSsEGJlicHKmz6fh+w9JmLh750oXcTnbvgUeVz075jcs6qTKjvUcsdGM/t8Q==",
       "dependencies": {
         "@aws-sdk/client-sso": "3.502.0",
-        "@aws-sdk/token-providers": "3.502.0",
+        "@aws-sdk/token-providers": "3.504.0",
         "@aws-sdk/types": "3.502.0",
         "@smithy/property-provider": "^2.1.1",
         "@smithy/shared-ini-file-loader": "^2.3.1",
@@ -975,11 +995,11 @@
       }
     },
     "node_modules/@aws-sdk/credential-provider-web-identity": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.502.0.tgz",
-      "integrity": "sha512-veBAjDqjMMgA2Qxxf9ywDfHYLeJpaeHWLWCQ9XCHwJJ6ZIGWmAZPTq3he/UMr5JIQXooIccqqyqXMDIXPenXpA==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.504.0.tgz",
+      "integrity": "sha512-L1ljCvGpIEFdJk087ijf2ohg7HBclOeB1UgBxUBBzf4iPRZTQzd2chGaKj0hm2VVaXz7nglswJeURH5PFcS5oA==",
       "dependencies": {
-        "@aws-sdk/client-sts": "3.502.0",
+        "@aws-sdk/client-sts": "3.504.0",
         "@aws-sdk/types": "3.502.0",
         "@smithy/property-provider": "^2.1.1",
         "@smithy/types": "^2.9.1",
@@ -1079,11 +1099,11 @@
       }
     },
     "node_modules/@aws-sdk/token-providers": {
-      "version": "3.502.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.502.0.tgz",
-      "integrity": "sha512-RQgMgIXYlSf0xGl6EUeD+pqIPBlb7e29dbqHOBFc66hJVYUC2ULZX7Y+jLvcGIEaMiIaTPyvntZRFip+U+9hag==",
+      "version": "3.504.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.504.0.tgz",
+      "integrity": "sha512-YIJWWsZi2ClUiILS1uh5L6VjmCUSTI6KKMuL9DkGjYqJ0aI6M8bd8fT9Wm7QmXCyjcArTgr/Atkhia4T7oKvzQ==",
       "dependencies": {
-        "@aws-sdk/client-sso-oidc": "3.502.0",
+        "@aws-sdk/client-sso-oidc": "3.504.0",
         "@aws-sdk/types": "3.502.0",
         "@smithy/property-provider": "^2.1.1",
         "@smithy/shared-ini-file-loader": "^2.3.1",
@@ -1667,21 +1687,21 @@
       }
     },
     "node_modules/@fastify/cookie": {
-      "version": "9.2.0",
-      "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.2.0.tgz",
-      "integrity": "sha512-fkg1yjjQRHPFAxSHeLC8CqYuNzvR6Lwlj/KjrzQcGjNBK+K82nW+UfCjfN71g1GkoVoc1GTOgIWkFJpcMfMkHQ==",
+      "version": "9.3.1",
+      "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz",
+      "integrity": "sha512-h1NAEhB266+ZbZ0e9qUE6NnNR07i7DnNXWG9VbbZ8uC6O/hxHpl+Zoe5sw1yfdZ2U6XhToUGDnzQtWJdCaPwfg==",
       "dependencies": {
         "cookie-signature": "^1.1.0",
         "fastify-plugin": "^4.0.0"
       }
     },
     "node_modules/@fastify/cors": {
-      "version": "8.4.1",
-      "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-8.4.1.tgz",
-      "integrity": "sha512-iYQJtrY3pFiDS5mo5zRaudzg2OcUdJ96PD6xfkKOOEilly5nnrFZx/W6Sce2T79xxlEn2qpU3t5+qS2phS369w==",
+      "version": "8.5.0",
+      "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-8.5.0.tgz",
+      "integrity": "sha512-/oZ1QSb02XjP0IK1U0IXktEsw/dUBTxJOW7IpIeO8c/tNalw/KjoNSJv1Sf6eqoBPO+TDGkifq6ynFK3v68HFQ==",
       "dependencies": {
         "fastify-plugin": "^4.0.0",
-        "mnemonist": "0.39.5"
+        "mnemonist": "0.39.6"
       }
     },
     "node_modules/@fastify/deepmerge": {
@@ -1790,9 +1810,9 @@
       }
     },
     "node_modules/@fastify/swagger": {
-      "version": "8.12.0",
-      "resolved": "https://registry.npmjs.org/@fastify/swagger/-/swagger-8.12.0.tgz",
-      "integrity": "sha512-IMRc0xYuzRvtFDMuaWHyVbvM7CuAi0g3o2jaVgLDvETXPrXWAMWsHYR5niIdWBDPgGUq+soHkag1DKXyhPDB0w==",
+      "version": "8.14.0",
+      "resolved": "https://registry.npmjs.org/@fastify/swagger/-/swagger-8.14.0.tgz",
+      "integrity": "sha512-sGiznEb3rl6pKGGUZ+JmfI7ct5cwbTQGo+IjewaTvtzfrshnryu4dZwEsjw0YHABpBA+kCz3kpRaHB7qpa67jg==",
       "dependencies": {
         "fastify-plugin": "^4.0.0",
         "json-schema-resolver": "^2.0.0",
@@ -2173,7 +2193,6 @@
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
       "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
-      "dev": true,
       "dependencies": {
         "@nodelib/fs.stat": "2.0.5",
         "run-parallel": "^1.1.9"
@@ -2186,7 +2205,6 @@
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
       "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
-      "dev": true,
       "engines": {
         "node": ">= 8"
       }
@@ -2195,7 +2213,6 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
       "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
-      "dev": true,
       "dependencies": {
         "@nodelib/fs.scandir": "2.1.5",
         "fastq": "^1.6.0"
@@ -5169,9 +5186,9 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/aws-sdk": {
-      "version": "2.1545.0",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1545.0.tgz",
-      "integrity": "sha512-iDUv6ksG7lTA0l/HlOgYdO6vfYFA1D2/JzAEXSdgKY0C901WgJqBtfs2CncOkCgDe2CjmlMuqciBzAfxCIiKFA==",
+      "version": "2.1553.0",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1553.0.tgz",
+      "integrity": "sha512-CfZaw8dR9e642aBOeFhkFL7KoQApeLR15uH2IQqfL/12snWYayAAesYh0tEaU+XbhrH0CUsf2Zro5IraEXEZMg==",
       "dependencies": {
         "buffer": "4.9.2",
         "events": "1.1.1",
@@ -5250,9 +5267,9 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.6.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.4.tgz",
-      "integrity": "sha512-heJnIs6N4aa1eSthhN9M5ioILu8Wi8vmQW9iHQ9NUvfkJb0lEEDUiIdQNAuBtfUt3FxReaKdpQA5DbmMOqzF/A==",
+      "version": "1.6.7",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.7.tgz",
+      "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==",
       "dependencies": {
         "follow-redirects": "^1.15.4",
         "form-data": "^4.0.0",
@@ -5422,7 +5439,6 @@
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
       "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
-      "dev": true,
       "dependencies": {
         "fill-range": "^7.0.1"
       },
@@ -5472,21 +5488,44 @@
       }
     },
     "node_modules/bullmq": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.1.1.tgz",
-      "integrity": "sha512-j3zbNEQWsyHjpqGWiem2XBfmxAjYcArbwsmGlkM1E9MAVcrqB5hQUsXmyy9gEBAdL+PVotMICr7xTquR4Y2sKQ==",
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.3.3.tgz",
+      "integrity": "sha512-Gc/68HxiCHLMPBiGIqtINxcf8HER/5wvBYMY/6x3tFejlvldUBFaAErMTLDv4TnPsTyzNPrfBKmFCEM58uVnJg==",
       "dependencies": {
         "cron-parser": "^4.6.0",
-        "glob": "^8.0.3",
+        "fast-glob": "^3.3.2",
         "ioredis": "^5.3.2",
         "lodash": "^4.17.21",
-        "msgpackr": "^1.6.2",
+        "minimatch": "^9.0.3",
+        "msgpackr": "^1.10.1",
         "node-abort-controller": "^3.1.1",
         "semver": "^7.5.4",
         "tslib": "^2.0.0",
         "uuid": "^9.0.0"
       }
     },
+    "node_modules/bullmq/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/bullmq/node_modules/minimatch": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
+      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/bundle-require": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-4.0.2.tgz",
@@ -5995,9 +6034,9 @@
       }
     },
     "node_modules/dotenv": {
-      "version": "16.3.1",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
-      "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==",
+      "version": "16.4.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.1.tgz",
+      "integrity": "sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==",
       "engines": {
         "node": ">=12"
       },
@@ -6886,7 +6925,6 @@
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
       "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==",
-      "dev": true,
       "dependencies": {
         "@nodelib/fs.stat": "^2.0.2",
         "@nodelib/fs.walk": "^1.2.3",
@@ -6972,9 +7010,19 @@
       }
     },
     "node_modules/fastify": {
-      "version": "4.24.3",
-      "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.24.3.tgz",
-      "integrity": "sha512-6HHJ+R2x2LS3y1PqxnwEIjOTZxFl+8h4kSC/TuDPXtA+v2JnV9yEtOsNSKK1RMD7sIR2y1ZsA4BEFaid/cK5pg==",
+      "version": "4.26.0",
+      "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.26.0.tgz",
+      "integrity": "sha512-Fq/7ziWKc6pYLYLIlCRaqJqEVTIZ5tZYfcW/mDK2AQ9v/sqjGFpj0On0/7hU50kbPVjLO4de+larPA1WwPZSfw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fastify"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fastify"
+        }
+      ],
       "dependencies": {
         "@fastify/ajv-compiler": "^3.5.0",
         "@fastify/error": "^3.4.0",
@@ -6983,10 +7031,10 @@
         "avvio": "^8.2.1",
         "fast-content-type-parse": "^1.1.0",
         "fast-json-stringify": "^5.8.0",
-        "find-my-way": "^7.7.0",
+        "find-my-way": "^8.0.0",
         "light-my-request": "^5.11.0",
-        "pino": "^8.16.0",
-        "process-warning": "^2.2.0",
+        "pino": "^8.17.0",
+        "process-warning": "^3.0.0",
         "proxy-addr": "^2.0.7",
         "rfdc": "^1.3.0",
         "secure-json-parse": "^2.7.0",
@@ -6999,6 +7047,11 @@
       "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.1.tgz",
       "integrity": "sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ=="
     },
+    "node_modules/fastify/node_modules/process-warning": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz",
+      "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="
+    },
     "node_modules/fastq": {
       "version": "1.15.0",
       "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
@@ -7023,7 +7076,6 @@
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
       "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
-      "dev": true,
       "dependencies": {
         "to-regex-range": "^5.0.1"
       },
@@ -7062,9 +7114,9 @@
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
     },
     "node_modules/find-my-way": {
-      "version": "7.7.0",
-      "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.7.0.tgz",
-      "integrity": "sha512-+SrHpvQ52Q6W9f3wJoJBbAQULJuNEEQwBvlvYwACDhBTLOTMiQ0HYWh4+vC3OivGP2ENcTI1oKlFA2OepJNjhQ==",
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-8.1.0.tgz",
+      "integrity": "sha512-41QwjCGcVTODUmLLqTMeoHeiozbMXYMAE1CKFiDyi9zVZ2Vjh0yz3MF0WQZoIb+cmzP/XlbFjlF2NtJmvZHznA==",
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
         "fast-querystring": "^1.0.0",
@@ -7475,7 +7527,6 @@
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
       "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
-      "dev": true,
       "dependencies": {
         "is-glob": "^4.0.1"
       },
@@ -8076,7 +8127,6 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
       "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
-      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -8107,7 +8157,6 @@
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
       "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
-      "dev": true,
       "dependencies": {
         "is-extglob": "^2.1.1"
       },
@@ -8142,7 +8191,6 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
       "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
-      "dev": true,
       "engines": {
         "node": ">=0.12.0"
       }
@@ -8899,7 +8947,6 @@
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
       "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
-      "dev": true,
       "engines": {
         "node": ">= 8"
       }
@@ -8916,7 +8963,6 @@
       "version": "4.0.5",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
       "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
-      "dev": true,
       "dependencies": {
         "braces": "^3.0.2",
         "picomatch": "^2.3.1"
@@ -8929,7 +8975,6 @@
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
       "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
-      "dev": true,
       "engines": {
         "node": ">=8.6"
       },
@@ -9049,9 +9094,9 @@
       }
     },
     "node_modules/mnemonist": {
-      "version": "0.39.5",
-      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.5.tgz",
-      "integrity": "sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ==",
+      "version": "0.39.6",
+      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.6.tgz",
+      "integrity": "sha512-A/0v5Z59y63US00cRSLiloEIw3t5G+MiKz4BhX21FI+YBJXBOGW0ohFxTxO08dsOYlzxo87T7vGfZKYp2bcAWA==",
       "dependencies": {
         "obliterator": "^2.0.1"
       }
@@ -9112,9 +9157,9 @@
       }
     },
     "node_modules/mysql2": {
-      "version": "3.6.5",
-      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.5.tgz",
-      "integrity": "sha512-pS/KqIb0xlXmtmqEuTvBXTmLoQ5LmAz5NW/r8UyQ1ldvnprNEj3P9GbmuQQ2J0A4LO+ynotGi6TbscPa8OUb+w==",
+      "version": "3.9.1",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.1.tgz",
+      "integrity": "sha512-3njoWAAhGBYy0tWBabqUQcLtczZUxrmmtc2vszQUekg3kTJyZ5/IeLC3Fo04u6y6Iy5Sba7pIIa2P/gs8D3ZeQ==",
       "dependencies": {
         "denque": "^2.1.0",
         "generate-function": "^2.3.1",
@@ -10273,9 +10318,9 @@
       "dev": true
     },
     "node_modules/posthog-node": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.0.tgz",
-      "integrity": "sha512-N/4//SIQR4fhwbHnDdJ2rQCYdu9wo0EVPK4lVgZswp5R/E42RKlpuO6ZfPsBl+Bcg06OYiOd/WR/jLV90FCoSw==",
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.2.tgz",
+      "integrity": "sha512-tVIaShR3SxBx17AlAUS86jQTweKuJIFRedBB504fCz7YPnXJTYSrVcUHn5IINE2wu4jUQimQK6ihQr90Djrdrg==",
       "dependencies": {
         "axios": "^1.6.2",
         "rusha": "^0.8.14"
@@ -10522,7 +10567,6 @@
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
       "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
-      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -10869,7 +10913,6 @@
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
       "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
-      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -11670,7 +11713,6 @@
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
       "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-      "dev": true,
       "dependencies": {
         "is-number": "^7.0.0"
       },
@@ -13812,9 +13854,9 @@
       }
     },
     "node_modules/zod-to-json-schema": {
-      "version": "3.22.0",
-      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.22.0.tgz",
-      "integrity": "sha512-XQr8EwxPMzJGhoR+d/nRFWdi15VaZ+R5Uhssm+Xx5yS30xCpuutfKRm4rerE0SK9j2dWB5Z3FvDD0w8WMVGzkA==",
+      "version": "3.22.4",
+      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.22.4.tgz",
+      "integrity": "sha512-2Ed5dJ+n/O3cU383xSY28cuVi0BCQhF8nYqWU5paEpl7fVdqdAmiLdqLyfblbNdfOFwFfi/mqU4O1pwc60iBhQ==",
       "peerDependencies": {
         "zod": "^3.22.4"
       }

backend/package.json

@@ -70,17 +70,17 @@
     "vitest": "^1.2.2"
   },
   "dependencies": {
-    "@aws-sdk/client-secrets-manager": "^3.502.0",
+    "@aws-sdk/client-secrets-manager": "^3.504.0",
     "@casl/ability": "^6.5.0",
-    "@fastify/cookie": "^9.2.0",
-    "@fastify/cors": "^8.4.1",
+    "@fastify/cookie": "^9.3.1",
+    "@fastify/cors": "^8.5.0",
     "@fastify/etag": "^5.1.0",
     "@fastify/formbody": "^7.4.0",
     "@fastify/helmet": "^11.1.1",
     "@fastify/passport": "^2.4.0",
     "@fastify/rate-limit": "^9.0.0",
     "@fastify/session": "^10.7.0",
-    "@fastify/swagger": "^8.12.0",
+    "@fastify/swagger": "^8.14.0",
     "@fastify/swagger-ui": "^2.1.0",
     "@node-saml/passport-saml": "^4.0.4",
     "@octokit/rest": "^20.0.2",
@@ -90,13 +90,13 @@
     "@ucast/mongo2js": "^1.3.4",
     "ajv": "^8.12.0",
     "argon2": "^0.31.2",
-    "aws-sdk": "^2.1545.0",
-    "axios": "^1.6.4",
+    "aws-sdk": "^2.1553.0",
+    "axios": "^1.6.7",
     "axios-retry": "^4.0.0",
     "bcrypt": "^5.1.1",
-    "bullmq": "^5.1.1",
-    "dotenv": "^16.3.1",
-    "fastify": "^4.24.3",
+    "bullmq": "^5.3.3",
+    "dotenv": "^16.4.1",
+    "fastify": "^4.26.0",
     "fastify-plugin": "^4.5.1",
     "handlebars": "^4.7.8",
     "ioredis": "^5.3.2",
@@ -106,7 +106,7 @@
     "knex": "^3.0.1",
     "libsodium-wrappers": "^0.7.13",
     "lodash.isequal": "^4.5.0",
-    "mysql2": "^3.6.5",
+    "mysql2": "^3.9.1",
     "nanoid": "^5.0.4",
     "node-cache": "^5.1.2",
     "nodemailer": "^6.9.9",
@@ -117,13 +117,13 @@
     "pg": "^8.11.3",
     "picomatch": "^3.0.1",
     "pino": "^8.16.2",
-    "posthog-node": "^3.6.0",
+    "posthog-node": "^3.6.2",
     "probot": "^13.0.0",
     "smee-client": "^2.0.0",
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
     "uuid": "^9.0.1",
     "zod": "^3.22.4",
-    "zod-to-json-schema": "^3.22.0"
+    "zod-to-json-schema": "^3.22.4"
   }
 }

backend/scripts/generate-schema-types.ts

@@ -44,7 +44,7 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob
   if (!value || value === "null") return;
   switch (type) {
     case "uuid":
-      return;
+      return `.default("00000000-0000-0000-0000-000000000000")`;
     case "character varying": {
       if (value === "gen_random_uuid()") return;
       if (typeof value === "string" && value.includes("::")) {
@@ -100,7 +100,8 @@ const main = async () => {
       const columnName = columnNames[colNum];
       const colInfo = columns[columnName];
       let ztype = getZodPrimitiveType(colInfo.type);
-      if (colInfo.defaultValue) {
+      // don't put optional on id
+      if (colInfo.defaultValue && columnName !== "id") {
         const { defaultValue } = colInfo;
         const zSchema = getZodDefaultValue(colInfo.type, defaultValue);
         if (zSchema) {
@@ -120,6 +121,7 @@ const main = async () => {
       .split("_")
       .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, "");
 
+    // the insert and update are changed to zod input type to use default cases
     writeFileSync(
       path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`),
       `// Code generated by automation script, DO NOT EDIT.
@@ -134,8 +136,8 @@ import { TImmutableDBKeys } from "./models";
 export const ${pascalCase}Schema = z.object({${schema}});
 
 export type T${pascalCase} = z.infer<typeof ${pascalCase}Schema>;
-export type T${pascalCase}Insert = Omit<T${pascalCase}, TImmutableDBKeys>;
-export type T${pascalCase}Update = Partial<Omit<T${pascalCase}, TImmutableDBKeys>>;
+export type T${pascalCase}Insert = Omit<z.input<typeof ${pascalCase}Schema>, TImmutableDBKeys>;
+export type T${pascalCase}Update = Partial<Omit<z.input<typeof ${pascalCase}Schema>, TImmutableDBKeys>>;
 `
     );
   }

backend/src/cache/redis.ts

@@ -1,6 +0,0 @@
-import Redis from "ioredis";
-
-export const initRedisConnection = (redisUrl: string) => {
-  const redis = new Redis(redisUrl);
-  return redis;
-};

backend/src/db/knexfile.ts

@@ -17,7 +17,15 @@ dotenv.config({
 export default {
   development: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10
@@ -31,7 +39,15 @@ export default {
   },
   production: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10

backend/src/db/migrations/20240216154123_ghost_users.ts

@@ -0,0 +1,39 @@
+import { Knex } from "knex";
+
+import { ProjectVersion, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (!hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.boolean("isGhost").defaultTo(false).notNullable();
+    });
+  }
+
+  if (!hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.integer("version").defaultTo(ProjectVersion.V1).notNullable();
+      t.string("upgradeStatus").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.dropColumn("isGhost");
+    });
+  }
+
+  if (hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.dropColumn("version");
+      t.dropColumn("upgradeStatus");
+    });
+  }
+}

backend/src/db/migrations/20240222201806_admin-signup-control.ts

@@ -0,0 +1,20 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const isTablePresent = await knex.schema.hasTable(TableName.SuperAdmin);
+  if (isTablePresent) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.string("allowedSignUpDomain");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.SuperAdmin, "allowedSignUpDomain")) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.dropColumn("allowedSignUpDomain");
+    });
+  }
+}

backend/src/db/migrations/20240226094411_instance-id.ts

@@ -0,0 +1,25 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const ADMIN_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+    t.uuid("instanceId").notNullable().defaultTo(knex.fn.uuid());
+  });
+
+  const superUserConfigExists = await knex(TableName.SuperAdmin).where("id", ADMIN_CONFIG_UUID).first();
+  if (!superUserConfigExists) {
+    // eslint-disable-next-line
+    await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+    t.dropColumn("instanceId");
+  });
+}

backend/src/db/schemas/api-keys.ts

@@ -19,5 +19,5 @@ export const ApiKeysSchema = z.object({
 });
 
 export type TApiKeys = z.infer<typeof ApiKeysSchema>;
-export type TApiKeysInsert = Omit<TApiKeys, TImmutableDBKeys>;
-export type TApiKeysUpdate = Partial<Omit<TApiKeys, TImmutableDBKeys>>;
+export type TApiKeysInsert = Omit<z.input<typeof ApiKeysSchema>, TImmutableDBKeys>;
+export type TApiKeysUpdate = Partial<Omit<z.input<typeof ApiKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/audit-logs.ts

@@ -24,5 +24,5 @@ export const AuditLogsSchema = z.object({
 });
 
 export type TAuditLogs = z.infer<typeof AuditLogsSchema>;
-export type TAuditLogsInsert = Omit<TAuditLogs, TImmutableDBKeys>;
-export type TAuditLogsUpdate = Partial<Omit<TAuditLogs, TImmutableDBKeys>>;
+export type TAuditLogsInsert = Omit<z.input<typeof AuditLogsSchema>, TImmutableDBKeys>;
+export type TAuditLogsUpdate = Partial<Omit<z.input<typeof AuditLogsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/auth-token-sessions.ts

@@ -20,5 +20,5 @@ export const AuthTokenSessionsSchema = z.object({
 });
 
 export type TAuthTokenSessions = z.infer<typeof AuthTokenSessionsSchema>;
-export type TAuthTokenSessionsInsert = Omit<TAuthTokenSessions, TImmutableDBKeys>;
-export type TAuthTokenSessionsUpdate = Partial<Omit<TAuthTokenSessions, TImmutableDBKeys>>;
+export type TAuthTokenSessionsInsert = Omit<z.input<typeof AuthTokenSessionsSchema>, TImmutableDBKeys>;
+export type TAuthTokenSessionsUpdate = Partial<Omit<z.input<typeof AuthTokenSessionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/auth-tokens.ts

@@ -21,5 +21,5 @@ export const AuthTokensSchema = z.object({
 });
 
 export type TAuthTokens = z.infer<typeof AuthTokensSchema>;
-export type TAuthTokensInsert = Omit<TAuthTokens, TImmutableDBKeys>;
-export type TAuthTokensUpdate = Partial<Omit<TAuthTokens, TImmutableDBKeys>>;
+export type TAuthTokensInsert = Omit<z.input<typeof AuthTokensSchema>, TImmutableDBKeys>;
+export type TAuthTokensUpdate = Partial<Omit<z.input<typeof AuthTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/backup-private-key.ts

@@ -22,5 +22,5 @@ export const BackupPrivateKeySchema = z.object({
 });
 
 export type TBackupPrivateKey = z.infer<typeof BackupPrivateKeySchema>;
-export type TBackupPrivateKeyInsert = Omit<TBackupPrivateKey, TImmutableDBKeys>;
-export type TBackupPrivateKeyUpdate = Partial<Omit<TBackupPrivateKey, TImmutableDBKeys>>;
+export type TBackupPrivateKeyInsert = Omit<z.input<typeof BackupPrivateKeySchema>, TImmutableDBKeys>;
+export type TBackupPrivateKeyUpdate = Partial<Omit<z.input<typeof BackupPrivateKeySchema>, TImmutableDBKeys>>;

backend/src/db/schemas/git-app-install-sessions.ts

@@ -17,5 +17,5 @@ export const GitAppInstallSessionsSchema = z.object({
 });
 
 export type TGitAppInstallSessions = z.infer<typeof GitAppInstallSessionsSchema>;
-export type TGitAppInstallSessionsInsert = Omit<TGitAppInstallSessions, TImmutableDBKeys>;
-export type TGitAppInstallSessionsUpdate = Partial<Omit<TGitAppInstallSessions, TImmutableDBKeys>>;
+export type TGitAppInstallSessionsInsert = Omit<z.input<typeof GitAppInstallSessionsSchema>, TImmutableDBKeys>;
+export type TGitAppInstallSessionsUpdate = Partial<Omit<z.input<typeof GitAppInstallSessionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/git-app-org.ts

@@ -17,5 +17,5 @@ export const GitAppOrgSchema = z.object({
 });
 
 export type TGitAppOrg = z.infer<typeof GitAppOrgSchema>;
-export type TGitAppOrgInsert = Omit<TGitAppOrg, TImmutableDBKeys>;
-export type TGitAppOrgUpdate = Partial<Omit<TGitAppOrg, TImmutableDBKeys>>;
+export type TGitAppOrgInsert = Omit<z.input<typeof GitAppOrgSchema>, TImmutableDBKeys>;
+export type TGitAppOrgUpdate = Partial<Omit<z.input<typeof GitAppOrgSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/identities.ts

@@ -16,5 +16,5 @@ export const IdentitiesSchema = z.object({
 });
 
 export type TIdentities = z.infer<typeof IdentitiesSchema>;
-export type TIdentitiesInsert = Omit<TIdentities, TImmutableDBKeys>;
-export type TIdentitiesUpdate = Partial<Omit<TIdentities, TImmutableDBKeys>>;
+export type TIdentitiesInsert = Omit<z.input<typeof IdentitiesSchema>, TImmutableDBKeys>;
+export type TIdentitiesUpdate = Partial<Omit<z.input<typeof IdentitiesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/identity-access-tokens.ts

@@ -23,5 +23,5 @@ export const IdentityAccessTokensSchema = z.object({
 });
 
 export type TIdentityAccessTokens = z.infer<typeof IdentityAccessTokensSchema>;
-export type TIdentityAccessTokensInsert = Omit<TIdentityAccessTokens, TImmutableDBKeys>;
-export type TIdentityAccessTokensUpdate = Partial<Omit<TIdentityAccessTokens, TImmutableDBKeys>>;
+export type TIdentityAccessTokensInsert = Omit<z.input<typeof IdentityAccessTokensSchema>, TImmutableDBKeys>;
+export type TIdentityAccessTokensUpdate = Partial<Omit<z.input<typeof IdentityAccessTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/identity-org-memberships.ts

@@ -18,5 +18,7 @@ export const IdentityOrgMembershipsSchema = z.object({
 });
 
 export type TIdentityOrgMemberships = z.infer<typeof IdentityOrgMembershipsSchema>;
-export type TIdentityOrgMembershipsInsert = Omit<TIdentityOrgMemberships, TImmutableDBKeys>;
-export type TIdentityOrgMembershipsUpdate = Partial<Omit<TIdentityOrgMemberships, TImmutableDBKeys>>;
+export type TIdentityOrgMembershipsInsert = Omit<z.input<typeof IdentityOrgMembershipsSchema>, TImmutableDBKeys>;
+export type TIdentityOrgMembershipsUpdate = Partial<
+  Omit<z.input<typeof IdentityOrgMembershipsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-project-memberships.ts

@@ -18,5 +18,10 @@ export const IdentityProjectMembershipsSchema = z.object({
 });
 
 export type TIdentityProjectMemberships = z.infer<typeof IdentityProjectMembershipsSchema>;
-export type TIdentityProjectMembershipsInsert = Omit<TIdentityProjectMemberships, TImmutableDBKeys>;
-export type TIdentityProjectMembershipsUpdate = Partial<Omit<TIdentityProjectMemberships, TImmutableDBKeys>>;
+export type TIdentityProjectMembershipsInsert = Omit<
+  z.input<typeof IdentityProjectMembershipsSchema>,
+  TImmutableDBKeys
+>;
+export type TIdentityProjectMembershipsUpdate = Partial<
+  Omit<z.input<typeof IdentityProjectMembershipsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-ua-client-secrets.ts

@@ -23,5 +23,7 @@ export const IdentityUaClientSecretsSchema = z.object({
 });
 
 export type TIdentityUaClientSecrets = z.infer<typeof IdentityUaClientSecretsSchema>;
-export type TIdentityUaClientSecretsInsert = Omit<TIdentityUaClientSecrets, TImmutableDBKeys>;
-export type TIdentityUaClientSecretsUpdate = Partial<Omit<TIdentityUaClientSecrets, TImmutableDBKeys>>;
+export type TIdentityUaClientSecretsInsert = Omit<z.input<typeof IdentityUaClientSecretsSchema>, TImmutableDBKeys>;
+export type TIdentityUaClientSecretsUpdate = Partial<
+  Omit<z.input<typeof IdentityUaClientSecretsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-universal-auths.ts

@@ -21,5 +21,7 @@ export const IdentityUniversalAuthsSchema = z.object({
 });
 
 export type TIdentityUniversalAuths = z.infer<typeof IdentityUniversalAuthsSchema>;
-export type TIdentityUniversalAuthsInsert = Omit<TIdentityUniversalAuths, TImmutableDBKeys>;
-export type TIdentityUniversalAuthsUpdate = Partial<Omit<TIdentityUniversalAuths, TImmutableDBKeys>>;
+export type TIdentityUniversalAuthsInsert = Omit<z.input<typeof IdentityUniversalAuthsSchema>, TImmutableDBKeys>;
+export type TIdentityUniversalAuthsUpdate = Partial<
+  Omit<z.input<typeof IdentityUniversalAuthsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/incident-contacts.ts

@@ -16,5 +16,5 @@ export const IncidentContactsSchema = z.object({
 });
 
 export type TIncidentContacts = z.infer<typeof IncidentContactsSchema>;
-export type TIncidentContactsInsert = Omit<TIncidentContacts, TImmutableDBKeys>;
-export type TIncidentContactsUpdate = Partial<Omit<TIncidentContacts, TImmutableDBKeys>>;
+export type TIncidentContactsInsert = Omit<z.input<typeof IncidentContactsSchema>, TImmutableDBKeys>;
+export type TIncidentContactsUpdate = Partial<Omit<z.input<typeof IncidentContactsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/integration-auths.ts

@@ -33,5 +33,5 @@ export const IntegrationAuthsSchema = z.object({
 });
 
 export type TIntegrationAuths = z.infer<typeof IntegrationAuthsSchema>;
-export type TIntegrationAuthsInsert = Omit<TIntegrationAuths, TImmutableDBKeys>;
-export type TIntegrationAuthsUpdate = Partial<Omit<TIntegrationAuths, TImmutableDBKeys>>;
+export type TIntegrationAuthsInsert = Omit<z.input<typeof IntegrationAuthsSchema>, TImmutableDBKeys>;
+export type TIntegrationAuthsUpdate = Partial<Omit<z.input<typeof IntegrationAuthsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/integrations.ts

@@ -31,5 +31,5 @@ export const IntegrationsSchema = z.object({
 });
 
 export type TIntegrations = z.infer<typeof IntegrationsSchema>;
-export type TIntegrationsInsert = Omit<TIntegrations, TImmutableDBKeys>;
-export type TIntegrationsUpdate = Partial<Omit<TIntegrations, TImmutableDBKeys>>;
+export type TIntegrationsInsert = Omit<z.input<typeof IntegrationsSchema>, TImmutableDBKeys>;
+export type TIntegrationsUpdate = Partial<Omit<z.input<typeof IntegrationsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/models.ts

@@ -112,6 +112,17 @@ export enum SecretType {
   Personal = "personal"
 }
 
+export enum ProjectVersion {
+  V1 = 1,
+  V2 = 2
+}
+
+export enum ProjectUpgradeStatus {
+  InProgress = "IN_PROGRESS",
+  // Completed -> Will be null if completed. So a completed status is not needed
+  Failed = "FAILED"
+}
+
 export enum IdentityAuthMethod {
   Univeral = "universal-auth"
 }

backend/src/db/schemas/org-bots.ts

@@ -27,5 +27,5 @@ export const OrgBotsSchema = z.object({
 });
 
 export type TOrgBots = z.infer<typeof OrgBotsSchema>;
-export type TOrgBotsInsert = Omit<TOrgBots, TImmutableDBKeys>;
-export type TOrgBotsUpdate = Partial<Omit<TOrgBots, TImmutableDBKeys>>;
+export type TOrgBotsInsert = Omit<z.input<typeof OrgBotsSchema>, TImmutableDBKeys>;
+export type TOrgBotsUpdate = Partial<Omit<z.input<typeof OrgBotsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/org-memberships.ts

@@ -20,5 +20,5 @@ export const OrgMembershipsSchema = z.object({
 });
 
 export type TOrgMemberships = z.infer<typeof OrgMembershipsSchema>;
-export type TOrgMembershipsInsert = Omit<TOrgMemberships, TImmutableDBKeys>;
-export type TOrgMembershipsUpdate = Partial<Omit<TOrgMemberships, TImmutableDBKeys>>;
+export type TOrgMembershipsInsert = Omit<z.input<typeof OrgMembershipsSchema>, TImmutableDBKeys>;
+export type TOrgMembershipsUpdate = Partial<Omit<z.input<typeof OrgMembershipsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/org-roles.ts

@@ -19,5 +19,5 @@ export const OrgRolesSchema = z.object({
 });
 
 export type TOrgRoles = z.infer<typeof OrgRolesSchema>;
-export type TOrgRolesInsert = Omit<TOrgRoles, TImmutableDBKeys>;
-export type TOrgRolesUpdate = Partial<Omit<TOrgRoles, TImmutableDBKeys>>;
+export type TOrgRolesInsert = Omit<z.input<typeof OrgRolesSchema>, TImmutableDBKeys>;
+export type TOrgRolesUpdate = Partial<Omit<z.input<typeof OrgRolesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/organizations.ts

@@ -19,5 +19,5 @@ export const OrganizationsSchema = z.object({
 });
 
 export type TOrganizations = z.infer<typeof OrganizationsSchema>;
-export type TOrganizationsInsert = Omit<TOrganizations, TImmutableDBKeys>;
-export type TOrganizationsUpdate = Partial<Omit<TOrganizations, TImmutableDBKeys>>;
+export type TOrganizationsInsert = Omit<z.input<typeof OrganizationsSchema>, TImmutableDBKeys>;
+export type TOrganizationsUpdate = Partial<Omit<z.input<typeof OrganizationsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-bots.ts

@@ -26,5 +26,5 @@ export const ProjectBotsSchema = z.object({
 });
 
 export type TProjectBots = z.infer<typeof ProjectBotsSchema>;
-export type TProjectBotsInsert = Omit<TProjectBots, TImmutableDBKeys>;
-export type TProjectBotsUpdate = Partial<Omit<TProjectBots, TImmutableDBKeys>>;
+export type TProjectBotsInsert = Omit<z.input<typeof ProjectBotsSchema>, TImmutableDBKeys>;
+export type TProjectBotsUpdate = Partial<Omit<z.input<typeof ProjectBotsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-environments.ts

@@ -18,5 +18,5 @@ export const ProjectEnvironmentsSchema = z.object({
 });
 
 export type TProjectEnvironments = z.infer<typeof ProjectEnvironmentsSchema>;
-export type TProjectEnvironmentsInsert = Omit<TProjectEnvironments, TImmutableDBKeys>;
-export type TProjectEnvironmentsUpdate = Partial<Omit<TProjectEnvironments, TImmutableDBKeys>>;
+export type TProjectEnvironmentsInsert = Omit<z.input<typeof ProjectEnvironmentsSchema>, TImmutableDBKeys>;
+export type TProjectEnvironmentsUpdate = Partial<Omit<z.input<typeof ProjectEnvironmentsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-keys.ts

@@ -19,5 +19,5 @@ export const ProjectKeysSchema = z.object({
 });
 
 export type TProjectKeys = z.infer<typeof ProjectKeysSchema>;
-export type TProjectKeysInsert = Omit<TProjectKeys, TImmutableDBKeys>;
-export type TProjectKeysUpdate = Partial<Omit<TProjectKeys, TImmutableDBKeys>>;
+export type TProjectKeysInsert = Omit<z.input<typeof ProjectKeysSchema>, TImmutableDBKeys>;
+export type TProjectKeysUpdate = Partial<Omit<z.input<typeof ProjectKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-memberships.ts

@@ -18,5 +18,5 @@ export const ProjectMembershipsSchema = z.object({
 });
 
 export type TProjectMemberships = z.infer<typeof ProjectMembershipsSchema>;
-export type TProjectMembershipsInsert = Omit<TProjectMemberships, TImmutableDBKeys>;
-export type TProjectMembershipsUpdate = Partial<Omit<TProjectMemberships, TImmutableDBKeys>>;
+export type TProjectMembershipsInsert = Omit<z.input<typeof ProjectMembershipsSchema>, TImmutableDBKeys>;
+export type TProjectMembershipsUpdate = Partial<Omit<z.input<typeof ProjectMembershipsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-roles.ts

@@ -19,5 +19,5 @@ export const ProjectRolesSchema = z.object({
 });
 
 export type TProjectRoles = z.infer<typeof ProjectRolesSchema>;
-export type TProjectRolesInsert = Omit<TProjectRoles, TImmutableDBKeys>;
-export type TProjectRolesUpdate = Partial<Omit<TProjectRoles, TImmutableDBKeys>>;
+export type TProjectRolesInsert = Omit<z.input<typeof ProjectRolesSchema>, TImmutableDBKeys>;
+export type TProjectRolesUpdate = Partial<Omit<z.input<typeof ProjectRolesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/projects.ts

@@ -14,9 +14,11 @@ export const ProjectsSchema = z.object({
   autoCapitalization: z.boolean().default(true).nullable().optional(),
   orgId: z.string().uuid(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  version: z.number().default(1),
+  upgradeStatus: z.string().nullable().optional()
 });
 
 export type TProjects = z.infer<typeof ProjectsSchema>;
-export type TProjectsInsert = Omit<TProjects, TImmutableDBKeys>;
-export type TProjectsUpdate = Partial<Omit<TProjects, TImmutableDBKeys>>;
+export type TProjectsInsert = Omit<z.input<typeof ProjectsSchema>, TImmutableDBKeys>;
+export type TProjectsUpdate = Partial<Omit<z.input<typeof ProjectsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/saml-configs.ts

@@ -27,5 +27,5 @@ export const SamlConfigsSchema = z.object({
 });
 
 export type TSamlConfigs = z.infer<typeof SamlConfigsSchema>;
-export type TSamlConfigsInsert = Omit<TSamlConfigs, TImmutableDBKeys>;
-export type TSamlConfigsUpdate = Partial<Omit<TSamlConfigs, TImmutableDBKeys>>;
+export type TSamlConfigsInsert = Omit<z.input<typeof SamlConfigsSchema>, TImmutableDBKeys>;
+export type TSamlConfigsUpdate = Partial<Omit<z.input<typeof SamlConfigsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/scim-tokens.ts

@@ -17,5 +17,5 @@ export const ScimTokensSchema = z.object({
 });
 
 export type TScimTokens = z.infer<typeof ScimTokensSchema>;
-export type TScimTokensInsert = Omit<TScimTokens, TImmutableDBKeys>;
-export type TScimTokensUpdate = Partial<Omit<TScimTokens, TImmutableDBKeys>>;
+export type TScimTokensInsert = Omit<z.input<typeof ScimTokensSchema>, TImmutableDBKeys>;
+export type TScimTokensUpdate = Partial<Omit<z.input<typeof ScimTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-approval-policies-approvers.ts

@@ -16,5 +16,10 @@ export const SecretApprovalPoliciesApproversSchema = z.object({
 });
 
 export type TSecretApprovalPoliciesApprovers = z.infer<typeof SecretApprovalPoliciesApproversSchema>;
-export type TSecretApprovalPoliciesApproversInsert = Omit<TSecretApprovalPoliciesApprovers, TImmutableDBKeys>;
-export type TSecretApprovalPoliciesApproversUpdate = Partial<Omit<TSecretApprovalPoliciesApprovers, TImmutableDBKeys>>;
+export type TSecretApprovalPoliciesApproversInsert = Omit<
+  z.input<typeof SecretApprovalPoliciesApproversSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalPoliciesApproversUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalPoliciesApproversSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-policies.ts

@@ -18,5 +18,7 @@ export const SecretApprovalPoliciesSchema = z.object({
 });
 
 export type TSecretApprovalPolicies = z.infer<typeof SecretApprovalPoliciesSchema>;
-export type TSecretApprovalPoliciesInsert = Omit<TSecretApprovalPolicies, TImmutableDBKeys>;
-export type TSecretApprovalPoliciesUpdate = Partial<Omit<TSecretApprovalPolicies, TImmutableDBKeys>>;
+export type TSecretApprovalPoliciesInsert = Omit<z.input<typeof SecretApprovalPoliciesSchema>, TImmutableDBKeys>;
+export type TSecretApprovalPoliciesUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalPoliciesSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-request-secret-tags.ts

@@ -16,5 +16,10 @@ export const SecretApprovalRequestSecretTagsSchema = z.object({
 });
 
 export type TSecretApprovalRequestSecretTags = z.infer<typeof SecretApprovalRequestSecretTagsSchema>;
-export type TSecretApprovalRequestSecretTagsInsert = Omit<TSecretApprovalRequestSecretTags, TImmutableDBKeys>;
-export type TSecretApprovalRequestSecretTagsUpdate = Partial<Omit<TSecretApprovalRequestSecretTags, TImmutableDBKeys>>;
+export type TSecretApprovalRequestSecretTagsInsert = Omit<
+  z.input<typeof SecretApprovalRequestSecretTagsSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalRequestSecretTagsUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestSecretTagsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-requests-reviewers.ts

@@ -17,5 +17,10 @@ export const SecretApprovalRequestsReviewersSchema = z.object({
 });
 
 export type TSecretApprovalRequestsReviewers = z.infer<typeof SecretApprovalRequestsReviewersSchema>;
-export type TSecretApprovalRequestsReviewersInsert = Omit<TSecretApprovalRequestsReviewers, TImmutableDBKeys>;
-export type TSecretApprovalRequestsReviewersUpdate = Partial<Omit<TSecretApprovalRequestsReviewers, TImmutableDBKeys>>;
+export type TSecretApprovalRequestsReviewersInsert = Omit<
+  z.input<typeof SecretApprovalRequestsReviewersSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalRequestsReviewersUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestsReviewersSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-requests-secrets.ts

@@ -35,5 +35,10 @@ export const SecretApprovalRequestsSecretsSchema = z.object({
 });
 
 export type TSecretApprovalRequestsSecrets = z.infer<typeof SecretApprovalRequestsSecretsSchema>;
-export type TSecretApprovalRequestsSecretsInsert = Omit<TSecretApprovalRequestsSecrets, TImmutableDBKeys>;
-export type TSecretApprovalRequestsSecretsUpdate = Partial<Omit<TSecretApprovalRequestsSecrets, TImmutableDBKeys>>;
+export type TSecretApprovalRequestsSecretsInsert = Omit<
+  z.input<typeof SecretApprovalRequestsSecretsSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalRequestsSecretsUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestsSecretsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-requests.ts

@@ -22,5 +22,7 @@ export const SecretApprovalRequestsSchema = z.object({
 });
 
 export type TSecretApprovalRequests = z.infer<typeof SecretApprovalRequestsSchema>;
-export type TSecretApprovalRequestsInsert = Omit<TSecretApprovalRequests, TImmutableDBKeys>;
-export type TSecretApprovalRequestsUpdate = Partial<Omit<TSecretApprovalRequests, TImmutableDBKeys>>;
+export type TSecretApprovalRequestsInsert = Omit<z.input<typeof SecretApprovalRequestsSchema>, TImmutableDBKeys>;
+export type TSecretApprovalRequestsUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-blind-indexes.ts

@@ -20,5 +20,5 @@ export const SecretBlindIndexesSchema = z.object({
 });
 
 export type TSecretBlindIndexes = z.infer<typeof SecretBlindIndexesSchema>;
-export type TSecretBlindIndexesInsert = Omit<TSecretBlindIndexes, TImmutableDBKeys>;
-export type TSecretBlindIndexesUpdate = Partial<Omit<TSecretBlindIndexes, TImmutableDBKeys>>;
+export type TSecretBlindIndexesInsert = Omit<z.input<typeof SecretBlindIndexesSchema>, TImmutableDBKeys>;
+export type TSecretBlindIndexesUpdate = Partial<Omit<z.input<typeof SecretBlindIndexesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-folder-versions.ts

@@ -18,5 +18,5 @@ export const SecretFolderVersionsSchema = z.object({
 });
 
 export type TSecretFolderVersions = z.infer<typeof SecretFolderVersionsSchema>;
-export type TSecretFolderVersionsInsert = Omit<TSecretFolderVersions, TImmutableDBKeys>;
-export type TSecretFolderVersionsUpdate = Partial<Omit<TSecretFolderVersions, TImmutableDBKeys>>;
+export type TSecretFolderVersionsInsert = Omit<z.input<typeof SecretFolderVersionsSchema>, TImmutableDBKeys>;
+export type TSecretFolderVersionsUpdate = Partial<Omit<z.input<typeof SecretFolderVersionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-folders.ts

@@ -18,5 +18,5 @@ export const SecretFoldersSchema = z.object({
 });
 
 export type TSecretFolders = z.infer<typeof SecretFoldersSchema>;
-export type TSecretFoldersInsert = Omit<TSecretFolders, TImmutableDBKeys>;
-export type TSecretFoldersUpdate = Partial<Omit<TSecretFolders, TImmutableDBKeys>>;
+export type TSecretFoldersInsert = Omit<z.input<typeof SecretFoldersSchema>, TImmutableDBKeys>;
+export type TSecretFoldersUpdate = Partial<Omit<z.input<typeof SecretFoldersSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-imports.ts

@@ -19,5 +19,5 @@ export const SecretImportsSchema = z.object({
 });
 
 export type TSecretImports = z.infer<typeof SecretImportsSchema>;
-export type TSecretImportsInsert = Omit<TSecretImports, TImmutableDBKeys>;
-export type TSecretImportsUpdate = Partial<Omit<TSecretImports, TImmutableDBKeys>>;
+export type TSecretImportsInsert = Omit<z.input<typeof SecretImportsSchema>, TImmutableDBKeys>;
+export type TSecretImportsUpdate = Partial<Omit<z.input<typeof SecretImportsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-rotation-outputs.ts

@@ -15,5 +15,5 @@ export const SecretRotationOutputsSchema = z.object({
 });
 
 export type TSecretRotationOutputs = z.infer<typeof SecretRotationOutputsSchema>;
-export type TSecretRotationOutputsInsert = Omit<TSecretRotationOutputs, TImmutableDBKeys>;
-export type TSecretRotationOutputsUpdate = Partial<Omit<TSecretRotationOutputs, TImmutableDBKeys>>;
+export type TSecretRotationOutputsInsert = Omit<z.input<typeof SecretRotationOutputsSchema>, TImmutableDBKeys>;
+export type TSecretRotationOutputsUpdate = Partial<Omit<z.input<typeof SecretRotationOutputsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-rotations.ts

@@ -26,5 +26,5 @@ export const SecretRotationsSchema = z.object({
 });
 
 export type TSecretRotations = z.infer<typeof SecretRotationsSchema>;
-export type TSecretRotationsInsert = Omit<TSecretRotations, TImmutableDBKeys>;
-export type TSecretRotationsUpdate = Partial<Omit<TSecretRotations, TImmutableDBKeys>>;
+export type TSecretRotationsInsert = Omit<z.input<typeof SecretRotationsSchema>, TImmutableDBKeys>;
+export type TSecretRotationsUpdate = Partial<Omit<z.input<typeof SecretRotationsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-scanning-git-risks.ts

@@ -42,5 +42,7 @@ export const SecretScanningGitRisksSchema = z.object({
 });
 
 export type TSecretScanningGitRisks = z.infer<typeof SecretScanningGitRisksSchema>;
-export type TSecretScanningGitRisksInsert = Omit<TSecretScanningGitRisks, TImmutableDBKeys>;
-export type TSecretScanningGitRisksUpdate = Partial<Omit<TSecretScanningGitRisks, TImmutableDBKeys>>;
+export type TSecretScanningGitRisksInsert = Omit<z.input<typeof SecretScanningGitRisksSchema>, TImmutableDBKeys>;
+export type TSecretScanningGitRisksUpdate = Partial<
+  Omit<z.input<typeof SecretScanningGitRisksSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-snapshot-folders.ts

@@ -17,5 +17,5 @@ export const SecretSnapshotFoldersSchema = z.object({
 });
 
 export type TSecretSnapshotFolders = z.infer<typeof SecretSnapshotFoldersSchema>;
-export type TSecretSnapshotFoldersInsert = Omit<TSecretSnapshotFolders, TImmutableDBKeys>;
-export type TSecretSnapshotFoldersUpdate = Partial<Omit<TSecretSnapshotFolders, TImmutableDBKeys>>;
+export type TSecretSnapshotFoldersInsert = Omit<z.input<typeof SecretSnapshotFoldersSchema>, TImmutableDBKeys>;
+export type TSecretSnapshotFoldersUpdate = Partial<Omit<z.input<typeof SecretSnapshotFoldersSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-snapshot-secrets.ts

@@ -17,5 +17,5 @@ export const SecretSnapshotSecretsSchema = z.object({
 });
 
 export type TSecretSnapshotSecrets = z.infer<typeof SecretSnapshotSecretsSchema>;
-export type TSecretSnapshotSecretsInsert = Omit<TSecretSnapshotSecrets, TImmutableDBKeys>;
-export type TSecretSnapshotSecretsUpdate = Partial<Omit<TSecretSnapshotSecrets, TImmutableDBKeys>>;
+export type TSecretSnapshotSecretsInsert = Omit<z.input<typeof SecretSnapshotSecretsSchema>, TImmutableDBKeys>;
+export type TSecretSnapshotSecretsUpdate = Partial<Omit<z.input<typeof SecretSnapshotSecretsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-snapshots.ts

@@ -17,5 +17,5 @@ export const SecretSnapshotsSchema = z.object({
 });
 
 export type TSecretSnapshots = z.infer<typeof SecretSnapshotsSchema>;
-export type TSecretSnapshotsInsert = Omit<TSecretSnapshots, TImmutableDBKeys>;
-export type TSecretSnapshotsUpdate = Partial<Omit<TSecretSnapshots, TImmutableDBKeys>>;
+export type TSecretSnapshotsInsert = Omit<z.input<typeof SecretSnapshotsSchema>, TImmutableDBKeys>;
+export type TSecretSnapshotsUpdate = Partial<Omit<z.input<typeof SecretSnapshotsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-tag-junction.ts

@@ -14,5 +14,5 @@ export const SecretTagJunctionSchema = z.object({
 });
 
 export type TSecretTagJunction = z.infer<typeof SecretTagJunctionSchema>;
-export type TSecretTagJunctionInsert = Omit<TSecretTagJunction, TImmutableDBKeys>;
-export type TSecretTagJunctionUpdate = Partial<Omit<TSecretTagJunction, TImmutableDBKeys>>;
+export type TSecretTagJunctionInsert = Omit<z.input<typeof SecretTagJunctionSchema>, TImmutableDBKeys>;
+export type TSecretTagJunctionUpdate = Partial<Omit<z.input<typeof SecretTagJunctionSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-tags.ts

@@ -19,5 +19,5 @@ export const SecretTagsSchema = z.object({
 });
 
 export type TSecretTags = z.infer<typeof SecretTagsSchema>;
-export type TSecretTagsInsert = Omit<TSecretTags, TImmutableDBKeys>;
-export type TSecretTagsUpdate = Partial<Omit<TSecretTags, TImmutableDBKeys>>;
+export type TSecretTagsInsert = Omit<z.input<typeof SecretTagsSchema>, TImmutableDBKeys>;
+export type TSecretTagsUpdate = Partial<Omit<z.input<typeof SecretTagsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-version-tag-junction.ts

@@ -14,5 +14,7 @@ export const SecretVersionTagJunctionSchema = z.object({
 });
 
 export type TSecretVersionTagJunction = z.infer<typeof SecretVersionTagJunctionSchema>;
-export type TSecretVersionTagJunctionInsert = Omit<TSecretVersionTagJunction, TImmutableDBKeys>;
-export type TSecretVersionTagJunctionUpdate = Partial<Omit<TSecretVersionTagJunction, TImmutableDBKeys>>;
+export type TSecretVersionTagJunctionInsert = Omit<z.input<typeof SecretVersionTagJunctionSchema>, TImmutableDBKeys>;
+export type TSecretVersionTagJunctionUpdate = Partial<
+  Omit<z.input<typeof SecretVersionTagJunctionSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-versions.ts

@@ -36,5 +36,5 @@ export const SecretVersionsSchema = z.object({
 });
 
 export type TSecretVersions = z.infer<typeof SecretVersionsSchema>;
-export type TSecretVersionsInsert = Omit<TSecretVersions, TImmutableDBKeys>;
-export type TSecretVersionsUpdate = Partial<Omit<TSecretVersions, TImmutableDBKeys>>;
+export type TSecretVersionsInsert = Omit<z.input<typeof SecretVersionsSchema>, TImmutableDBKeys>;
+export type TSecretVersionsUpdate = Partial<Omit<z.input<typeof SecretVersionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secrets.ts

@@ -34,5 +34,5 @@ export const SecretsSchema = z.object({
 });
 
 export type TSecrets = z.infer<typeof SecretsSchema>;
-export type TSecretsInsert = Omit<TSecrets, TImmutableDBKeys>;
-export type TSecretsUpdate = Partial<Omit<TSecrets, TImmutableDBKeys>>;
+export type TSecretsInsert = Omit<z.input<typeof SecretsSchema>, TImmutableDBKeys>;
+export type TSecretsUpdate = Partial<Omit<z.input<typeof SecretsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/service-tokens.ts

@@ -25,5 +25,5 @@ export const ServiceTokensSchema = z.object({
 });
 
 export type TServiceTokens = z.infer<typeof ServiceTokensSchema>;
-export type TServiceTokensInsert = Omit<TServiceTokens, TImmutableDBKeys>;
-export type TServiceTokensUpdate = Partial<Omit<TServiceTokens, TImmutableDBKeys>>;
+export type TServiceTokensInsert = Omit<z.input<typeof ServiceTokensSchema>, TImmutableDBKeys>;
+export type TServiceTokensUpdate = Partial<Omit<z.input<typeof ServiceTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/super-admin.ts

@@ -12,9 +12,11 @@ export const SuperAdminSchema = z.object({
   initialized: z.boolean().default(false).nullable().optional(),
   allowSignUp: z.boolean().default(true).nullable().optional(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  allowedSignUpDomain: z.string().nullable().optional(),
+  instanceId: z.string().uuid().default("00000000-0000-0000-0000-000000000000")
 });
 
 export type TSuperAdmin = z.infer<typeof SuperAdminSchema>;
-export type TSuperAdminInsert = Omit<TSuperAdmin, TImmutableDBKeys>;
-export type TSuperAdminUpdate = Partial<Omit<TSuperAdmin, TImmutableDBKeys>>;
+export type TSuperAdminInsert = Omit<z.input<typeof SuperAdminSchema>, TImmutableDBKeys>;
+export type TSuperAdminUpdate = Partial<Omit<z.input<typeof SuperAdminSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/trusted-ips.ts

@@ -20,5 +20,5 @@ export const TrustedIpsSchema = z.object({
 });
 
 export type TTrustedIps = z.infer<typeof TrustedIpsSchema>;
-export type TTrustedIpsInsert = Omit<TTrustedIps, TImmutableDBKeys>;
-export type TTrustedIpsUpdate = Partial<Omit<TTrustedIps, TImmutableDBKeys>>;
+export type TTrustedIpsInsert = Omit<z.input<typeof TrustedIpsSchema>, TImmutableDBKeys>;
+export type TTrustedIpsUpdate = Partial<Omit<z.input<typeof TrustedIpsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/user-actions.ts

@@ -16,5 +16,5 @@ export const UserActionsSchema = z.object({
 });
 
 export type TUserActions = z.infer<typeof UserActionsSchema>;
-export type TUserActionsInsert = Omit<TUserActions, TImmutableDBKeys>;
-export type TUserActionsUpdate = Partial<Omit<TUserActions, TImmutableDBKeys>>;
+export type TUserActionsInsert = Omit<z.input<typeof UserActionsSchema>, TImmutableDBKeys>;
+export type TUserActionsUpdate = Partial<Omit<z.input<typeof UserActionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/user-encryption-keys.ts

@@ -25,5 +25,5 @@ export const UserEncryptionKeysSchema = z.object({
 });
 
 export type TUserEncryptionKeys = z.infer<typeof UserEncryptionKeysSchema>;
-export type TUserEncryptionKeysInsert = Omit<TUserEncryptionKeys, TImmutableDBKeys>;
-export type TUserEncryptionKeysUpdate = Partial<Omit<TUserEncryptionKeys, TImmutableDBKeys>>;
+export type TUserEncryptionKeysInsert = Omit<z.input<typeof UserEncryptionKeysSchema>, TImmutableDBKeys>;
+export type TUserEncryptionKeysUpdate = Partial<Omit<z.input<typeof UserEncryptionKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/users.ts

@@ -19,9 +19,10 @@ export const UsersSchema = z.object({
   mfaMethods: z.string().array().nullable().optional(),
   devices: z.unknown().nullable().optional(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  isGhost: z.boolean().default(false)
 });
 
 export type TUsers = z.infer<typeof UsersSchema>;
-export type TUsersInsert = Omit<TUsers, TImmutableDBKeys>;
-export type TUsersUpdate = Partial<Omit<TUsers, TImmutableDBKeys>>;
+export type TUsersInsert = Omit<z.input<typeof UsersSchema>, TImmutableDBKeys>;
+export type TUsersUpdate = Partial<Omit<z.input<typeof UsersSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/webhooks.ts

@@ -25,5 +25,5 @@ export const WebhooksSchema = z.object({
 });
 
 export type TWebhooks = z.infer<typeof WebhooksSchema>;
-export type TWebhooksInsert = Omit<TWebhooks, TImmutableDBKeys>;
-export type TWebhooksUpdate = Partial<Omit<TWebhooks, TImmutableDBKeys>>;
+export type TWebhooksInsert = Omit<z.input<typeof WebhooksSchema>, TImmutableDBKeys>;
+export type TWebhooksUpdate = Partial<Omit<z.input<typeof WebhooksSchema>, TImmutableDBKeys>>;

backend/src/db/seed-data.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/no-mutable-exports */
 import crypto from "node:crypto";
 
 import argon2, { argon2id } from "argon2";
@@ -15,9 +16,12 @@ import {
 
 import { TSecrets, TUserEncryptionKeys } from "./schemas";
 
+export let userPrivateKey: string | undefined;
+export let userPublicKey: string | undefined;
+
 export const seedData1 = {
   id: "3dafd81d-4388-432b-a4c5-f735616868c1",
-  email: "test@localhost.local",
+  email: process.env.TEST_USER_EMAIL || "test@localhost.local",
   password: process.env.TEST_USER_PASSWORD || "testInfisical@1",
   organization: {
     id: "180870b7-f464-4740-8ffe-9d11c9245ea7",
@@ -42,6 +46,12 @@ export const seedData1 = {
   },
   token: {
     id: "a9dfafba-a3b7-42e3-8618-91abb702fd36"
+  },
+
+  // We set these values during user creation, and later re-use them during project seeding.
+  encryptionKeys: {
+    publicKey: "",
+    privateKey: ""
   }
 };
 

backend/src/ee/routes/v1/scim-router.ts

@@ -5,7 +5,7 @@ import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
 
 export const registerScimRouter = async (server: FastifyZodProvider) => {
-  server.addContentTypeParser("application/scim+json", { parseAs: "string" }, function (req, body, done) {
+  server.addContentTypeParser("application/scim+json", { parseAs: "string" }, (_, body, done) => {
     try {
       const strBody = body instanceof Buffer ? body.toString() : body;
 

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -22,23 +22,13 @@ export const auditLogQueueServiceFactory = ({
   licenseService
 }: TAuditLogQueueServiceFactoryDep) => {
   const pushToLog = async (data: TCreateAuditLogDTO) => {
-    await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, {
-      removeOnFail: {
-        count: 5
-      },
-      removeOnComplete: true
-    });
-  };
-
-  queueService.start(QueueName.AuditLog, async (job) => {
-    const { actor, event, ipAddress, projectId, userAgent, userAgentType } = job.data;
-    let { orgId } = job.data;
+    let { orgId } = data;
     const MS_IN_DAY = 24 * 60 * 60 * 1000;
 
     if (!orgId) {
       // it will never be undefined for both org and project id
       // TODO(akhilmhdh): use caching here in dal to avoid db calls
-      const project = await projectDAL.findById(projectId as string);
+      const project = await projectDAL.findById(data.projectId as string);
       orgId = project.orgId;
     }
 
@@ -46,6 +36,26 @@ export const auditLogQueueServiceFactory = ({
     const ttl = plan.auditLogsRetentionDays * MS_IN_DAY;
     // skip inserting if audit log retention is 0 meaning its not supported
     if (ttl === 0) return;
+
+    await queueService.queue(
+      QueueName.AuditLog,
+      QueueJobs.AuditLog,
+      { ...data, orgId },
+      {
+        removeOnFail: {
+          count: 3
+        },
+        removeOnComplete: true
+      }
+    );
+  };
+
+  queueService.start(QueueName.AuditLog, async (job) => {
+    const { actor, event, ipAddress, projectId, userAgent, userAgentType, orgId } = job.data;
+    const MS_IN_DAY = 24 * 60 * 60 * 1000;
+
+    const plan = await licenseService.getPlan(orgId as string);
+    const ttl = plan.auditLogsRetentionDays * MS_IN_DAY;
     await auditLogDAL.create({
       actor: actor.type,
       actorMetadata: actor.metadata,

backend/src/ee/services/audit-log/audit-log-service.ts

@@ -58,6 +58,7 @@ export const auditLogServiceFactory = ({
     if (data.event.type !== EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH) {
       if (!data.projectId && !data.orgId) throw new BadRequestError({ message: "Must either project id or org id" });
     }
+
     return auditLogQueue.pushToLog(data);
   };
 

backend/src/ee/services/license/license-service.ts

@@ -505,6 +505,9 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
     get isValidLicense() {
       return isValidLicense;
     },
+    getInstanceType() {
+      return instanceType;
+    },
     getPlan,
     updateSubscriptionOrgMemberCount,
     refreshPlan,

backend/src/ee/services/permission/permission-service.ts

@@ -177,6 +177,8 @@ export const permissionServiceFactory = ({
 
   const getServiceTokenProjectPermission = async (serviceTokenId: string, projectId: string) => {
     const serviceToken = await serviceTokenDAL.findById(serviceTokenId);
+    if (!serviceToken) throw new BadRequestError({ message: "Service token not found" });
+
     if (serviceToken.projectId !== projectId)
       throw new UnauthorizedError({
         message: "Failed to find service authorization for given project"

backend/src/ee/services/saml-config/saml-config-service.ts

@@ -338,7 +338,8 @@ export const samlConfigServiceFactory = ({
             email,
             firstName,
             lastName,
-            authMethods: [AuthMethod.EMAIL]
+            authMethods: [AuthMethod.EMAIL],
+            isGhost: false
           },
           tx
         );

backend/src/ee/services/scim/scim-service.ts

@@ -251,7 +251,8 @@ export const scimServiceFactory = ({
             email,
             firstName,
             lastName,
-            authMethods: [AuthMethod.EMAIL]
+            authMethods: [AuthMethod.EMAIL],
+            isGhost: false
           },
           tx
         );

backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts

@@ -1,8 +1,13 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { SecretApprovalRequestsSecretsSchema, TableName, TSecretTags } from "@app/db/schemas";
-import { DatabaseError } from "@app/lib/errors";
+import {
+  SecretApprovalRequestsSecretsSchema,
+  TableName,
+  TSecretApprovalRequestsSecrets,
+  TSecretTags
+} from "@app/db/schemas";
+import { BadRequestError, DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
 
 export type TSecretApprovalRequestSecretDALFactory = ReturnType<typeof secretApprovalRequestSecretDALFactory>;
@@ -11,6 +16,35 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
   const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret);
   const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag);
 
+  const bulkUpdateNoVersionIncrement = async (data: TSecretApprovalRequestsSecrets[], tx?: Knex) => {
+    try {
+      const existingApprovalSecrets = await secretApprovalRequestSecretOrm.find(
+        {
+          $in: {
+            id: data.map((el) => el.id)
+          }
+        },
+        { tx }
+      );
+
+      if (existingApprovalSecrets.length !== data.length) {
+        throw new BadRequestError({ message: "Some of the secret approvals do not exist" });
+      }
+
+      if (data.length === 0) return [];
+
+      const updatedApprovalSecrets = await (tx || db)(TableName.SecretApprovalRequestSecret)
+        .insert(data)
+        .onConflict("id") // this will cause a conflict then merge the data
+        .merge() // Merge the data with the existing data
+        .returning("*");
+
+      return updatedApprovalSecrets;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
   const findByRequestId = async (requestId: string, tx?: Knex) => {
     try {
       const doc = await (tx || db)({
@@ -190,6 +224,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
   return {
     ...secretApprovalRequestSecretOrm,
     findByRequestId,
+    bulkUpdateNoVersionIncrement,
     insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany
   };
 };

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -11,6 +11,7 @@ import { BadRequestError, UnauthorizedError } from "@app/lib/errors";
 import { groupBy, pick, unique } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { ActorType } from "@app/services/auth/auth-type";
+import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
 import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -47,6 +48,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany">;
+  projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
   secretService: Pick<
     TSecretServiceFactory,
     | "fnSecretBulkInsert"
@@ -67,6 +69,7 @@ export const secretApprovalRequestServiceFactory = ({
   secretApprovalRequestReviewerDAL,
   secretApprovalRequestSecretDAL,
   secretBlindIndexDAL,
+  projectDAL,
   permissionService,
   snapshotService,
   secretService,
@@ -434,6 +437,8 @@ export const secretApprovalRequestServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "GenSecretApproval" });
     const folderId = folder.id;

backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts

@@ -240,7 +240,7 @@ export const secretRotationQueueFactory = ({
         );
       });
 
-      telemetryService.sendPostHogEvents({
+      await telemetryService.sendPostHogEvents({
         event: PostHogEventTypes.SecretRotated,
         distinctId: "",
         properties: {

backend/src/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue.ts

@@ -158,7 +158,7 @@ export const secretScanningQueueFactory = ({
       });
     }
 
-    telemetryService.sendPostHogEvents({
+    await telemetryService.sendPostHogEvents({
       event: PostHogEventTypes.SecretScannerPush,
       distinctId: repository.fullName,
       properties: {
@@ -228,7 +228,7 @@ export const secretScanningQueueFactory = ({
       });
     }
 
-    telemetryService.sendPostHogEvents({
+    await telemetryService.sendPostHogEvents({
       event: PostHogEventTypes.SecretScannerFull,
       distinctId: repository.fullName,
       properties: {

backend/src/keystore/keystore.ts

@@ -0,0 +1,20 @@
+import { Redis } from "ioredis";
+
+export type TKeyStoreFactory = ReturnType<typeof keyStoreFactory>;
+
+export const keyStoreFactory = (redisUrl: string) => {
+  const redis = new Redis(redisUrl);
+
+  const setItem = async (key: string, value: string | number | Buffer) => redis.set(key, value);
+
+  const getItem = async (key: string) => redis.get(key);
+
+  const setItemWithExpiry = async (key: string, exp: number | string, value: string | number | Buffer) =>
+    redis.setex(key, exp, value);
+
+  const deleteItem = async (key: string) => redis.del(key);
+
+  const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
+
+  return { setItem, getItem, setItemWithExpiry, deleteItem, incrementBy };
+};

backend/src/lib/config/env.ts

@@ -94,14 +94,17 @@ const envSchema = z
     SECRET_SCANNING_WEBHOOK_SECRET: zpStr(z.string().optional()),
     SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()),
     SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional()),
-    // LICENCE
+    // LICENSE
     LICENSE_SERVER_URL: zpStr(z.string().optional().default("https://portal.infisical.com")),
     LICENSE_SERVER_KEY: zpStr(z.string().optional()),
     LICENSE_KEY: zpStr(z.string().optional()),
+
+    // GENERIC
     STANDALONE_MODE: z
       .enum(["true", "false"])
       .transform((val) => val === "true")
-      .optional()
+      .optional(),
+    INFISICAL_CLOUD: zodStrBool.default("false")
   })
   .transform((data) => ({
     ...data,

backend/src/lib/crypto/encryption.ts

@@ -8,6 +8,9 @@ import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
 
 import { getConfig } from "../config/env";
 
+export const decodeBase64 = (s: string) => naclUtils.decodeBase64(s);
+export const encodeBase64 = (u: Uint8Array) => naclUtils.encodeBase64(u);
+
 export type TDecryptSymmetricInput = {
   ciphertext: string;
   iv: string;

backend/src/lib/crypto/index.ts

@@ -1,12 +1,20 @@
 export {
   buildSecretBlindIndexFromName,
   createSecretBlindIndex,
+  decodeBase64,
   decryptAsymmetric,
   decryptSymmetric,
   decryptSymmetric128BitHexKeyUTF8,
+  encodeBase64,
   encryptAsymmetric,
   encryptSymmetric,
   encryptSymmetric128BitHexKeyUTF8,
   generateAsymmetricKeyPair
 } from "./encryption";
+export {
+  decryptIntegrationAuths,
+  decryptSecretApprovals,
+  decryptSecrets,
+  decryptSecretVersions
+} from "./secret-encryption";
 export { generateSrpServerKey, srpCheckClientProof } from "./srp";

backend/src/lib/crypto/secret-encryption.ts

@@ -0,0 +1,293 @@
+import crypto from "crypto";
+import { z } from "zod";
+
+import {
+  IntegrationAuthsSchema,
+  SecretApprovalRequestsSecretsSchema,
+  SecretsSchema,
+  SecretVersionsSchema,
+  TIntegrationAuths,
+  TProjectKeys,
+  TSecretApprovalRequestsSecrets,
+  TSecrets,
+  TSecretVersions
+} from "../../db/schemas";
+import { decryptAsymmetric } from "./encryption";
+
+const DecryptedValuesSchema = z.object({
+  id: z.string(),
+  secretKey: z.string(),
+  secretValue: z.string(),
+  secretComment: z.string().optional()
+});
+
+const DecryptedSecretSchema = z.object({
+  decrypted: DecryptedValuesSchema,
+  original: SecretsSchema
+});
+
+const DecryptedIntegrationAuthsSchema = z.object({
+  decrypted: z.object({
+    id: z.string(),
+    access: z.string(),
+    accessId: z.string(),
+    refresh: z.string()
+  }),
+  original: IntegrationAuthsSchema
+});
+
+const DecryptedSecretVersionsSchema = z.object({
+  decrypted: DecryptedValuesSchema,
+  original: SecretVersionsSchema
+});
+
+const DecryptedSecretApprovalsSchema = z.object({
+  decrypted: DecryptedValuesSchema,
+  original: SecretApprovalRequestsSecretsSchema
+});
+
+type DecryptedSecret = z.infer<typeof DecryptedSecretSchema>;
+type DecryptedSecretVersions = z.infer<typeof DecryptedSecretVersionsSchema>;
+type DecryptedSecretApprovals = z.infer<typeof DecryptedSecretApprovalsSchema>;
+type DecryptedIntegrationAuths = z.infer<typeof DecryptedIntegrationAuthsSchema>;
+
+type TLatestKey = TProjectKeys & {
+  sender: {
+    publicKey: string;
+  };
+};
+
+const decryptCipher = ({
+  ciphertext,
+  iv,
+  tag,
+  key
+}: {
+  ciphertext: string;
+  iv: string;
+  tag: string;
+  key: string | Buffer;
+}) => {
+  const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64"));
+  decipher.setAuthTag(Buffer.from(tag, "base64"));
+
+  let cleartext = decipher.update(ciphertext, "base64", "utf8");
+  cleartext += decipher.final("utf8");
+
+  return cleartext;
+};
+
+const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => {
+  const results: string[] = [];
+
+  for (const { ciphertext, iv, tag } of data) {
+    if (!ciphertext || !iv || !tag) {
+      results.push("");
+    } else {
+      results.push(decryptCipher({ ciphertext, iv, tag, key }));
+    }
+  }
+
+  return results;
+};
+export const decryptSecrets = (encryptedSecrets: TSecrets[], privateKey: string, latestKey: TLatestKey) => {
+  const key = decryptAsymmetric({
+    ciphertext: latestKey.encryptedKey,
+    nonce: latestKey.nonce,
+    publicKey: latestKey.sender.publicKey,
+    privateKey
+  });
+
+  const decryptedSecrets: DecryptedSecret[] = [];
+
+  encryptedSecrets.forEach((encSecret) => {
+    const [secretKey, secretValue, secretComment] = getDecryptedValues(
+      [
+        {
+          ciphertext: encSecret.secretKeyCiphertext,
+          iv: encSecret.secretKeyIV,
+          tag: encSecret.secretKeyTag
+        },
+        {
+          ciphertext: encSecret.secretValueCiphertext,
+          iv: encSecret.secretValueIV,
+          tag: encSecret.secretValueTag
+        },
+        {
+          ciphertext: encSecret.secretCommentCiphertext || "",
+          iv: encSecret.secretCommentIV || "",
+          tag: encSecret.secretCommentTag || ""
+        }
+      ],
+      key
+    );
+
+    const decryptedSecret: DecryptedSecret = {
+      decrypted: {
+        secretKey,
+        secretValue,
+        secretComment,
+        id: encSecret.id
+      },
+      original: encSecret
+    };
+
+    decryptedSecrets.push(DecryptedSecretSchema.parse(decryptedSecret));
+  });
+
+  return decryptedSecrets;
+};
+
+export const decryptSecretVersions = (
+  encryptedSecretVersions: TSecretVersions[],
+  privateKey: string,
+  latestKey: TLatestKey
+) => {
+  const key = decryptAsymmetric({
+    ciphertext: latestKey.encryptedKey,
+    nonce: latestKey.nonce,
+    publicKey: latestKey.sender.publicKey,
+    privateKey
+  });
+
+  const decryptedSecrets: DecryptedSecretVersions[] = [];
+
+  encryptedSecretVersions.forEach((encSecret) => {
+    const [secretKey, secretValue, secretComment] = getDecryptedValues(
+      [
+        {
+          ciphertext: encSecret.secretKeyCiphertext,
+          iv: encSecret.secretKeyIV,
+          tag: encSecret.secretKeyTag
+        },
+        {
+          ciphertext: encSecret.secretValueCiphertext,
+          iv: encSecret.secretValueIV,
+          tag: encSecret.secretValueTag
+        },
+        {
+          ciphertext: encSecret.secretCommentCiphertext || "",
+          iv: encSecret.secretCommentIV || "",
+          tag: encSecret.secretCommentTag || ""
+        }
+      ],
+      key
+    );
+
+    const decryptedSecret: DecryptedSecretVersions = {
+      decrypted: {
+        secretKey,
+        secretValue,
+        secretComment,
+        id: encSecret.id
+      },
+      original: encSecret
+    };
+
+    decryptedSecrets.push(DecryptedSecretVersionsSchema.parse(decryptedSecret));
+  });
+
+  return decryptedSecrets;
+};
+
+export const decryptSecretApprovals = (
+  encryptedSecretApprovals: TSecretApprovalRequestsSecrets[],
+  privateKey: string,
+  latestKey: TLatestKey
+) => {
+  const key = decryptAsymmetric({
+    ciphertext: latestKey.encryptedKey,
+    nonce: latestKey.nonce,
+    publicKey: latestKey.sender.publicKey,
+    privateKey
+  });
+
+  const decryptedSecrets: DecryptedSecretApprovals[] = [];
+
+  encryptedSecretApprovals.forEach((encApproval) => {
+    const [secretKey, secretValue, secretComment] = getDecryptedValues(
+      [
+        {
+          ciphertext: encApproval.secretKeyCiphertext,
+          iv: encApproval.secretKeyIV,
+          tag: encApproval.secretKeyTag
+        },
+        {
+          ciphertext: encApproval.secretValueCiphertext,
+          iv: encApproval.secretValueIV,
+          tag: encApproval.secretValueTag
+        },
+        {
+          ciphertext: encApproval.secretCommentCiphertext || "",
+          iv: encApproval.secretCommentIV || "",
+          tag: encApproval.secretCommentTag || ""
+        }
+      ],
+      key
+    );
+
+    const decryptedSecret: DecryptedSecretApprovals = {
+      decrypted: {
+        secretKey,
+        secretValue,
+        secretComment,
+        id: encApproval.id
+      },
+      original: encApproval
+    };
+
+    decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret));
+  });
+
+  return decryptedSecrets;
+};
+
+export const decryptIntegrationAuths = (
+  encryptedIntegrationAuths: TIntegrationAuths[],
+  privateKey: string,
+  latestKey: TLatestKey
+) => {
+  const key = decryptAsymmetric({
+    ciphertext: latestKey.encryptedKey,
+    nonce: latestKey.nonce,
+    publicKey: latestKey.sender.publicKey,
+    privateKey
+  });
+
+  const decryptedIntegrationAuths: DecryptedIntegrationAuths[] = [];
+
+  encryptedIntegrationAuths.forEach((encAuth) => {
+    const [access, accessId, refresh] = getDecryptedValues(
+      [
+        {
+          ciphertext: encAuth.accessCiphertext || "",
+          iv: encAuth.accessIV || "",
+          tag: encAuth.accessTag || ""
+        },
+        {
+          ciphertext: encAuth.accessIdCiphertext || "",
+          iv: encAuth.accessIdIV || "",
+          tag: encAuth.accessIdTag || ""
+        },
+        {
+          ciphertext: encAuth.refreshCiphertext || "",
+          iv: encAuth.refreshIV || "",
+          tag: encAuth.refreshTag || ""
+        }
+      ],
+      key
+    );
+
+    decryptedIntegrationAuths.push({
+      decrypted: {
+        id: encAuth.id,
+        access,
+        accessId,
+        refresh
+      },
+      original: encAuth
+    });
+  });
+
+  return decryptedIntegrationAuths;
+};

backend/src/lib/crypto/srp.ts

@@ -1,4 +1,12 @@
+import argon2 from "argon2";
+import crypto from "crypto";
 import jsrp from "jsrp";
+import nacl from "tweetnacl";
+import tweetnacl from "tweetnacl-util";
+
+import { TUserEncryptionKeys } from "@app/db/schemas";
+
+import { decryptSymmetric, encryptAsymmetric, encryptSymmetric } from "./encryption";
 
 export const generateSrpServerKey = async (salt: string, verifier: string) => {
   // eslint-disable-next-line new-cap
@@ -24,3 +32,99 @@ export const srpCheckClientProof = async (
   server.setClientPublicKey(clientPublicKey);
   return server.checkClientProof(clientProof);
 };
+
+// Ghost user related:
+// This functionality is intended for ghost user logic. This happens on the frontend when a user is being created.
+// We replicate the same functionality on the backend when creating a ghost user.
+export const generateUserSrpKeys = async (email: string, password: string) => {
+  const pair = nacl.box.keyPair();
+  const secretKeyUint8Array = pair.secretKey;
+  const publicKeyUint8Array = pair.publicKey;
+  const privateKey = tweetnacl.encodeBase64(secretKeyUint8Array);
+  const publicKey = tweetnacl.encodeBase64(publicKeyUint8Array);
+
+  // eslint-disable-next-line
+  const client = new jsrp.client();
+  await new Promise((resolve) => {
+    client.init({ username: email, password }, () => resolve(null));
+  });
+  const { salt, verifier } = await new Promise<{ salt: string; verifier: string }>((resolve, reject) => {
+    client.createVerifier((err, res) => {
+      if (err) return reject(err);
+      return resolve(res);
+    });
+  });
+  const derivedKey = await argon2.hash(password, {
+    salt: Buffer.from(salt),
+    memoryCost: 65536,
+    timeCost: 3,
+    parallelism: 1,
+    hashLength: 32,
+    type: argon2.argon2id,
+    raw: true
+  });
+  if (!derivedKey) throw new Error("Failed to derive key from password");
+
+  const key = crypto.randomBytes(32);
+
+  // create encrypted private key by encrypting the private
+  // key with the symmetric key [key]
+  const {
+    ciphertext: encryptedPrivateKey,
+    iv: encryptedPrivateKeyIV,
+    tag: encryptedPrivateKeyTag
+  } = encryptSymmetric(privateKey, key.toString("base64"));
+
+  // create the protected key by encrypting the symmetric key
+  // [key] with the derived key
+  const {
+    ciphertext: protectedKey,
+    iv: protectedKeyIV,
+    tag: protectedKeyTag
+  } = encryptSymmetric(key.toString("hex"), derivedKey.toString("base64"));
+
+  return {
+    protectedKey,
+    plainPrivateKey: privateKey,
+    protectedKeyIV,
+    protectedKeyTag,
+    publicKey,
+    encryptedPrivateKey,
+    encryptedPrivateKeyIV,
+    encryptedPrivateKeyTag,
+    salt,
+    verifier
+  };
+};
+
+export const getUserPrivateKey = async (password: string, user: TUserEncryptionKeys) => {
+  const derivedKey = await argon2.hash(password, {
+    salt: Buffer.from(user.salt),
+    memoryCost: 65536,
+    timeCost: 3,
+    parallelism: 1,
+    hashLength: 32,
+    type: argon2.argon2id,
+    raw: true
+  });
+  if (!derivedKey) throw new Error("Failed to derive key from password");
+  const key = decryptSymmetric({
+    ciphertext: user.protectedKey!,
+    iv: user.protectedKeyIV!,
+    tag: user.protectedKeyTag!,
+    key: derivedKey.toString("base64")
+  });
+  const privateKey = decryptSymmetric({
+    ciphertext: user.encryptedPrivateKey,
+    iv: user.iv,
+    tag: user.tag,
+    key
+  });
+  return privateKey;
+};
+
+export const buildUserProjectKey = async (privateKey: string, publickey: string) => {
+  const randomBytes = crypto.randomBytes(16).toString("hex");
+  const { nonce, ciphertext } = encryptAsymmetric(randomBytes, publickey, privateKey);
+  return { nonce, ciphertext };
+};

backend/src/main.ts

@@ -1,6 +1,7 @@
 import dotenv from "dotenv";
 
 import { initDbConnection } from "./db";
+import { keyStoreFactory } from "./keystore/keystore";
 import { formatSmtpConfig, initEnvConfig } from "./lib/config/env";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
@@ -19,8 +20,9 @@ const run = async () => {
 
   const smtp = smtpServiceFactory(formatSmtpConfig());
   const queue = queueServiceFactory(appCfg.REDIS_URL);
+  const keyStore = keyStoreFactory(appCfg.REDIS_URL);
 
-  const server = await main({ db, smtp, logger, queue });
+  const server = await main({ db, smtp, logger, queue, keyStore });
   const bootstrap = await bootstrapCheck({ db });
   // eslint-disable-next-line
   process.on("SIGINT", async () => {

backend/src/queue/queue-service.ts

@@ -1,6 +1,7 @@
 import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
 import Redis from "ioredis";
 
+import { SecretKeyEncoding } from "@app/db/schemas";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
 import {
   TScanFullRepoEventPayload,
@@ -12,10 +13,12 @@ export enum QueueName {
   SecretReminder = "secret-reminder",
   AuditLog = "audit-log",
   AuditLogPrune = "audit-log-prune",
+  TelemetryInstanceStats = "telemtry-self-hosted-stats",
   IntegrationSync = "sync-integrations",
   SecretWebhook = "secret-webhook",
   SecretFullRepoScan = "secret-full-repo-scan",
-  SecretPushEventScan = "secret-push-event-scan"
+  SecretPushEventScan = "secret-push-event-scan",
+  UpgradeProjectToGhost = "upgrade-project-to-ghost"
 }
 
 export enum QueueJobs {
@@ -24,8 +27,10 @@ export enum QueueJobs {
   AuditLog = "audit-log-job",
   AuditLogPrune = "audit-log-prune-job",
   SecWebhook = "secret-webhook-trigger",
+  TelemetryInstanceStats = "telemetry-self-hosted-stats",
   IntegrationSync = "secret-integration-pull",
-  SecretScan = "secret-scan"
+  SecretScan = "secret-scan",
+  UpgradeProjectToGhost = "upgrade-project-to-ghost-job"
 }
 
 export type TQueueJobTypes = {
@@ -64,6 +69,23 @@ export type TQueueJobTypes = {
     payload: TScanFullRepoEventPayload;
   };
   [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
+  [QueueName.UpgradeProjectToGhost]: {
+    name: QueueJobs.UpgradeProjectToGhost;
+    payload: {
+      projectId: string;
+      startedByUserId: string;
+      encryptedPrivateKey: {
+        encryptedKey: string;
+        encryptedKeyIv: string;
+        encryptedKeyTag: string;
+        keyEncoding: SecretKeyEncoding;
+      };
+    };
+  };
+  [QueueName.TelemetryInstanceStats]: {
+    name: QueueJobs.TelemetryInstanceStats;
+    payload: undefined;
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/server/app.ts

@@ -14,6 +14,7 @@ import fasitfy from "fastify";
 import { Knex } from "knex";
 import { Logger } from "pino";
 
+import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { TQueueServiceFactory } from "@app/queue";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
@@ -31,10 +32,11 @@ type TMain = {
   smtp: TSmtpService;
   logger?: Logger;
   queue: TQueueServiceFactory;
+  keyStore: TKeyStoreFactory;
 };
 
 // Run the server!
-export const main = async ({ db, smtp, logger, queue }: TMain) => {
+export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
   const appCfg = getConfig();
   const server = fasitfy({
     logger: appCfg.NODE_ENV === "test" ? false : logger,
@@ -70,7 +72,7 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => {
     }
     await server.register(helmet, { contentSecurityPolicy: false });
 
-    await server.register(registerRoutes, { smtp, queue, db });
+    await server.register(registerRoutes, { smtp, queue, db, keyStore });
 
     if (appCfg.isProductionMode) {
       await server.register(registerExternalNextjs, {

backend/src/server/lib/telemetry.ts

@@ -0,0 +1,17 @@
+import { FastifyRequest } from "fastify";
+
+import { ActorType } from "@app/services/auth/auth-type";
+
+// this is a unique id for sending posthog event
+export const getTelemetryDistinctId = (req: FastifyRequest) => {
+  if (req.auth.actor === ActorType.USER) {
+    return req.auth.user.email;
+  }
+  if (req.auth.actor === ActorType.IDENTITY) {
+    return `identity-${req.auth.identityId}`;
+  }
+  if (req.auth.actor === ActorType.SERVICE) {
+    return req.auth.serviceToken.createdByEmail || `service-token-null-creator-${req.auth.serviceTokenId}`; // when user gets removed from system
+  }
+  return "unknown-auth-data";
+};

backend/src/server/plugins/auth/inject-identity.ts

@@ -27,7 +27,7 @@ export type TAuthMode =
     }
   | {
       authMode: AuthMode.SERVICE_TOKEN;
-      serviceToken: TServiceTokens;
+      serviceToken: TServiceTokens & { createdByEmail: string };
       actor: ActorType.SERVICE;
       serviceTokenId: string;
     }

backend/src/server/routes/index.ts

@@ -34,6 +34,7 @@ import { snapshotFolderDALFactory } from "@app/ee/services/secret-snapshot/snaps
 import { snapshotSecretDALFactory } from "@app/ee/services/secret-snapshot/snapshot-secret-dal";
 import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal";
 import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service";
+import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { TQueueServiceFactory } from "@app/queue";
 import { apiKeyDALFactory } from "@app/services/api-key/api-key-dal";
@@ -65,6 +66,7 @@ import { orgRoleDALFactory } from "@app/services/org/org-role-dal";
 import { orgRoleServiceFactory } from "@app/services/org/org-role-service";
 import { orgServiceFactory } from "@app/services/org/org-service";
 import { projectDALFactory } from "@app/services/project/project-dal";
+import { projectQueueFactory } from "@app/services/project/project-queue";
 import { projectServiceFactory } from "@app/services/project/project-service";
 import { projectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
 import { projectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
@@ -95,6 +97,8 @@ import { serviceTokenServiceFactory } from "@app/services/service-token/service-
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
 import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
+import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal";
+import { telemetryQueueServiceFactory } from "@app/services/telemetry/telemetry-queue";
 import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
 import { userDALFactory } from "@app/services/user/user-dal";
 import { userServiceFactory } from "@app/services/user/user-service";
@@ -111,7 +115,12 @@ import { registerV3Routes } from "./v3";
 
 export const registerRoutes = async (
   server: FastifyZodProvider,
-  { db, smtp: smtpService, queue: queueService }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory }
+  {
+    db,
+    smtp: smtpService,
+    queue: queueService,
+    keyStore
+  }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory }
 ) => {
   await server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" });
 
@@ -158,6 +167,7 @@ export const registerRoutes = async (
   const auditLogDAL = auditLogDALFactory(db);
   const trustedIpDAL = trustedIpDALFactory(db);
   const scimDAL = scimDALFactory(db);
+  const telemetryDAL = telemetryDALFactory(db);
 
   // ee db layer ops
   const permissionDAL = permissionDALFactory(db);
@@ -225,7 +235,16 @@ export const registerRoutes = async (
     smtpService
   });
 
-  const telemetryService = telemetryServiceFactory();
+  const telemetryService = telemetryServiceFactory({
+    keyStore,
+    licenseService
+  });
+  const telemetryQueue = telemetryQueueServiceFactory({
+    keyStore,
+    telemetryDAL,
+    queueService
+  });
+
   const tokenService = tokenServiceFactory({ tokenDAL: authTokenDAL, userDAL });
   const userService = userServiceFactory({ userDAL });
   const loginService = authLoginServiceFactory({ userDAL, smtpService, tokenService });
@@ -262,7 +281,8 @@ export const registerRoutes = async (
     userDAL,
     authService: loginService,
     serverCfgDAL: superAdminDAL,
-    orgService
+    orgService,
+    keyStore
   });
   const apiKeyService = apiKeyServiceFactory({ apiKeyDAL, userDAL });
 
@@ -280,19 +300,13 @@ export const registerRoutes = async (
     secretScanningDAL,
     secretScanningQueue
   });
-  const projectService = projectServiceFactory({
-    permissionService,
-    projectDAL,
-    secretBlindIndexDAL,
-    projectEnvDAL,
-    projectMembershipDAL,
-    folderDAL,
-    licenseService
-  });
+  const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL, projectDAL });
+
   const projectMembershipService = projectMembershipServiceFactory({
     projectMembershipDAL,
     projectDAL,
     permissionService,
+    projectBotDAL,
     orgDAL,
     userDAL,
     smtpService,
@@ -300,6 +314,46 @@ export const registerRoutes = async (
     projectRoleDAL,
     licenseService
   });
+  const projectKeyService = projectKeyServiceFactory({
+    permissionService,
+    projectKeyDAL,
+    projectMembershipDAL
+  });
+
+  const projectQueueService = projectQueueFactory({
+    queueService,
+    secretDAL,
+    folderDAL,
+    projectDAL,
+    orgDAL,
+    integrationAuthDAL,
+    orgService,
+    projectEnvDAL,
+    userDAL,
+    secretVersionDAL,
+    projectKeyDAL,
+    projectBotDAL,
+    projectMembershipDAL,
+    secretApprovalRequestDAL,
+    secretApprovalSecretDAL: sarSecretDAL
+  });
+
+  const projectService = projectServiceFactory({
+    permissionService,
+    projectDAL,
+    projectQueue: projectQueueService,
+    secretBlindIndexDAL,
+    identityProjectDAL,
+    identityOrgMembershipDAL,
+    projectBotDAL,
+    projectKeyDAL,
+    userDAL,
+    projectEnvDAL,
+    orgService,
+    projectMembershipDAL,
+    folderDAL,
+    licenseService
+  });
   const projectEnvService = projectEnvServiceFactory({
     permissionService,
     projectEnvDAL,
@@ -307,11 +361,7 @@ export const registerRoutes = async (
     projectDAL,
     folderDAL
   });
-  const projectKeyService = projectKeyServiceFactory({
-    permissionService,
-    projectKeyDAL,
-    projectMembershipDAL
-  });
+
   const projectRoleService = projectRoleServiceFactory({ permissionService, projectRoleDAL });
 
   const snapshotService = secretSnapshotServiceFactory({
@@ -346,9 +396,9 @@ export const registerRoutes = async (
     folderDAL,
     permissionService,
     secretImportDAL,
+    projectDAL,
     secretDAL
   });
-  const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL });
   const integrationAuthService = integrationAuthServiceFactory({
     integrationAuthDAL,
     integrationDAL,
@@ -382,6 +432,7 @@ export const registerRoutes = async (
     secretVersionTagDAL,
     secretBlindIndexDAL,
     permissionService,
+    projectDAL,
     secretDAL,
     secretTagDAL,
     snapshotService,
@@ -395,6 +446,7 @@ export const registerRoutes = async (
     secretTagDAL,
     secretApprovalRequestSecretDAL: sarSecretDAL,
     secretApprovalRequestReviewerDAL: sarReviewerDAL,
+    projectDAL,
     secretVersionDAL,
     secretBlindIndexDAL,
     secretApprovalRequestDAL,
@@ -458,9 +510,13 @@ export const registerRoutes = async (
   });
 
   await superAdminService.initServerCfg();
-  await auditLogQueue.startAuditLogPruneJob();
+  //
   // setup the communication with license key server
   await licenseService.init();
+
+  await auditLogQueue.startAuditLogPruneJob();
+  await telemetryQueue.startTelemetryCheck();
+
   // inject all services
   server.decorate<FastifyZodProvider["services"]>("services", {
     login: loginService,
@@ -552,4 +608,8 @@ export const registerRoutes = async (
   );
   await server.register(registerV2Routes, { prefix: "/api/v2" });
   await server.register(registerV3Routes, { prefix: "/api/v3" });
+
+  server.addHook("onClose", async () => {
+    await telemetryService.flushAll();
+  });
 };

backend/src/server/routes/v1/admin-router.ts

@@ -16,7 +16,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
     schema: {
       response: {
         200: z.object({
-          config: SuperAdminSchema
+          config: SuperAdminSchema.omit({ createdAt: true, updatedAt: true })
         })
       }
     },
@@ -31,7 +31,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
     method: "PATCH",
     schema: {
       body: z.object({
-        allowSignUp: z.boolean().optional()
+        allowSignUp: z.boolean().optional(),
+        allowedSignUpDomain: z.string().optional().nullable()
       }),
       response: {
         200: z.object({
@@ -89,7 +90,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
         userAgent: req.headers["user-agent"] || ""
       });
 
-      server.services.telemetry.sendPostHogEvents({
+      await server.services.telemetry.sendPostHogEvents({
         event: PostHogEventTypes.AdminInit,
         distinctId: user.user.email,
         properties: {