update april_2024_db_update_closed

Fix: Error handling when deleting roles that are assigned to identities or users

.env.example

@@ -8,19 +8,17 @@ ENCRYPTION_KEY=6c1fe4e407b8911c104518103505b218
 # THIS IS A SAMPLE AUTH_SECRET KEY AND SHOULD NEVER BE USED FOR PRODUCTION
 AUTH_SECRET=5lrMXKKWCVocS/uerPsl7V+TX/aaUaI7iDkgl3tSmLE=
 
-# MongoDB
+# Postgres creds
-# Backend will connect to the MongoDB instance at connection string MONGO_URL which can either be a ref
+POSTGRES_PASSWORD=infisical
-# to the MongoDB container instance or Mongo Cloud
+POSTGRES_USER=infisical
+POSTGRES_DB=infisical
+
 # Required
-MONGO_URL=mongodb://root:example@mongo:27017/?authSource=admin
+DB_CONNECTION_URI=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
 
 # Redis
 REDIS_URL=redis://redis:6379
 
-# Optional credentials for MongoDB container instance and Mongo-Express
-MONGO_USERNAME=root
-MONGO_PASSWORD=example
-
 # Website URL
 # Required
 SITE_URL=http://localhost:8080

.env.migration.example

@@ -0,0 +1 @@
+DB_CONNECTION_URI=

.env.test.example

@@ -0,0 +1,4 @@
+REDIS_URL=redis://localhost:6379
+DB_CONNECTION_URI=postgres://infisical:infisical@localhost/infisical?sslmode=disable
+AUTH_SECRET=4bnfe4e407b8921c104518903515b218
+ENCRYPTION_KEY=4bnfe4e407b8921c104518903515b218

.github/resources/changelog-generator.py

@@ -0,0 +1,190 @@
+# inspired by https://www.photoroom.com/inside-photoroom/how-we-automated-our-changelog-thanks-to-chatgpt
+import os
+import requests
+import re
+from openai import OpenAI
+import subprocess
+from datetime import datetime
+
+import uuid
+
+# Constants
+REPO_OWNER = "infisical"
+REPO_NAME = "infisical"
+TOKEN = os.environ["GITHUB_TOKEN"]
+SLACK_WEBHOOK_URL = os.environ["SLACK_WEBHOOK_URL"]
+OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
+SLACK_MSG_COLOR = "#36a64f"
+
+headers = {
+    "Authorization": f"Bearer {TOKEN}",
+    "Accept": "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
+
+
+def set_multiline_output(name, value):
+    with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+        delimiter = uuid.uuid1()
+        print(f'{name}<<{delimiter}', file=fh)
+        print(value, file=fh)
+        print(delimiter, file=fh)
+
+def post_changelog_to_slack(changelog, tag):
+    slack_payload = {
+        "text": "Hey team, it's changelog time! :wave:",
+        "attachments": [
+            {
+                "color": SLACK_MSG_COLOR,
+                "title": f"🗓️Infisical Changelog - {tag}",
+                "text": changelog,
+            }
+        ],
+    }
+
+    response = requests.post(SLACK_WEBHOOK_URL, json=slack_payload)
+
+    if response.status_code != 200:
+        raise Exception("Failed to post changelog to Slack.")
+
+def find_previous_release_tag(release_tag:str):
+    previous_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0", f"{release_tag}^"]).decode("utf-8").strip()
+    while not(previous_tag.startswith("infisical/")):
+        previous_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0", f"{previous_tag}^"]).decode("utf-8").strip()
+    return previous_tag
+
+def get_tag_creation_date(tag_name):
+    url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/git/refs/tags/{tag_name}"
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    commit_sha = response.json()['object']['sha']
+    
+    commit_url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/commits/{commit_sha}"
+    commit_response = requests.get(commit_url, headers=headers)
+    commit_response.raise_for_status()
+    creation_date = commit_response.json()['commit']['author']['date']
+
+    return datetime.strptime(creation_date, '%Y-%m-%dT%H:%M:%SZ')
+
+
+def fetch_prs_between_tags(previous_tag_date:datetime, release_tag_date:datetime):
+    # Use GitHub API to fetch PRs merged between the commits
+    url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/pulls?state=closed&merged=true"
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception("Error fetching PRs from GitHub API!")
+
+    prs = []
+    for pr in response.json():
+        # the idea is as tags happen recently we get last 100 closed PRs and then filter by tag creation date
+        if pr["merged_at"] and datetime.strptime(pr["merged_at"],'%Y-%m-%dT%H:%M:%SZ') < release_tag_date and  datetime.strptime(pr["merged_at"],'%Y-%m-%dT%H:%M:%SZ') > previous_tag_date:
+            prs.append(pr)
+
+    return prs
+
+
+def extract_commit_details_from_prs(prs):
+    commit_details = []
+    for pr in prs:
+        commit_message = pr["title"]
+        commit_url = pr["html_url"]
+        pr_number = pr["number"]
+        branch_name = pr["head"]["ref"]
+        issue_numbers = re.findall(r"(www-\d+|web-\d+)", branch_name)
+
+        # If no issue numbers are found, add the PR details without issue numbers and URLs
+        if not issue_numbers:
+            commit_details.append(
+                {
+                    "message": commit_message,
+                    "pr_number": pr_number,
+                    "pr_url": commit_url,
+                    "issue_number": None,
+                    "issue_url": None,
+                }
+            )
+            continue
+
+        for issue in issue_numbers:
+            commit_details.append(
+                {
+                    "message": commit_message,
+                    "pr_number": pr_number,
+                    "pr_url": commit_url,
+                    "issue_number": issue,
+                }
+            )
+
+    return commit_details
+
+# Function to generate changelog using OpenAI
+def generate_changelog_with_openai(commit_details):
+    commit_messages = []
+    for details in commit_details:
+        base_message = f"{details['pr_url']} - {details['message']}"
+        # Add the issue URL if available
+        # if details["issue_url"]:
+        #     base_message += f" (Linear Issue: {details['issue_url']})"
+        commit_messages.append(base_message)
+
+    commit_list = "\n".join(commit_messages)
+    prompt = """
+Generate a changelog for Infisical, opensource secretops
+The changelog should:
+1. Be Informative: Using the provided list of GitHub commits, break them down into categories such as Features, Fixes & Improvements, and Technical Updates. Summarize each commit concisely, ensuring the key points are highlighted.
+2. Have a Professional yet Friendly tone: The tone should be balanced, not too corporate or too informal.
+3. Celebratory Introduction and Conclusion: Start the changelog with a celebratory note acknowledging the team's hard work and progress. End with a shoutout to the team and wishes for a pleasant weekend.
+4. Formatting: you cannot use Markdown formatting, and you can only use emojis for the introductory paragraph or the conclusion paragraph, nowhere else.
+5. Links: the syntax to create links is the following: `<http://www.example.com|This message is a link>`.
+6. Linear Links: note that the Linear link is optional, include it only if provided.
+7. Do not wrap your answer in a codeblock. Just output the text, nothing else
+Here's a good example to follow, please try to match the formatting as closely as possible, only changing the content of the changelog and have some liberty with the introduction. Notice the importance of the formatting of a changelog item:
+- <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>))
+And here's an example of the full changelog:
+
+*Features*
+• <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>)
+*Fixes & Improvements*
+• <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>)
+*Technical Updates*
+• <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>)
+
+Stay tuned for more exciting updates coming soon!
+And here are the commits:
+{}
+    """.format(
+        commit_list
+    )
+
+    client  = OpenAI(api_key=OPENAI_API_KEY)
+    messages = [{"role": "user", "content": prompt}]
+    response = client.chat.completions.create(model="gpt-3.5-turbo", messages=messages)
+
+    if "error" in response.choices[0].message:
+        raise Exception("Error generating changelog with OpenAI!")
+
+    return response.choices[0].message.content.strip()
+
+
+if __name__ == "__main__":
+    try:
+        # Get the latest and previous release tags
+        latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"]).decode("utf-8").strip()
+        previous_tag = find_previous_release_tag(latest_tag)
+
+        latest_tag_date = get_tag_creation_date(latest_tag)
+        previous_tag_date = get_tag_creation_date(previous_tag)
+
+        prs = fetch_prs_between_tags(previous_tag_date,latest_tag_date)
+        pr_details = extract_commit_details_from_prs(prs)
+
+        # Generate changelog
+        changelog = generate_changelog_with_openai(pr_details)
+
+        post_changelog_to_slack(changelog,latest_tag)
+        # Print or post changelog to Slack
+        # set_multiline_output("changelog", changelog)
+
+    except Exception as e:
+        print(str(e))

.github/values.yaml

@@ -13,11 +13,10 @@ fullnameOverride: ""
 ##
 
 infisical:
-  ## @param backend.enabled Enable backend
+  autoDatabaseSchemaMigration: false
-  ##
+  
   enabled: false
-  ## @param backend.name Backend name
+
-  ##
   name: infisical
   replicaCount: 3
   image:
@@ -28,7 +27,7 @@ infisical:
   deploymentAnnotations:
     secrets.infisical.com/auto-reload: "true"
 
-  kubeSecretRef: "infisical-gamma-secrets"
+  kubeSecretRef: "managed-secret"
 
 ingress:
   ## @param ingress.enabled Enable ingress
@@ -50,3 +49,9 @@ ingress:
     - secretName: letsencrypt-prod
       hosts:
         - gamma.infisical.com
+
+postgresql:
+  enabled: false
+
+redis:
+  enabled: false

.github/workflows/build-docker-image-to-prod.yml

@@ -41,6 +41,7 @@ jobs:
           load: true
           context: backend
           tags: infisical/infisical:test
+          platforms: linux/amd64,linux/arm64
       - name: ⏻ Spawn backend container and dependencies
         run: |
           docker compose -f .github/resources/docker-compose.be-test.yml up --wait --quiet-pull
@@ -92,6 +93,7 @@ jobs:
           project: 64mmf0n610
           context: frontend
           tags: infisical/frontend:test
+          platforms: linux/amd64,linux/arm64
           build-args: |
             POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
             NEXT_INFISICAL_PLATFORM_VERSION=${{ steps.extract_version.outputs.version }}

.github/workflows/build-staging-and-deploy-aws.yml

@@ -0,0 +1,140 @@
+name: Deployment pipeline
+on: [workflow_dispatch]
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  infisical-image:
+    name: Build backend image
+    runs-on: ubuntu-latest
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+      - name: 📦 Install dependencies to test all dependencies
+        run: npm ci --only-production
+        working-directory: backend
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: 🔧 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: 🐋 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
+      - name: 🏗️ Build backend and push to docker hub
+        uses: depot/build-push-action@v1
+        with:
+          project: 64mmf0n610
+          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
+          push: true
+          context: .
+          file: Dockerfile.standalone-infisical
+          tags: |
+            infisical/staging_infisical:${{ steps.commit.outputs.short }}
+            infisical/staging_infisical:latest
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
+            INFISICAL_PLATFORM_VERSION=${{ steps.commit.outputs.short }}
+    
+  gamma-deployment:
+    name: Deploy to gamma
+    runs-on: ubuntu-latest
+    needs: [infisical-image]
+    environment:
+      name: Gamma
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v2
+        with:
+          node-version: "20"
+      - name: Change directory to backend and install dependencies
+        env:
+          DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
+        run: |
+          cd backend
+          npm install
+          npm run migration:latest
+      - name: Configure AWS Credentials 
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: us-east-1
+          role-to-assume: arn:aws:iam::905418227878:role/deploy-new-ecs-img
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: Download task definition
+        run: |
+          aws ecs describe-task-definition --task-definition infisical-prod-platform --query taskDefinition > task-definition.json
+      - name: Render Amazon ECS task definition
+        id: render-web-container
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: infisical-prod-platform
+          image: infisical/staging_infisical:${{ steps.commit.outputs.short }}
+          environment-variables: "LOG_LEVEL=info"
+      - name: Deploy to Amazon ECS service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.render-web-container.outputs.task-definition }}
+          service: infisical-prod-platform
+          cluster: infisical-prod-platform
+          wait-for-service-stability: true
+
+  production-postgres-deployment:
+    name: Deploy to production
+    runs-on: ubuntu-latest
+    needs: [gamma-deployment]
+    environment:
+      name: Production
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v2
+        with:
+          node-version: "20"
+      - name: Change directory to backend and install dependencies
+        env:
+          DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
+        run: |
+          cd backend
+          npm install
+          npm run migration:latest
+      - name: Configure AWS Credentials 
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: us-east-1
+          role-to-assume: arn:aws:iam::381492033652:role/gha-make-prod-deployment
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: Download task definition
+        run: |
+          aws ecs describe-task-definition --task-definition infisical-prod-platform --query taskDefinition > task-definition.json
+      - name: Render Amazon ECS task definition
+        id: render-web-container
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: infisical-prod-platform
+          image: infisical/staging_infisical:${{ steps.commit.outputs.short }}
+          environment-variables: "LOG_LEVEL=info"
+      - name: Deploy to Amazon ECS service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.render-web-container.outputs.task-definition }}
+          service: infisical-prod-platform
+          cluster: infisical-prod-platform
+          wait-for-service-stability: true

.github/workflows/build-staging-and-deploy.yml

@@ -1,120 +0,0 @@
-name: Build, Publish and Deploy to Gamma
-on: [workflow_dispatch]
-
-jobs:
-  infisical-image:
-    name: Build backend image
-    runs-on: ubuntu-latest
-    steps:
-      - name: ☁️ Checkout source
-        uses: actions/checkout@v3
-      - name: 📦 Install dependencies to test all dependencies
-        run: npm ci --only-production
-        working-directory: backend
-      # - name: 🧪 Run tests
-      #   run: npm run test:ci
-      #   working-directory: backend
-      - name: Save commit hashes for tag
-        id: commit
-        uses: pr-mpt/actions-commit-hash@v2
-      - name: 🔧 Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: 🐋 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Set up Depot CLI
-        uses: depot/setup-action@v1
-      - name: 📦 Build backend and export to Docker
-        uses: depot/build-push-action@v1
-        with:
-          project: 64mmf0n610
-          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
-          load: true
-          context: .
-          file: Dockerfile.standalone-infisical
-          tags: infisical/infisical:test
-      # - name: ⏻ Spawn backend container and dependencies
-      #   run: |
-      #     docker compose -f .github/resources/docker-compose.be-test.yml up --wait --quiet-pull
-      # - name: 🧪 Test backend image
-      #   run: |
-      #     ./.github/resources/healthcheck.sh infisical-backend-test
-      # - name: ⏻ Shut down backend container and dependencies
-      #   run: |
-      #     docker compose -f .github/resources/docker-compose.be-test.yml down
-      - name: 🏗️ Build backend and push
-        uses: depot/build-push-action@v1
-        with:
-          project: 64mmf0n610
-          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
-          push: true
-          context: .
-          file: Dockerfile.standalone-infisical
-          tags: |
-            infisical/staging_infisical:${{ steps.commit.outputs.short }}
-            infisical/staging_infisical:latest
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
-            INFISICAL_PLATFORM_VERSION=${{ steps.extract_version.outputs.version }}
-  postgres-migration:
-    name: Run latest migration files
-    runs-on: ubuntu-latest
-    needs: [infisical-image]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v2
-        with:
-          node-version: "20"
-      - name: Change directory to backend and install dependencies
-        env:
-          DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
-        run: |
-          cd backend
-          npm install
-          npm run migration:latest
-      # - name: Run postgres DB migration files
-      #   env:
-      #     DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
-      #   run: npm run migration:latest
-  gamma-deployment:
-    name: Deploy to gamma
-    runs-on: ubuntu-latest
-    needs: [postgres-migration]
-    steps:
-      - name: ☁️ Checkout source
-        uses: actions/checkout@v3
-      - name: Install Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.10.0
-      - name: Install infisical helm chart
-        run: |
-          helm repo add infisical-helm-charts 'https://dl.cloudsmith.io/public/infisical/helm-charts/helm/charts/'
-          helm repo update
-      - name: Install kubectl
-        uses: azure/setup-kubectl@v3
-      - name: Install doctl
-        uses: digitalocean/action-doctl@v2
-        with:
-          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-      - name: Save DigitalOcean kubeconfig with short-lived credentials
-        run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 infisical-gamma-postgres
-      - name: switch to gamma namespace
-        run: kubectl config set-context --current --namespace=gamma
-      - name: test kubectl
-        run: kubectl get ingress
-      - name: Download helm values to file and upgrade gamma deploy
-        run: |
-          wget https://raw.githubusercontent.com/Infisical/infisical/main/.github/values.yaml
-          helm upgrade infisical infisical-helm-charts/infisical-standalone  --values values.yaml --wait --install
-          if [[ $(helm status infisical) == *"FAILED"* ]]; then
-            echo "Helm upgrade failed"
-            exit 1
-          else
-            echo "Helm upgrade was successful"
-          fi

.github/workflows/check-api-for-breaking-changes.yml

@@ -0,0 +1,75 @@
+name: "Check API For Breaking Changes"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "backend/src/server/routes/**"
+
+jobs:
+  check-be-api-changes:
+    name: Check API Changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+      # - name: Setup Node 20
+      #   uses: actions/setup-node@v3
+      #   with:
+      #     node-version: "20"
+      # uncomment this when testing locally using nektos/act
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        if: ${{ env.ACT }}
+        name: Install `docker-compose` for local simulations
+        with:
+          version: "2.14.2"
+      - name: 📦Build the latest image
+        run: docker build --tag infisical-api .
+        working-directory: backend
+      - name: Start postgres and redis
+        run: touch .env && docker-compose -f docker-compose.dev.yml up -d db redis
+      - name: Start the server
+        run: |
+            echo "SECRET_SCANNING_GIT_APP_ID=793712" >> .env
+            echo "SECRET_SCANNING_PRIVATE_KEY=some-random" >> .env
+            echo "SECRET_SCANNING_WEBHOOK_SECRET=some-random" >> .env
+            docker run --name infisical-api -d -p 4000:4000 -e DB_CONNECTION_URI=$DB_CONNECTION_URI -e REDIS_URL=$REDIS_URL -e JWT_AUTH_SECRET=$JWT_AUTH_SECRET --env-file .env --entrypoint '/bin/sh' infisical-api -c "npm run migration:latest && ls && node dist/main.mjs"
+        env:
+          REDIS_URL: redis://172.17.0.1:6379
+          DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable
+          JWT_AUTH_SECRET: something-random
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.5'
+      - name: Wait for container to be stable and check logs
+        run: |
+          SECONDS=0
+          HEALTHY=0
+          while [ $SECONDS -lt 60 ]; do
+            if docker ps | grep infisical-api | grep -q healthy; then
+              echo "Container is healthy."
+              HEALTHY=1
+              break
+            fi
+            echo "Waiting for container to be healthy... ($SECONDS seconds elapsed)"
+
+            docker logs infisical-api
+
+            sleep 2
+            SECONDS=$((SECONDS+2))
+          done
+      
+          if [ $HEALTHY -ne 1 ]; then
+            echo "Container did not become healthy in time"
+            exit 1
+          fi
+      - name: Install openapi-diff
+        run: go install github.com/tufin/oasdiff@latest
+      - name: Running OpenAPI Spec diff action
+        run: oasdiff breaking https://app.infisical.com/api/docs/json http://localhost:4000/api/docs/json --fail-on ERR
+      - name: cleanup
+        run: |
+          docker-compose -f "docker-compose.dev.yml" down
+          docker stop infisical-api
+          docker remove infisical-api

.github/workflows/check-be-pull-request.yml

@@ -1,43 +0,0 @@
-name: "Check Backend Pull Request"
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-    paths:
-      - "backend/**"
-      - "!backend/README.md"
-      - "!backend/.*"
-      - "backend/.eslintrc.js"
-
-jobs:
-  check-be-pr:
-    name: Check
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: ☁️ Checkout source
-        uses: actions/checkout@v3
-      - name: 🔧 Setup Node 16
-        uses: actions/setup-node@v3
-        with:
-          node-version: "16"
-          cache: "npm"
-          cache-dependency-path: backend/package-lock.json
-      - name: 📦 Install dependencies
-        run: npm ci --only-production
-        working-directory: backend
-      # - name: 🧪 Run tests
-      #   run: npm run test:ci
-      #   working-directory: backend
-      # - name: 📁 Upload test results
-      #   uses: actions/upload-artifact@v3
-      #   if: always()
-      #   with:
-      #     name: be-test-results
-      #     path: |
-      #       ./backend/reports
-      #       ./backend/coverage
-      - name: 🏗️ Run build
-        run: npm run build
-        working-directory: backend

.github/workflows/check-be-ts-and-lint.yml

@@ -0,0 +1,35 @@
+name: "Check Backend PR types and lint"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "backend/**"
+      - "!backend/README.md"
+      - "!backend/.*"
+      - "backend/.eslintrc.js"
+
+jobs:
+  check-be-pr:
+    name: Check TS and Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+      - name: 🔧 Setup Node 20
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: backend/package-lock.json
+      - name: Install dependencies
+        run: npm install
+        working-directory: backend
+      - name: Run type check
+        run: npm run type:check
+        working-directory: backend
+      - name: Run lint check
+        run: npm run lint
+        working-directory: backend

.github/workflows/check-fe-ts-and-lint.yml

@@ -1,4 +1,4 @@
-name: Check Frontend Pull Request
+name: Check Frontend Type and Lint check
 
 on:
   pull_request:
@@ -10,8 +10,8 @@ on:
       - "frontend/.eslintrc.js"
 
 jobs:
-  check-fe-pr:
+  check-fe-ts-lint:
-    name: Check
+    name: Check Frontend Type and Lint check
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
@@ -25,12 +25,11 @@ jobs:
           cache: "npm"
           cache-dependency-path: frontend/package-lock.json
       - name: 📦 Install dependencies
-        run: npm ci --only-production --ignore-scripts
+        run: npm install
         working-directory: frontend
-      # -
+      - name: 🏗️ Run Type check
-      #   name: 🧪 Run tests
+        run: npm run type:check 
-      #   run: npm run test:ci
+        working-directory: frontend
-      #   working-directory: frontend
+      - name: 🏗️ Run Link check
-      - name: 🏗️ Run build
+        run: npm run lint:fix 
-        run: npm run build
         working-directory: frontend

.github/workflows/generate-release-changelog.yml

@@ -0,0 +1,34 @@
+name: Generate Changelog
+permissions:
+  contents: write
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "infisical/v*.*.*-postgres"
+
+jobs:
+  generate_changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12.0"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests openai
+      - name: Generate Changelog and Post to Slack
+        id: gen-changelog
+        run: python .github/resources/changelog-generator.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/release-standalone-docker-img-postgres-offical.yml

@@ -5,9 +5,14 @@ on:
       - "infisical/v*.*.*-postgres"
 
 jobs:
+  infisical-tests:
+    name: Run tests before deployment
+    # https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
+    uses: ./.github/workflows/run-backend-tests.yml
   infisical-standalone:
     name: Build infisical standalone image postgres
     runs-on: ubuntu-latest
+    needs: [infisical-tests]
     steps:
       - name: Extract version from tag
         id: extract_version

.github/workflows/release_build_infisical_cli.yml

@@ -23,6 +23,8 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🔧 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - run: git fetch --force --tags
       - run: echo "Ref name ${{github.ref_name}}"
       - uses: actions/setup-go@v3

.github/workflows/run-backend-tests.yml

@@ -0,0 +1,47 @@
+name: "Run backend tests"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "backend/**"
+      - "!backend/README.md"
+      - "!backend/.*"
+      - "backend/.eslintrc.js"
+  workflow_call:
+
+jobs:
+  check-be-pr:
+    name: Run integration test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        if: ${{ env.ACT }}
+        name: Install `docker-compose` for local simulations
+        with:
+          version: "2.14.2"
+      - name: 🔧 Setup Node 20
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: backend/package-lock.json
+      - name: Install dependencies
+        run: npm install
+        working-directory: backend
+      - name: Start postgres and redis
+        run: touch .env && docker-compose -f docker-compose.dev.yml up -d db redis
+      - name: Start integration test
+        run: npm run test:e2e
+        working-directory: backend
+        env:
+          REDIS_URL: redis://172.17.0.1:6379
+          DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable
+          AUTH_SECRET: something-random
+          ENCRYPTION_KEY: 4bnfe4e407b8921c104518903515b218
+      - name: cleanup
+        run: |
+          docker-compose -f "docker-compose.dev.yml" down

.gitignore

@@ -6,7 +6,7 @@ node_modules
 .env.gamma
 .env.prod
 .env.infisical
-
+.env.migration
 *~
 *.swp
 *.swo
@@ -63,3 +63,5 @@ yarn-error.log*
 .vscode/*
 
 frontend-build
+
+*.tgz

.goreleaser.yaml

@@ -190,10 +190,34 @@ dockers:
   - dockerfile: docker/alpine
     goos: linux
     goarch: amd64
+    use: buildx
     ids:
       - all-other-builds
     image_templates:
-      - "infisical/cli:{{ .Version }}"
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-amd64"
-      - "infisical/cli:{{ .Major }}.{{ .Minor }}"
+      - "infisical/cli:latest-amd64"
-      - "infisical/cli:{{ .Major }}"
+    build_flag_templates:
-      - "infisical/cli:latest"
+      - "--pull"
+      - "--platform=linux/amd64"
+  - dockerfile: docker/alpine
+    goos: linux
+    goarch: amd64
+    use: buildx
+    ids:
+      - all-other-builds
+    image_templates:
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-arm64"
+      - "infisical/cli:latest-arm64"
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/arm64"
+
+docker_manifests:
+  - name_template: "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
+    image_templates:
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-amd64"
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-arm64"
+  - name_template: "infisical/cli:latest"
+    image_templates:
+      - "infisical/cli:latest-amd64"
+      - "infisical/cli:latest-arm64"

CONTRIBUTING.md

@@ -2,6 +2,6 @@
 
 Thanks for taking the time to contribute! 😃 🚀
 
-Please refer to our [Contributing Guide](https://infisical.com/docs/contributing/overview) for instructions on how to contribute.
+Please refer to our [Contributing Guide](https://infisical.com/docs/contributing/getting-started/overview) for instructions on how to contribute.
 
 We also have some 🔥amazing🔥 merch for our contributors. Please reach out to tony@infisical.com for more info 👀

Dockerfile.standalone-infisical

@@ -118,9 +118,6 @@ WORKDIR /backend
 
 ENV TELEMETRY_ENABLED true
 
-HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \  
-  CMD node healthcheck.js
-
 EXPOSE 8080
 EXPOSE 443
 

Makefile

@@ -5,16 +5,13 @@ push:
 	docker-compose -f docker-compose.yml push
 
 up-dev:
-	docker-compose -f docker-compose.dev.yml up --build
+	docker compose -f docker-compose.dev.yml up --build
 
-up-pg-dev:
+up-dev-ldap:
-	docker compose -f docker-compose.pg.yml up --build
+	docker compose -f docker-compose.dev.yml --profile ldap up --build
-
-i-dev:
-	infisical run -- docker-compose -f docker-compose.dev.yml up --build
 
 up-prod:
-	docker-compose -f docker-compose.yml up --build
+	docker-compose -f docker-compose.prod.yml up --build
 
 down:
-	docker-compose down
+	docker compose -f docker-compose.dev.yml down

README.md

@@ -10,7 +10,8 @@
   <a href="https://infisical.com/">Infisical Cloud</a> |
   <a href="https://infisical.com/docs/self-hosting/overview">Self-Hosting</a> |
   <a href="https://infisical.com/docs/documentation/getting-started/introduction">Docs</a> |
-  <a href="https://www.infisical.com">Website</a>
+  <a href="https://www.infisical.com">Website</a> |
+  <a href="https://infisical.com/careers">Hiring (Remote/SF)</a>
 </h4>
 
 <p align="center">
@@ -84,13 +85,13 @@ To set up and run Infisical locally, make sure you have Git and Docker installed
 Linux/macOS:
 
 ```console
-git clone https://github.com/Infisical/infisical && cd "$(basename $_ .git)" && cp .env.example .env && docker-compose -f docker-compose.yml up
+git clone https://github.com/Infisical/infisical && cd "$(basename $_ .git)" && cp .env.example .env && docker-compose -f docker-compose.prod.yml up
 ```
 
 Windows Command Prompt:
 
 ```console
-git clone https://github.com/Infisical/infisical && cd infisical && copy .env.example .env && docker-compose -f docker-compose.yml up
+git clone https://github.com/Infisical/infisical && cd infisical && copy .env.example .env && docker-compose -f docker-compose.prod.yml up
 ```
 
 Create an account at `http://localhost:80`

backend/.eslintignore

@@ -1,2 +1,3 @@
 vitest-environment-infisical.ts
 vitest.config.ts
+vitest.e2e.config.ts

backend/.eslintrc.js

@@ -21,6 +21,18 @@ module.exports = {
     tsconfigRootDir: __dirname
   },
   root: true,
+  overrides: [
+    {
+      files: ["./e2e-test/**/*"],
+      rules: {
+        "@typescript-eslint/no-unsafe-member-access": "off",
+        "@typescript-eslint/no-unsafe-assignment": "off",
+        "@typescript-eslint/no-unsafe-argument": "off",
+        "@typescript-eslint/no-unsafe-return": "off",
+        "@typescript-eslint/no-unsafe-call": "off",
+      }
+    }
+  ],
   rules: {
     "@typescript-eslint/no-empty-function": "off",
     "@typescript-eslint/no-unsafe-enum-comparison": "off",

backend/e2e-test/mocks/keystore.ts

@@ -0,0 +1,30 @@
+import { TKeyStoreFactory } from "@app/keystore/keystore";
+
+export const mockKeyStore = (): TKeyStoreFactory => {
+  const store: Record<string, string | number | Buffer> = {};
+
+  return {
+    setItem: async (key, value) => {
+      store[key] = value;
+      return "OK";
+    },
+    setItemWithExpiry: async (key, value) => {
+      store[key] = value;
+      return "OK";
+    },
+    deleteItem: async (key) => {
+      delete store[key];
+      return 1;
+    },
+    getItem: async (key) => {
+      const value = store[key];
+      if (typeof value === "string") {
+        return value;
+      }
+      return null;
+    },
+    incrementBy: async () => {
+      return 1;
+    }
+  };
+};

backend/e2e-test/routes/v1/identity.spec.ts

@@ -0,0 +1,71 @@
+import { OrgMembershipRole } from "@app/db/schemas";
+import { seedData1 } from "@app/db/seed-data";
+
+export const createIdentity = async (name: string, role: string) => {
+  const createIdentityRes = await testServer.inject({
+    method: "POST",
+    url: "/api/v1/identities",
+    body: {
+      name,
+      role,
+      organizationId: seedData1.organization.id
+    },
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(createIdentityRes.statusCode).toBe(200);
+  return createIdentityRes.json().identity;
+};
+
+export const deleteIdentity = async (id: string) => {
+  const deleteIdentityRes = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/identities/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(deleteIdentityRes.statusCode).toBe(200);
+  return deleteIdentityRes.json().identity;
+};
+
+describe("Identity v1", async () => {
+  test("Create identity", async () => {
+    const newIdentity = await createIdentity("mac1", OrgMembershipRole.Admin);
+    expect(newIdentity.name).toBe("mac1");
+    expect(newIdentity.authMethod).toBeNull();
+
+    await deleteIdentity(newIdentity.id);
+  });
+
+  test("Update identity", async () => {
+    const newIdentity = await createIdentity("mac1", OrgMembershipRole.Admin);
+    expect(newIdentity.name).toBe("mac1");
+    expect(newIdentity.authMethod).toBeNull();
+
+    const updatedIdentity = await testServer.inject({
+      method: "PATCH",
+      url: `/api/v1/identities/${newIdentity.id}`,
+      headers: {
+        authorization: `Bearer ${jwtAuthToken}`
+      },
+      body: {
+        name: "updated-mac-1",
+        role: OrgMembershipRole.Member
+      }
+    });
+
+    expect(updatedIdentity.statusCode).toBe(200);
+    expect(updatedIdentity.json().identity.name).toBe("updated-mac-1");
+
+    await deleteIdentity(newIdentity.id);
+  });
+
+  test("Delete Identity", async () => {
+    const newIdentity = await createIdentity("mac1", OrgMembershipRole.Admin);
+
+    const deletedIdentity = await deleteIdentity(newIdentity.id);
+    expect(deletedIdentity.name).toBe("mac1");
+  });
+});

backend/e2e-test/routes/v1/login.spec.ts

@@ -1,6 +1,7 @@
-import { seedData1 } from "@app/db/seed-data";
 import jsrp from "jsrp";
 
+import { seedData1 } from "@app/db/seed-data";
+
 describe("Login V1 Router", async () => {
   // eslint-disable-next-line
   const client = new jsrp.client();

backend/e2e-test/routes/v1/project-env.spec.ts

@@ -1,6 +1,40 @@
 import { seedData1 } from "@app/db/seed-data";
 import { DEFAULT_PROJECT_ENVS } from "@app/db/seeds/3-project";
 
+const createProjectEnvironment = async (name: string, slug: string) => {
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/workspace/${seedData1.project.id}/environments`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      name,
+      slug
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("environment");
+  return payload.environment;
+};
+
+const deleteProjectEnvironment = async (envId: string) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/workspace/${seedData1.project.id}/environments/${envId}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("environment");
+  return payload.environment;
+};
+
 describe("Project Environment Router", async () => {
   test("Get default environments", async () => {
     const res = await testServer.inject({
@@ -31,24 +65,10 @@ describe("Project Environment Router", async () => {
     expect(payload.workspace.environments.length).toBe(3);
   });
 
-  const mockProjectEnv = { name: "temp", slug: "temp", id: "" }; // id will be filled in create op
+  const mockProjectEnv = { name: "temp", slug: "temp" }; // id will be filled in create op
   test("Create environment", async () => {
-    const res = await testServer.inject({
+    const newEnvironment = await createProjectEnvironment(mockProjectEnv.name, mockProjectEnv.slug);
-      method: "POST",
+    expect(newEnvironment).toEqual(
-      url: `/api/v1/workspace/${seedData1.project.id}/environments`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        name: mockProjectEnv.name,
-        slug: mockProjectEnv.slug
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("environment");
-    expect(payload.environment).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         name: mockProjectEnv.name,
@@ -59,14 +79,15 @@ describe("Project Environment Router", async () => {
         updatedAt: expect.any(String)
       })
     );
-    mockProjectEnv.id = payload.environment.id;
+    await deleteProjectEnvironment(newEnvironment.id);
   });
 
   test("Update environment", async () => {
+    const newEnvironment = await createProjectEnvironment(mockProjectEnv.name, mockProjectEnv.slug);
     const updatedName = { name: "temp#2", slug: "temp2" };
     const res = await testServer.inject({
       method: "PATCH",
-      url: `/api/v1/workspace/${seedData1.project.id}/environments/${mockProjectEnv.id}`,
+      url: `/api/v1/workspace/${seedData1.project.id}/environments/${newEnvironment.id}`,
       headers: {
         authorization: `Bearer ${jwtAuthToken}`
       },
@@ -82,7 +103,7 @@ describe("Project Environment Router", async () => {
     expect(payload).toHaveProperty("environment");
     expect(payload.environment).toEqual(
       expect.objectContaining({
-        id: expect.any(String),
+        id: newEnvironment.id,
         name: updatedName.name,
         slug: updatedName.slug,
         projectId: seedData1.project.id,
@@ -91,61 +112,21 @@ describe("Project Environment Router", async () => {
         updatedAt: expect.any(String)
       })
     );
-    mockProjectEnv.name = updatedName.name;
+    await deleteProjectEnvironment(newEnvironment.id);
-    mockProjectEnv.slug = updatedName.slug;
   });
 
   test("Delete environment", async () => {
-    const res = await testServer.inject({
+    const newEnvironment = await createProjectEnvironment(mockProjectEnv.name, mockProjectEnv.slug);
-      method: "DELETE",
+    const deletedProjectEnvironment = await deleteProjectEnvironment(newEnvironment.id);
-      url: `/api/v1/workspace/${seedData1.project.id}/environments/${mockProjectEnv.id}`,
+    expect(deletedProjectEnvironment).toEqual(
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("environment");
-    expect(payload.environment).toEqual(
       expect.objectContaining({
-        id: expect.any(String),
+        id: deletedProjectEnvironment.id,
         name: mockProjectEnv.name,
         slug: mockProjectEnv.slug,
-        position: 1,
+        position: 4,
         createdAt: expect.any(String),
         updatedAt: expect.any(String)
       })
     );
   });
-
-  // after all these opreations the list of environment should be still same
-  test("Default list of environment", async () => {
-    const res = await testServer.inject({
-      method: "GET",
-      url: `/api/v1/workspace/${seedData1.project.id}`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("workspace");
-    // check for default environments
-    expect(payload).toEqual({
-      workspace: expect.objectContaining({
-        name: seedData1.project.name,
-        id: seedData1.project.id,
-        slug: seedData1.project.slug,
-        environments: expect.arrayContaining([
-          expect.objectContaining(DEFAULT_PROJECT_ENVS[0]),
-          expect.objectContaining(DEFAULT_PROJECT_ENVS[1]),
-          expect.objectContaining(DEFAULT_PROJECT_ENVS[2])
-        ])
-      })
-    });
-    // ensure only two default environments exist
-    expect(payload.workspace.environments.length).toBe(3);
-  });
 });

backend/e2e-test/routes/v1/secret-folder.spec.ts

@@ -1,5 +1,40 @@
 import { seedData1 } from "@app/db/seed-data";
 
+const createFolder = async (dto: { path: string; name: string }) => {
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/folders`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      name: dto.name,
+      path: dto.path
+    }
+  });
+  expect(res.statusCode).toBe(200);
+  return res.json().folder;
+};
+
+const deleteFolder = async (dto: { path: string; id: string }) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/folders/${dto.id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      path: dto.path
+    }
+  });
+  expect(res.statusCode).toBe(200);
+  return res.json().folder;
+};
+
 describe("Secret Folder Router", async () => {
   test.each([
     { name: "folder1", path: "/" }, // one in root
@@ -7,30 +42,15 @@ describe("Secret Folder Router", async () => {
     { name: "folder2", path: "/" },
     { name: "folder1", path: "/level1/level2" } // this should not create folder return same thing
   ])("Create folder $name in $path", async ({ name, path }) => {
-    const res = await testServer.inject({
+    const createdFolder = await createFolder({ path, name });
-      method: "POST",
-      url: `/api/v1/folders`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        name,
-        path
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("folder");
     // check for default environments
-    expect(payload).toEqual({
+    expect(createdFolder).toEqual(
-      folder: expect.objectContaining({
+      expect.objectContaining({
         name,
         id: expect.any(String)
       })
-    });
+    );
+    await deleteFolder({ path, id: createdFolder.id });
   });
 
   test.each([
@@ -43,6 +63,8 @@ describe("Secret Folder Router", async () => {
     },
     { path: "/level1/level2", expected: { folders: [{ name: "folder1" }], length: 1 } }
   ])("Get folders $path", async ({ path, expected }) => {
+    const newFolders = await Promise.all(expected.folders.map(({ name }) => createFolder({ name, path })));
+
     const res = await testServer.inject({
       method: "GET",
       url: `/api/v1/folders`,
@@ -59,36 +81,22 @@ describe("Secret Folder Router", async () => {
     expect(res.statusCode).toBe(200);
     const payload = JSON.parse(res.payload);
     expect(payload).toHaveProperty("folders");
-    expect(payload.folders.length).toBe(expected.length);
+    expect(payload.folders.length >= expected.folders.length).toBeTruthy();
-    expect(payload).toEqual({ folders: expected.folders.map((el) => expect.objectContaining(el)) });
+    expect(payload).toEqual({
-  });
+      folders: expect.arrayContaining(expected.folders.map((el) => expect.objectContaining(el)))
-
-  let toBeDeleteFolderId = "";
-  test("Update a deep folder", async () => {
-    const res = await testServer.inject({
-      method: "PATCH",
-      url: `/api/v1/folders/folder1`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        name: "folder-updated",
-        path: "/level1/level2"
-      }
     });
 
-    expect(res.statusCode).toBe(200);
+    await Promise.all(newFolders.map(({ id }) => deleteFolder({ path, id })));
-    const payload = JSON.parse(res.payload);
+  });
-    expect(payload).toHaveProperty("folder");
+
-    expect(payload.folder).toEqual(
+  test("Update a deep folder", async () => {
+    const newFolder = await createFolder({ name: "folder-updated", path: "/level1/level2" });
+    expect(newFolder).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         name: "folder-updated"
       })
     );
-    toBeDeleteFolderId = payload.folder.id;
 
     const resUpdatedFolders = await testServer.inject({
       method: "GET",
@@ -106,14 +114,16 @@ describe("Secret Folder Router", async () => {
     expect(resUpdatedFolders.statusCode).toBe(200);
     const updatedFolderList = JSON.parse(resUpdatedFolders.payload);
     expect(updatedFolderList).toHaveProperty("folders");
-    expect(updatedFolderList.folders.length).toEqual(1);
     expect(updatedFolderList.folders[0].name).toEqual("folder-updated");
+
+    await deleteFolder({ path: "/level1/level2", id: newFolder.id });
   });
 
   test("Delete a deep folder", async () => {
+    const newFolder = await createFolder({ name: "folder-updated", path: "/level1/level2" });
     const res = await testServer.inject({
       method: "DELETE",
-      url: `/api/v1/folders/${toBeDeleteFolderId}`,
+      url: `/api/v1/folders/${newFolder.id}`,
       headers: {
         authorization: `Bearer ${jwtAuthToken}`
       },

backend/e2e-test/routes/v1/secret-import.spec.ts

@@ -1,32 +1,57 @@
 import { seedData1 } from "@app/db/seed-data";
 
-describe("Secret Folder Router", async () => {
+const createSecretImport = async (importPath: string, importEnv: string) => {
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/secret-imports`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      path: "/",
+      import: {
+        environment: importEnv,
+        path: importPath
+      }
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("secretImport");
+  return payload.secretImport;
+};
+
+const deleteSecretImport = async (id: string) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/secret-imports/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      path: "/"
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("secretImport");
+  return payload.secretImport;
+};
+
+describe("Secret Import Router", async () => {
   test.each([
     { importEnv: "dev", importPath: "/" }, // one in root
     { importEnv: "staging", importPath: "/" } // then create a deep one creating intermediate ones
   ])("Create secret import $importEnv with path $importPath", async ({ importPath, importEnv }) => {
-    const res = await testServer.inject({
-      method: "POST",
-      url: `/api/v1/secret-imports`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        path: "/",
-        import: {
-          environment: importEnv,
-          path: importPath
-        }
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("secretImport");
     // check for default environments
-    expect(payload.secretImport).toEqual(
+    const payload = await createSecretImport(importPath, importEnv);
+    expect(payload).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         importPath: expect.any(String),
@@ -37,10 +62,12 @@ describe("Secret Folder Router", async () => {
         })
       })
     );
+    await deleteSecretImport(payload.id);
   });
 
-  let testSecretImportId = "";
   test("Get secret imports", async () => {
+    const createdImport1 = await createSecretImport("/", "dev");
+    const createdImport2 = await createSecretImport("/", "staging");
     const res = await testServer.inject({
       method: "GET",
       url: `/api/v1/secret-imports`,
@@ -58,7 +85,6 @@ describe("Secret Folder Router", async () => {
     const payload = JSON.parse(res.payload);
     expect(payload).toHaveProperty("secretImports");
     expect(payload.secretImports.length).toBe(2);
-    testSecretImportId = payload.secretImports[0].id;
     expect(payload.secretImports).toEqual(
       expect.arrayContaining([
         expect.objectContaining({
@@ -72,12 +98,20 @@ describe("Secret Folder Router", async () => {
         })
       ])
     );
+    await deleteSecretImport(createdImport1.id);
+    await deleteSecretImport(createdImport2.id);
   });
 
   test("Update secret import position", async () => {
-    const res = await testServer.inject({
+    const devImportDetails = { path: "/", envSlug: "dev" };
+    const stagingImportDetails = { path: "/", envSlug: "staging" };
+
+    const createdImport1 = await createSecretImport(devImportDetails.path, devImportDetails.envSlug);
+    const createdImport2 = await createSecretImport(stagingImportDetails.path, stagingImportDetails.envSlug);
+
+    const updateImportRes = await testServer.inject({
       method: "PATCH",
-      url: `/api/v1/secret-imports/${testSecretImportId}`,
+      url: `/api/v1/secret-imports/${createdImport1.id}`,
       headers: {
         authorization: `Bearer ${jwtAuthToken}`
       },
@@ -91,8 +125,8 @@ describe("Secret Folder Router", async () => {
       }
     });
 
-    expect(res.statusCode).toBe(200);
+    expect(updateImportRes.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
+    const payload = JSON.parse(updateImportRes.payload);
     expect(payload).toHaveProperty("secretImport");
     // check for default environments
     expect(payload.secretImport).toEqual(
@@ -102,7 +136,7 @@ describe("Secret Folder Router", async () => {
         position: 2,
         importEnv: expect.objectContaining({
           name: expect.any(String),
-          slug: expect.any(String),
+          slug: expect.stringMatching(devImportDetails.envSlug),
           id: expect.any(String)
         })
       })
@@ -124,28 +158,19 @@ describe("Secret Folder Router", async () => {
     expect(secretImportsListRes.statusCode).toBe(200);
     const secretImportList = JSON.parse(secretImportsListRes.payload);
     expect(secretImportList).toHaveProperty("secretImports");
-    expect(secretImportList.secretImports[1].id).toEqual(testSecretImportId);
+    expect(secretImportList.secretImports[1].id).toEqual(createdImport1.id);
+    expect(secretImportList.secretImports[0].id).toEqual(createdImport2.id);
+
+    await deleteSecretImport(createdImport1.id);
+    await deleteSecretImport(createdImport2.id);
   });
 
   test("Delete secret import position", async () => {
-    const res = await testServer.inject({
+    const createdImport1 = await createSecretImport("/", "dev");
-      method: "DELETE",
+    const createdImport2 = await createSecretImport("/", "staging");
-      url: `/api/v1/secret-imports/${testSecretImportId}`,
+    const deletedImport = await deleteSecretImport(createdImport1.id);
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        path: "/"
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("secretImport");
     // check for default environments
-    expect(payload.secretImport).toEqual(
+    expect(deletedImport).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         importPath: expect.any(String),
@@ -175,5 +200,7 @@ describe("Secret Folder Router", async () => {
     expect(secretImportList).toHaveProperty("secretImports");
     expect(secretImportList.secretImports.length).toEqual(1);
     expect(secretImportList.secretImports[0].position).toEqual(1);
+
+    await deleteSecretImport(createdImport2.id);
   });
 });

backend/e2e-test/routes/v2/service-token.spec.ts

@@ -0,0 +1,579 @@
+import crypto from "node:crypto";
+
+import { SecretType, TSecrets } from "@app/db/schemas";
+import { decryptSecret, encryptSecret, getUserPrivateKey, seedData1 } from "@app/db/seed-data";
+import { decryptAsymmetric, decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+
+const createServiceToken = async (
+  scopes: { environment: string; secretPath: string }[],
+  permissions: ("read" | "write")[]
+) => {
+  const projectKeyRes = await testServer.inject({
+    method: "GET",
+    url: `/api/v2/workspace/${seedData1.project.id}/encrypted-key`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  const projectKeyEnc = JSON.parse(projectKeyRes.payload);
+
+  const userInfoRes = await testServer.inject({
+    method: "GET",
+    url: "/api/v2/users/me",
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  const { user: userInfo } = JSON.parse(userInfoRes.payload);
+  const privateKey = await getUserPrivateKey(seedData1.password, userInfo);
+  const projectKey = decryptAsymmetric({
+    ciphertext: projectKeyEnc.encryptedKey,
+    nonce: projectKeyEnc.nonce,
+    publicKey: projectKeyEnc.sender.publicKey,
+    privateKey
+  });
+
+  const randomBytes = crypto.randomBytes(16).toString("hex");
+  const { ciphertext, iv, tag } = encryptSymmetric128BitHexKeyUTF8(projectKey, randomBytes);
+  const serviceTokenRes = await testServer.inject({
+    method: "POST",
+    url: "/api/v2/service-token",
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      name: "test-token",
+      workspaceId: seedData1.project.id,
+      scopes,
+      encryptedKey: ciphertext,
+      iv,
+      tag,
+      permissions,
+      expiresIn: null
+    }
+  });
+  expect(serviceTokenRes.statusCode).toBe(200);
+  const serviceTokenInfo = serviceTokenRes.json();
+  expect(serviceTokenInfo).toHaveProperty("serviceToken");
+  expect(serviceTokenInfo).toHaveProperty("serviceTokenData");
+  return `${serviceTokenInfo.serviceToken}.${randomBytes}`;
+};
+
+const deleteServiceToken = async () => {
+  const serviceTokenListRes = await testServer.inject({
+    method: "GET",
+    url: `/api/v1/workspace/${seedData1.project.id}/service-token-data`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(serviceTokenListRes.statusCode).toBe(200);
+  const serviceTokens = JSON.parse(serviceTokenListRes.payload).serviceTokenData as { name: string; id: string }[];
+  expect(serviceTokens.length).toBeGreaterThan(0);
+  const serviceTokenInfo = serviceTokens.find(({ name }) => name === "test-token");
+  expect(serviceTokenInfo).toBeDefined();
+
+  const deleteTokenRes = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v2/service-token/${serviceTokenInfo?.id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(deleteTokenRes.statusCode).toBe(200);
+};
+
+const createSecret = async (dto: {
+  projectKey: string;
+  path: string;
+  key: string;
+  value: string;
+  comment: string;
+  type?: SecretType;
+  token: string;
+}) => {
+  const createSecretReqBody = {
+    workspaceId: seedData1.project.id,
+    environment: seedData1.environment.slug,
+    type: dto.type || SecretType.Shared,
+    secretPath: dto.path,
+    ...encryptSecret(dto.projectKey, dto.key, dto.value, dto.comment)
+  };
+  const createSecRes = await testServer.inject({
+    method: "POST",
+    url: `/api/v3/secrets/${dto.key}`,
+    headers: {
+      authorization: `Bearer ${dto.token}`
+    },
+    body: createSecretReqBody
+  });
+  expect(createSecRes.statusCode).toBe(200);
+  const createdSecretPayload = JSON.parse(createSecRes.payload);
+  expect(createdSecretPayload).toHaveProperty("secret");
+  return createdSecretPayload.secret;
+};
+
+const deleteSecret = async (dto: { path: string; key: string; token: string }) => {
+  const deleteSecRes = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v3/secrets/${dto.key}`,
+    headers: {
+      authorization: `Bearer ${dto.token}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      secretPath: dto.path
+    }
+  });
+  expect(deleteSecRes.statusCode).toBe(200);
+  const updatedSecretPayload = JSON.parse(deleteSecRes.payload);
+  expect(updatedSecretPayload).toHaveProperty("secret");
+  return updatedSecretPayload.secret;
+};
+
+describe("Service token secret ops", async () => {
+  let serviceToken = "";
+  let projectKey = "";
+  let folderId = "";
+  beforeAll(async () => {
+    serviceToken = await createServiceToken(
+      [{ secretPath: "/**", environment: seedData1.environment.slug }],
+      ["read", "write"]
+    );
+
+    // this is ensure cli service token decryptiong working fine
+    const serviceTokenInfoRes = await testServer.inject({
+      method: "GET",
+      url: "/api/v2/service-token",
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(serviceTokenInfoRes.statusCode).toBe(200);
+    const serviceTokenInfo = serviceTokenInfoRes.json();
+    const serviceTokenParts = serviceToken.split(".");
+    projectKey = decryptSymmetric128BitHexKeyUTF8({
+      key: serviceTokenParts[3],
+      tag: serviceTokenInfo.tag,
+      ciphertext: serviceTokenInfo.encryptedKey,
+      iv: serviceTokenInfo.iv
+    });
+
+    // create a deep folder
+    const folderCreate = await testServer.inject({
+      method: "POST",
+      url: `/api/v1/folders`,
+      headers: {
+        authorization: `Bearer ${jwtAuthToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        name: "folder",
+        path: "/nested1/nested2"
+      }
+    });
+    expect(folderCreate.statusCode).toBe(200);
+    folderId = folderCreate.json().folder.id;
+  });
+
+  afterAll(async () => {
+    await deleteServiceToken();
+
+    // create a deep folder
+    const deleteFolder = await testServer.inject({
+      method: "DELETE",
+      url: `/api/v1/folders/${folderId}`,
+      headers: {
+        authorization: `Bearer ${jwtAuthToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        path: "/nested1/nested2"
+      }
+    });
+    expect(deleteFolder.statusCode).toBe(200);
+  });
+
+  const testSecrets = [
+    {
+      path: "/",
+      secret: {
+        key: "ST-SEC",
+        value: "something-secret",
+        comment: "some comment"
+      }
+    },
+    {
+      path: "/nested1/nested2/folder",
+      secret: {
+        key: "NESTED-ST-SEC",
+        value: "something-secret",
+        comment: "some comment"
+      }
+    }
+  ];
+
+  const getSecrets = async (environment: string, secretPath = "/") => {
+    const res = await testServer.inject({
+      method: "GET",
+      url: `/api/v3/secrets`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      query: {
+        secretPath,
+        environment,
+        workspaceId: seedData1.project.id
+      }
+    });
+    const secrets: TSecrets[] = JSON.parse(res.payload).secrets || [];
+    return secrets.map((el) => ({ ...decryptSecret(projectKey, el), type: el.type }));
+  };
+
+  test.each(testSecrets)("Create secret in path $path", async ({ secret, path }) => {
+    const createdSecret = await createSecret({ projectKey, path, ...secret, token: serviceToken });
+    const decryptedSecret = decryptSecret(projectKey, createdSecret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+    expect(decryptedSecret.value).toEqual(secret.value);
+    expect(decryptedSecret.comment).toEqual(secret.comment);
+    expect(decryptedSecret.version).toEqual(1);
+
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          key: secret.key,
+          value: secret.value,
+          type: SecretType.Shared
+        })
+      ])
+    );
+    await deleteSecret({ path, key: secret.key, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Get secret by name in path $path", async ({ secret, path }) => {
+    await createSecret({ projectKey, path, ...secret, token: serviceToken });
+
+    const getSecByNameRes = await testServer.inject({
+      method: "GET",
+      url: `/api/v3/secrets/${secret.key}`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      query: {
+        secretPath: path,
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug
+      }
+    });
+    expect(getSecByNameRes.statusCode).toBe(200);
+    const getSecretByNamePayload = JSON.parse(getSecByNameRes.payload);
+    expect(getSecretByNamePayload).toHaveProperty("secret");
+    const decryptedSecret = decryptSecret(projectKey, getSecretByNamePayload.secret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+    expect(decryptedSecret.value).toEqual(secret.value);
+    expect(decryptedSecret.comment).toEqual(secret.comment);
+
+    await deleteSecret({ path, key: secret.key, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Update secret in path $path", async ({ path, secret }) => {
+    await createSecret({ projectKey, path, ...secret, token: serviceToken });
+    const updateSecretReqBody = {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      type: SecretType.Shared,
+      secretPath: path,
+      ...encryptSecret(projectKey, secret.key, "new-value", secret.comment)
+    };
+    const updateSecRes = await testServer.inject({
+      method: "PATCH",
+      url: `/api/v3/secrets/${secret.key}`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: updateSecretReqBody
+    });
+    expect(updateSecRes.statusCode).toBe(200);
+    const updatedSecretPayload = JSON.parse(updateSecRes.payload);
+    expect(updatedSecretPayload).toHaveProperty("secret");
+    const decryptedSecret = decryptSecret(projectKey, updatedSecretPayload.secret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+    expect(decryptedSecret.value).toEqual("new-value");
+    expect(decryptedSecret.comment).toEqual(secret.comment);
+
+    // list secret should have updated value
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          key: secret.key,
+          value: "new-value",
+          type: SecretType.Shared
+        })
+      ])
+    );
+
+    await deleteSecret({ path, key: secret.key, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Delete secret in path $path", async ({ secret, path }) => {
+    await createSecret({ projectKey, path, ...secret, token: serviceToken });
+    const deletedSecret = await deleteSecret({ path, key: secret.key, token: serviceToken });
+    const decryptedSecret = decryptSecret(projectKey, deletedSecret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+
+    // shared secret deletion should delete personal ones also
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.not.arrayContaining([
+        expect.objectContaining({
+          key: secret.key,
+          type: SecretType.Shared
+        })
+      ])
+    );
+  });
+
+  test.each(testSecrets)("Bulk create secrets in path $path", async ({ secret, path }) => {
+    const createSharedSecRes = await testServer.inject({
+      method: "POST",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`,
+          ...encryptSecret(projectKey, `BULK-${secret.key}-${i + 1}`, secret.value, secret.comment)
+        }))
+      }
+    });
+    expect(createSharedSecRes.statusCode).toBe(200);
+    const createSharedSecPayload = JSON.parse(createSharedSecRes.payload);
+    expect(createSharedSecPayload).toHaveProperty("secrets");
+
+    // bulk ones should exist
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining(
+        Array.from(Array(5)).map((_e, i) =>
+          expect.objectContaining({
+            key: `BULK-${secret.key}-${i + 1}`,
+            type: SecretType.Shared
+          })
+        )
+      )
+    );
+
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        deleteSecret({ path, token: serviceToken, key: `BULK-${secret.key}-${i + 1}` })
+      )
+    );
+  });
+
+  test.each(testSecrets)("Bulk create fail on existing secret in path $path", async ({ secret, path }) => {
+    await createSecret({ projectKey, ...secret, key: `BULK-${secret.key}-1`, path, token: serviceToken });
+
+    const createSharedSecRes = await testServer.inject({
+      method: "POST",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`,
+          ...encryptSecret(projectKey, `BULK-${secret.key}-${i + 1}`, secret.value, secret.comment)
+        }))
+      }
+    });
+    expect(createSharedSecRes.statusCode).toBe(400);
+
+    await deleteSecret({ path, key: `BULK-${secret.key}-1`, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Bulk update secrets in path $path", async ({ secret, path }) => {
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        createSecret({ projectKey, token: serviceToken, ...secret, key: `BULK-${secret.key}-${i + 1}`, path })
+      )
+    );
+
+    const updateSharedSecRes = await testServer.inject({
+      method: "PATCH",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`,
+          ...encryptSecret(projectKey, `BULK-${secret.key}-${i + 1}`, "update-value", secret.comment)
+        }))
+      }
+    });
+    expect(updateSharedSecRes.statusCode).toBe(200);
+    const updateSharedSecPayload = JSON.parse(updateSharedSecRes.payload);
+    expect(updateSharedSecPayload).toHaveProperty("secrets");
+
+    // bulk ones should exist
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining(
+        Array.from(Array(5)).map((_e, i) =>
+          expect.objectContaining({
+            key: `BULK-${secret.key}-${i + 1}`,
+            value: "update-value",
+            type: SecretType.Shared
+          })
+        )
+      )
+    );
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        deleteSecret({ path, key: `BULK-${secret.key}-${i + 1}`, token: serviceToken })
+      )
+    );
+  });
+
+  test.each(testSecrets)("Bulk delete secrets in path $path", async ({ secret, path }) => {
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        createSecret({ projectKey, token: serviceToken, ...secret, key: `BULK-${secret.key}-${i + 1}`, path })
+      )
+    );
+
+    const deletedSharedSecRes = await testServer.inject({
+      method: "DELETE",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`
+        }))
+      }
+    });
+
+    expect(deletedSharedSecRes.statusCode).toBe(200);
+    const deletedSecretPayload = JSON.parse(deletedSharedSecRes.payload);
+    expect(deletedSecretPayload).toHaveProperty("secrets");
+
+    // bulk ones should exist
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.not.arrayContaining(
+        Array.from(Array(5)).map((_e, i) =>
+          expect.objectContaining({
+            key: `BULK-${secret.value}-${i + 1}`,
+            type: SecretType.Shared
+          })
+        )
+      )
+    );
+  });
+});
+
+describe("Service token fail cases", async () => {
+  test("Unauthorized secret path access", async () => {
+    const serviceToken = await createServiceToken(
+      [{ secretPath: "/", environment: seedData1.environment.slug }],
+      ["read", "write"]
+    );
+    const fetchSecrets = await testServer.inject({
+      method: "GET",
+      url: "/api/v3/secrets",
+      query: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: "/nested/deep"
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(fetchSecrets.statusCode).toBe(401);
+    expect(fetchSecrets.json().error).toBe("PermissionDenied");
+    await deleteServiceToken();
+  });
+
+  test("Unauthorized secret environment access", async () => {
+    const serviceToken = await createServiceToken(
+      [{ secretPath: "/", environment: seedData1.environment.slug }],
+      ["read", "write"]
+    );
+    const fetchSecrets = await testServer.inject({
+      method: "GET",
+      url: "/api/v3/secrets",
+      query: {
+        workspaceId: seedData1.project.id,
+        environment: "prod",
+        secretPath: "/"
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(fetchSecrets.statusCode).toBe(401);
+    expect(fetchSecrets.json().error).toBe("PermissionDenied");
+    await deleteServiceToken();
+  });
+
+  test("Unauthorized write operation", async () => {
+    const serviceToken = await createServiceToken(
+      [{ secretPath: "/", environment: seedData1.environment.slug }],
+      ["read"]
+    );
+    const writeSecrets = await testServer.inject({
+      method: "POST",
+      url: `/api/v3/secrets/NEW`,
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        type: SecretType.Shared,
+        secretPath: "/",
+        // doesn't matter project key because this will fail before that due to read only access
+        ...encryptSecret(crypto.randomBytes(16).toString("hex"), "NEW", "value", "")
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(writeSecrets.statusCode).toBe(401);
+    expect(writeSecrets.json().error).toBe("PermissionDenied");
+
+    // but read access should still work fine
+    const fetchSecrets = await testServer.inject({
+      method: "GET",
+      url: "/api/v3/secrets",
+      query: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: "/"
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(fetchSecrets.statusCode).toBe(200);
+    await deleteServiceToken();
+  });
+});

backend/e2e-test/vitest-environment-knex.ts

@@ -1,26 +1,31 @@
-// import { main } from "@app/server/app";
+// eslint-disable-next-line
-import { initEnvConfig } from "@app/lib/config/env";
+import "ts-node/register";
+
 import dotenv from "dotenv";
+import jwt from "jsonwebtoken";
 import knex from "knex";
 import path from "path";
-import { mockSmtpServer } from "./mocks/smtp";
-import { initLogger } from "@app/lib/logger";
-import jwt from "jsonwebtoken";
 
-import "ts-node/register";
-import { main } from "@app/server/app";
-import { mockQueue } from "./mocks/queue";
-import { AuthTokenType } from "@app/services/auth/auth-type";
 import { seedData1 } from "@app/db/seed-data";
+import { initEnvConfig } from "@app/lib/config/env";
+import { initLogger } from "@app/lib/logger";
+import { main } from "@app/server/app";
+import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
 
-dotenv.config({ path: path.join(__dirname, "../.env.test") });
+import { mockQueue } from "./mocks/queue";
+import { mockSmtpServer } from "./mocks/smtp";
+import { mockKeyStore } from "./mocks/keystore";
+
+dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true });
 export default {
   name: "knex-env",
   transformMode: "ssr",
   async setup() {
+    const logger = await initLogger();
+    const cfg = initEnvConfig(logger);
     const db = knex({
       client: "pg",
-      connection: process.env.DB_CONNECTION_URI,
+      connection: cfg.DB_CONNECTION_URI,
       migrations: {
         directory: path.join(__dirname, "../src/db/migrations"),
         extension: "ts",
@@ -37,9 +42,8 @@ export default {
       await db.seed.run();
       const smtp = mockSmtpServer();
       const queue = mockQueue();
-      const logger = await initLogger();
+      const keyStore = mockKeyStore();
-      const cfg = initEnvConfig(logger);
+      const server = await main({ db, smtp, logger, queue, keyStore });
-      const server = await main({ db, smtp, logger, queue });
       // @ts-expect-error type
       globalThis.testServer = server;
       // @ts-expect-error type
@@ -48,12 +52,15 @@ export default {
           authTokenType: AuthTokenType.ACCESS_TOKEN,
           userId: seedData1.id,
           tokenVersionId: seedData1.token.id,
+          authMethod: AuthMethod.EMAIL,
+          organizationId: seedData1.organization.id,
           accessVersion: 1
         },
         cfg.AUTH_SECRET,
         { expiresIn: cfg.JWT_AUTH_LIFETIME }
       );
     } catch (error) {
+      console.log("[TEST] Error setting up environment", error);
       await db.destroy();
       throw error;
     }

backend/package.json

@@ -24,8 +24,8 @@
     "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest",
     "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback",
     "seed:new": "tsx ./scripts/create-seed-file.ts",
-    "seed:run": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
+    "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
-    "db:reset": "npm run migration:rollback -- --all && npm run migration:latest && npm run seed:run"
+    "db:reset": "npm run migration:rollback -- --all && npm run migration:latest"
   },
   "keywords": [],
   "author": "",
@@ -67,21 +67,22 @@
     "tsx": "^4.4.0",
     "typescript": "^5.3.2",
     "vite-tsconfig-paths": "^4.2.2",
-    "vitest": "^1.0.4"
+    "vitest": "^1.2.2"
   },
   "dependencies": {
-    "@aws-sdk/client-secrets-manager": "^3.485.0",
+    "@aws-sdk/client-iam": "^3.525.0",
+    "@aws-sdk/client-secrets-manager": "^3.504.0",
     "@casl/ability": "^6.5.0",
-    "@fastify/cookie": "^9.2.0",
+    "@fastify/cookie": "^9.3.1",
-    "@fastify/cors": "^8.4.1",
+    "@fastify/cors": "^8.5.0",
     "@fastify/etag": "^5.1.0",
     "@fastify/formbody": "^7.4.0",
     "@fastify/helmet": "^11.1.1",
     "@fastify/passport": "^2.4.0",
     "@fastify/rate-limit": "^9.0.0",
     "@fastify/session": "^10.7.0",
-    "@fastify/swagger": "^8.12.0",
+    "@fastify/swagger": "^8.14.0",
-    "@fastify/swagger-ui": "^1.10.1",
+    "@fastify/swagger-ui": "^2.1.0",
     "@node-saml/passport-saml": "^4.0.4",
     "@octokit/rest": "^20.0.2",
     "@octokit/webhooks-types": "^7.3.1",
@@ -90,13 +91,13 @@
     "@ucast/mongo2js": "^1.3.4",
     "ajv": "^8.12.0",
     "argon2": "^0.31.2",
-    "aws-sdk": "^2.1532.0",
+    "aws-sdk": "^2.1553.0",
-    "axios": "^1.6.2",
+    "axios": "^1.6.7",
     "axios-retry": "^4.0.0",
     "bcrypt": "^5.1.1",
-    "bullmq": "^5.1.1",
+    "bullmq": "^5.3.3",
-    "dotenv": "^16.3.1",
+    "dotenv": "^16.4.1",
-    "fastify": "^4.24.3",
+    "fastify": "^4.26.0",
     "fastify-plugin": "^4.5.1",
     "handlebars": "^4.7.8",
     "ioredis": "^5.3.2",
@@ -106,23 +107,26 @@
     "knex": "^3.0.1",
     "libsodium-wrappers": "^0.7.13",
     "lodash.isequal": "^4.5.0",
-    "mysql2": "^3.6.5",
+    "ms": "^2.1.3",
+    "mysql2": "^3.9.1",
     "nanoid": "^5.0.4",
-    "node-cache": "^5.1.2",
+    "nodemailer": "^6.9.9",
-    "nodemailer": "^6.9.7",
+    "ora": "^7.0.1",
     "passport-github": "^1.1.0",
     "passport-gitlab2": "^5.0.0",
     "passport-google-oauth20": "^2.0.0",
+    "passport-ldapauth": "^3.0.1",
     "pg": "^8.11.3",
+    "pg-query-stream": "^4.5.3",
     "picomatch": "^3.0.1",
     "pino": "^8.16.2",
-    "posthog-node": "^3.6.0",
+    "posthog-node": "^3.6.2",
-    "probot": "^12.3.3",
+    "probot": "^13.0.0",
     "smee-client": "^2.0.0",
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
     "uuid": "^9.0.1",
     "zod": "^3.22.4",
-    "zod-to-json-schema": "^3.22.0"
+    "zod-to-json-schema": "^3.22.4"
   }
 }

backend/scripts/create-backend-file.ts

@@ -103,11 +103,15 @@ export const ${dalName} = (db: TDbClient) => {
     `import { z } from "zod";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
+import { readLimit } from "@app/server/config/rateLimiter";
 
 export const register${pascalCase}Router = async (server: FastifyZodProvider) => {
   server.route({
-    url: "/",
     method: "GET",
+    url: "/",
+    config: {
+      rateLimit: readLimit
+    },
     schema: {
       params: z.object({}),
       response: {

backend/scripts/create-migration.ts

@@ -7,10 +7,10 @@ const prompt = promptSync({ sigint: true });
 
 const migrationName = prompt("Enter name for migration: ");
 
+// Remove spaces from migration name and replace with hyphens
+const formattedMigrationName = migrationName.replace(/\s+/g, "-");
+
 execSync(
-  `npx knex migrate:make --knexfile ${path.join(
+  `npx knex migrate:make --knexfile ${path.join(__dirname, "../src/db/knexfile.ts")} -x ts ${formattedMigrationName}`,
-    __dirname,
-    "../src/db/knexfile.ts"
-  )} -x ts ${migrationName}`,
   { stdio: "inherit" }
 );

backend/scripts/create-seed-file.ts

@@ -7,11 +7,10 @@ import promptSync from "prompt-sync";
 const prompt = promptSync({ sigint: true });
 
 const migrationName = prompt("Enter name for seedfile: ");
-const fileCounter = readdirSync(path.join(__dirname, "../src/db/seed")).length || 1;
+const fileCounter = readdirSync(path.join(__dirname, "../src/db/seeds")).length || 1;
 execSync(
-  `npx knex seed:make --knexfile ${path.join(
+  `npx knex seed:make --knexfile ${path.join(__dirname, "../src/db/knexfile.ts")} -x ts ${
-    __dirname,
+    fileCounter + 1
-    "../src/db/knexfile.ts"
+  }-${migrationName}`,
-  )} -x ts ${fileCounter}-${migrationName}`,
   { stdio: "inherit" }
 );

backend/scripts/generate-schema-types.ts

@@ -3,13 +3,9 @@ import dotenv from "dotenv";
 import path from "path";
 import knex from "knex";
 import { writeFileSync } from "fs";
-import promptSync from "prompt-sync";
-
-const prompt = promptSync({ sigint: true });
 
 dotenv.config({
-  path: path.join(__dirname, "../.env"),
+  path: path.join(__dirname, "../../.env.migration")
-  debug: true
 });
 
 const db = knex({
@@ -48,7 +44,7 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob
   if (!value || value === "null") return;
   switch (type) {
     case "uuid":
-      return;
+      return `.default("00000000-0000-0000-0000-000000000000")`;
     case "character varying": {
       if (value === "gen_random_uuid()") return;
       if (typeof value === "string" && value.includes("::")) {
@@ -94,17 +90,7 @@ const main = async () => {
       .orderBy("table_name")
   ).filter((el) => !el.tableName.includes("_migrations"));
 
-  console.log("Select a table to generate schema");
-  console.table(tables);
-  console.log("all: all tables");
-  const selectedTables = prompt("Type table numbers comma seperated: ");
-  const tableNumbers =
-    selectedTables !== "all" ? selectedTables.split(",").map((el) => Number(el)) : [];
-
   for (let i = 0; i < tables.length; i += 1) {
-    // skip if not desired table
-    if (selectedTables !== "all" && !tableNumbers.includes(i)) continue;
-
     const { tableName } = tables[i];
     const columns = await db(tableName).columnInfo();
     const columnNames = Object.keys(columns);
@@ -114,7 +100,8 @@ const main = async () => {
       const columnName = columnNames[colNum];
       const colInfo = columns[columnName];
       let ztype = getZodPrimitiveType(colInfo.type);
-      if (colInfo.defaultValue) {
+      // don't put optional on id
+      if (colInfo.defaultValue && columnName !== "id") {
         const { defaultValue } = colInfo;
         const zSchema = getZodDefaultValue(colInfo.type, defaultValue);
         if (zSchema) {
@@ -124,16 +111,17 @@ const main = async () => {
       if (colInfo.nullable) {
         ztype = ztype.concat(".nullable().optional()");
       }
-      schema = schema.concat(`${!schema ? "\n" : ""}  ${columnName}: ${ztype},\n`);
+      schema = schema.concat(
+        `${!schema ? "\n" : ""}  ${columnName}: ${ztype}${colNum === columnNames.length - 1 ? "" : ","}\n`
+      );
     }
 
     const dashcase = tableName.split("_").join("-");
     const pascalCase = tableName
       .split("_")
-      .reduce(
+      .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, "");
-        (prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`,
+
-        ""
+    // the insert and update are changed to zod input type to use default cases
-      );
     writeFileSync(
       path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`),
       `// Code generated by automation script, DO NOT EDIT.
@@ -148,19 +136,10 @@ import { TImmutableDBKeys } from "./models";
 export const ${pascalCase}Schema = z.object({${schema}});
 
 export type T${pascalCase} = z.infer<typeof ${pascalCase}Schema>;
-export type T${pascalCase}Insert = Omit<T${pascalCase}, TImmutableDBKeys>;
+export type T${pascalCase}Insert = Omit<z.input<typeof ${pascalCase}Schema>, TImmutableDBKeys>;
-export type T${pascalCase}Update = Partial<Omit<T${pascalCase}, TImmutableDBKeys>>;
+export type T${pascalCase}Update = Partial<Omit<z.input<typeof ${pascalCase}Schema>, TImmutableDBKeys>>;
 `
     );
-
-    // const file = readFileSync(path.join(__dirname, "../src/db/schemas/index.ts"), "utf8");
-    // if (!file.includes(`export * from "./${dashcase};"`)) {
-    //   appendFileSync(
-    //     path.join(__dirname, "../src/db/schemas/index.ts"),
-    //     `\nexport * from "./${dashcase}";`,
-    //     "utf8"
-    //   );
-    // }
   }
 
   process.exit(0);

backend/src/@types/fastify.d.ts

@@ -3,9 +3,15 @@ import "fastify";
 import { TUsers } from "@app/db/schemas";
 import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
+import { TDynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-service";
+import { TDynamicSecretLeaseServiceFactory } from "@app/ee/services/dynamic-secret-lease/dynamic-secret-lease-service";
+import { TIdentityProjectAdditionalPrivilegeServiceFactory } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-service";
+import { TLdapConfigServiceFactory } from "@app/ee/services/ldap-config/ldap-config-service";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
+import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service";
 import { TSamlConfigServiceFactory } from "@app/ee/services/saml-config/saml-config-service";
+import { TScimServiceFactory } from "@app/ee/services/scim/scim-service";
 import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
 import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
 import { TSecretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
@@ -17,7 +23,7 @@ import { TApiKeyServiceFactory } from "@app/services/api-key/api-key-service";
 import { TAuthLoginFactory } from "@app/services/auth/auth-login-service";
 import { TAuthPasswordFactory } from "@app/services/auth/auth-password-service";
 import { TAuthSignupFactory } from "@app/services/auth/auth-signup-service";
-import { ActorType } from "@app/services/auth/auth-type";
+import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type";
 import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-service";
 import { TIdentityServiceFactory } from "@app/services/identity/identity-service";
 import { TIdentityAccessTokenServiceFactory } from "@app/services/identity-access-token/identity-access-token-service";
@@ -51,13 +57,16 @@ declare module "fastify" {
     // used for mfa session authentication
     mfa: {
       userId: string;
+      orgId?: string;
       user: TUsers;
     };
     // identity injection. depending on which kinda of token the information is filled in auth
     auth: TAuthMode;
     permission: {
+      authMethod: ActorAuthMethod;
       type: ActorType;
       id: string;
+      orgId: string;
     };
     // passport data
     passportUser: {
@@ -66,6 +75,7 @@ declare module "fastify" {
     };
     auditLogInfo: Pick<TCreateAuditLogDTO, "userAgent" | "userAgentType" | "ipAddress" | "actor">;
     ssoConfig: Awaited<ReturnType<TSamlConfigServiceFactory["getSaml"]>>;
+    ldapConfig: Awaited<ReturnType<TLdapConfigServiceFactory["getLdapCfg"]>>;
   }
 
   interface FastifyInstance {
@@ -103,12 +113,18 @@ declare module "fastify" {
       secretRotation: TSecretRotationServiceFactory;
       snapshot: TSecretSnapshotServiceFactory;
       saml: TSamlConfigServiceFactory;
+      scim: TScimServiceFactory;
+      ldap: TLdapConfigServiceFactory;
       auditLog: TAuditLogServiceFactory;
       secretScanning: TSecretScanningServiceFactory;
       license: TLicenseServiceFactory;
       trustedIp: TTrustedIpServiceFactory;
       secretBlindIndex: TSecretBlindIndexServiceFactory;
       telemetry: TTelemetryServiceFactory;
+      dynamicSecret: TDynamicSecretServiceFactory;
+      dynamicSecretLease: TDynamicSecretLeaseServiceFactory;
+      projectUserAdditionalPrivilege: TProjectUserAdditionalPrivilegeServiceFactory;
+      identityProjectAdditionalPrivilege: TIdentityProjectAdditionalPrivilegeServiceFactory;
     };
     // this is exclusive use for middlewares in which we need to inject data
     // everywhere else access using service layer

backend/src/@types/knex.d.ts

@@ -17,6 +17,12 @@ import {
   TBackupPrivateKey,
   TBackupPrivateKeyInsert,
   TBackupPrivateKeyUpdate,
+  TDynamicSecretLeases,
+  TDynamicSecretLeasesInsert,
+  TDynamicSecretLeasesUpdate,
+  TDynamicSecrets,
+  TDynamicSecretsInsert,
+  TDynamicSecretsUpdate,
   TGitAppInstallSessions,
   TGitAppInstallSessionsInsert,
   TGitAppInstallSessionsUpdate,
@@ -32,6 +38,12 @@ import {
   TIdentityOrgMemberships,
   TIdentityOrgMembershipsInsert,
   TIdentityOrgMembershipsUpdate,
+  TIdentityProjectAdditionalPrivilege,
+  TIdentityProjectAdditionalPrivilegeInsert,
+  TIdentityProjectAdditionalPrivilegeUpdate,
+  TIdentityProjectMembershipRole,
+  TIdentityProjectMembershipRoleInsert,
+  TIdentityProjectMembershipRoleUpdate,
   TIdentityProjectMemberships,
   TIdentityProjectMembershipsInsert,
   TIdentityProjectMembershipsUpdate,
@@ -50,6 +62,9 @@ import {
   TIntegrations,
   TIntegrationsInsert,
   TIntegrationsUpdate,
+  TLdapConfigs,
+  TLdapConfigsInsert,
+  TLdapConfigsUpdate,
   TOrganizations,
   TOrganizationsInsert,
   TOrganizationsUpdate,
@@ -80,9 +95,18 @@ import {
   TProjects,
   TProjectsInsert,
   TProjectsUpdate,
+  TProjectUserAdditionalPrivilege,
+  TProjectUserAdditionalPrivilegeInsert,
+  TProjectUserAdditionalPrivilegeUpdate,
+  TProjectUserMembershipRoles,
+  TProjectUserMembershipRolesInsert,
+  TProjectUserMembershipRolesUpdate,
   TSamlConfigs,
   TSamlConfigsInsert,
   TSamlConfigsUpdate,
+  TScimTokens,
+  TScimTokensInsert,
+  TScimTokensUpdate,
   TSecretApprovalPolicies,
   TSecretApprovalPoliciesApprovers,
   TSecretApprovalPoliciesApproversInsert,
@@ -158,6 +182,9 @@ import {
   TUserActions,
   TUserActionsInsert,
   TUserActionsUpdate,
+  TUserAliases,
+  TUserAliasesInsert,
+  TUserAliasesUpdate,
   TUserEncryptionKeys,
   TUserEncryptionKeysInsert,
   TUserEncryptionKeysUpdate,
@@ -172,6 +199,7 @@ import {
 declare module "knex/types/tables" {
   interface Tables {
     [TableName.Users]: Knex.CompositeTableType<TUsers, TUsersInsert, TUsersUpdate>;
+    [TableName.UserAliases]: Knex.CompositeTableType<TUserAliases, TUserAliasesInsert, TUserAliasesUpdate>;
     [TableName.UserEncryptionKey]: Knex.CompositeTableType<
       TUserEncryptionKeys,
       TUserEncryptionKeysInsert,
@@ -211,7 +239,17 @@ declare module "knex/types/tables" {
       TProjectEnvironmentsUpdate
     >;
     [TableName.ProjectBot]: Knex.CompositeTableType<TProjectBots, TProjectBotsInsert, TProjectBotsUpdate>;
+    [TableName.ProjectUserMembershipRole]: Knex.CompositeTableType<
+      TProjectUserMembershipRoles,
+      TProjectUserMembershipRolesInsert,
+      TProjectUserMembershipRolesUpdate
+    >;
     [TableName.ProjectRoles]: Knex.CompositeTableType<TProjectRoles, TProjectRolesInsert, TProjectRolesUpdate>;
+    [TableName.ProjectUserAdditionalPrivilege]: Knex.CompositeTableType<
+      TProjectUserAdditionalPrivilege,
+      TProjectUserAdditionalPrivilegeInsert,
+      TProjectUserAdditionalPrivilegeUpdate
+    >;
     [TableName.ProjectKeys]: Knex.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
     [TableName.Secret]: Knex.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
     [TableName.SecretBlindIndex]: Knex.CompositeTableType<
@@ -262,6 +300,17 @@ declare module "knex/types/tables" {
       TIdentityProjectMembershipsInsert,
       TIdentityProjectMembershipsUpdate
     >;
+    [TableName.IdentityProjectMembershipRole]: Knex.CompositeTableType<
+      TIdentityProjectMembershipRole,
+      TIdentityProjectMembershipRoleInsert,
+      TIdentityProjectMembershipRoleUpdate
+    >;
+    [TableName.IdentityProjectAdditionalPrivilege]: Knex.CompositeTableType<
+      TIdentityProjectAdditionalPrivilege,
+      TIdentityProjectAdditionalPrivilegeInsert,
+      TIdentityProjectAdditionalPrivilegeUpdate
+    >;
+    [TableName.ScimToken]: Knex.CompositeTableType<TScimTokens, TScimTokensInsert, TScimTokensUpdate>;
     [TableName.SecretApprovalPolicy]: Knex.CompositeTableType<
       TSecretApprovalPolicies,
       TSecretApprovalPoliciesInsert,
@@ -313,7 +362,14 @@ declare module "knex/types/tables" {
       TSecretSnapshotFoldersInsert,
       TSecretSnapshotFoldersUpdate
     >;
+    [TableName.DynamicSecret]: Knex.CompositeTableType<TDynamicSecrets, TDynamicSecretsInsert, TDynamicSecretsUpdate>;
+    [TableName.DynamicSecretLease]: Knex.CompositeTableType<
+      TDynamicSecretLeases,
+      TDynamicSecretLeasesInsert,
+      TDynamicSecretLeasesUpdate
+    >;
     [TableName.SamlConfig]: Knex.CompositeTableType<TSamlConfigs, TSamlConfigsInsert, TSamlConfigsUpdate>;
+    [TableName.LdapConfig]: Knex.CompositeTableType<TLdapConfigs, TLdapConfigsInsert, TLdapConfigsUpdate>;
     [TableName.OrgBot]: Knex.CompositeTableType<TOrgBots, TOrgBotsInsert, TOrgBotsUpdate>;
     [TableName.AuditLog]: Knex.CompositeTableType<TAuditLogs, TAuditLogsInsert, TAuditLogsUpdate>;
     [TableName.GitAppInstallSession]: Knex.CompositeTableType<

backend/src/cache/redis.ts

@@ -1,6 +0,0 @@
-import Redis from "ioredis";
-
-export const initRedisConnection = (redisUrl: string) => {
-  const redis = new Redis(redisUrl);
-  return redis;
-};

backend/src/db/instance.ts

@@ -6,6 +6,13 @@ export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnection
     client: "pg",
     connection: {
       connectionString: dbConnectionUri,
+      host: process.env.DB_HOST,
+      // @ts-expect-error I have no clue why only for the port there is a type error
+      // eslint-disable-next-line
+      port: process.env.DB_PORT,
+      user: process.env.DB_USER,
+      database: process.env.DB_NAME,
+      password: process.env.DB_PASSWORD,
       ssl: dbRootCert
         ? {
             rejectUnauthorized: true,

backend/src/db/knexfile.ts

@@ -5,15 +5,31 @@ import dotenv from "dotenv";
 import type { Knex } from "knex";
 import path from "path";
 
-// Update with your config settings.
+// Update with your config settings. .
 dotenv.config({
-  path: path.join(__dirname, "../../.env"),
+  path: path.join(__dirname, "../../../.env.migration")
-  debug: true
 });
+dotenv.config({
+  path: path.join(__dirname, "../../../.env")
+});
+
 export default {
   development: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      host: process.env.DB_HOST,
+      port: process.env.DB_PORT,
+      user: process.env.DB_USER,
+      database: process.env.DB_NAME,
+      password: process.env.DB_PASSWORD,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10
@@ -27,7 +43,20 @@ export default {
   },
   production: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      host: process.env.DB_HOST,
+      port: process.env.DB_PORT,
+      user: process.env.DB_USER,
+      database: process.env.DB_NAME,
+      password: process.env.DB_PASSWORD,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10

backend/src/db/migrations/20240204171758_org-based-auth.ts

@@ -0,0 +1,25 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.boolean("authEnforced").defaultTo(false);
+    t.index("slug");
+  });
+
+  await knex.schema.alterTable(TableName.SamlConfig, (t) => {
+    t.datetime("lastUsed");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.dropColumn("authEnforced");
+    t.dropIndex("slug");
+  });
+
+  await knex.schema.alterTable(TableName.SamlConfig, (t) => {
+    t.dropColumn("lastUsed");
+  });
+}

backend/src/db/migrations/20240208234120_scim-token.ts

@@ -0,0 +1,31 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.ScimToken))) {
+    await knex.schema.createTable(TableName.ScimToken, (t) => {
+      t.string("id", 36).primary().defaultTo(knex.fn.uuid());
+      t.bigInteger("ttlDays").defaultTo(365).notNullable();
+      t.string("description").notNullable();
+      t.uuid("orgId").notNullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.boolean("scimEnabled").defaultTo(false);
+  });
+
+  await createOnUpdateTrigger(knex, TableName.ScimToken);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.ScimToken);
+  await dropOnUpdateTrigger(knex, TableName.ScimToken);
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.dropColumn("scimEnabled");
+  });
+}

backend/src/db/migrations/20240216154123_ghost_users.ts

@@ -0,0 +1,39 @@
+import { Knex } from "knex";
+
+import { ProjectVersion, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (!hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.boolean("isGhost").defaultTo(false).notNullable();
+    });
+  }
+
+  if (!hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.integer("version").defaultTo(ProjectVersion.V1).notNullable();
+      t.string("upgradeStatus").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.dropColumn("isGhost");
+    });
+  }
+
+  if (hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.dropColumn("version");
+      t.dropColumn("upgradeStatus");
+    });
+  }
+}

backend/src/db/migrations/20240222201806_admin-signup-control.ts

@@ -0,0 +1,20 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const isTablePresent = await knex.schema.hasTable(TableName.SuperAdmin);
+  if (isTablePresent) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.string("allowedSignUpDomain");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.SuperAdmin, "allowedSignUpDomain")) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.dropColumn("allowedSignUpDomain");
+    });
+  }
+}

backend/src/db/migrations/20240226094411_instance-id.ts

@@ -0,0 +1,25 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const ADMIN_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+    t.uuid("instanceId").notNullable().defaultTo(knex.fn.uuid());
+  });
+
+  const superUserConfigExists = await knex(TableName.SuperAdmin).where("id", ADMIN_CONFIG_UUID).first();
+  if (!superUserConfigExists) {
+    // eslint-disable-next-line
+    await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+    t.dropColumn("instanceId");
+  });
+}

backend/src/db/migrations/20240307232900_integration-last-used.ts

@@ -0,0 +1,15 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    t.datetime("lastUsed");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    t.dropColumn("lastUsed");
+  });
+}

backend/src/db/migrations/20240311210135_ldap-config.ts

@@ -0,0 +1,68 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.LdapConfig))) {
+    await knex.schema.createTable(TableName.LdapConfig, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("orgId").notNullable().unique();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.boolean("isActive").notNullable();
+      t.string("url").notNullable();
+      t.string("encryptedBindDN").notNullable();
+      t.string("bindDNIV").notNullable();
+      t.string("bindDNTag").notNullable();
+      t.string("encryptedBindPass").notNullable();
+      t.string("bindPassIV").notNullable();
+      t.string("bindPassTag").notNullable();
+      t.string("searchBase").notNullable();
+      t.text("encryptedCACert").notNullable();
+      t.string("caCertIV").notNullable();
+      t.string("caCertTag").notNullable();
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.LdapConfig);
+
+  if (!(await knex.schema.hasTable(TableName.UserAliases))) {
+    await knex.schema.createTable(TableName.UserAliases, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("userId").notNullable();
+      t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE");
+      t.string("username").notNullable();
+      t.string("aliasType").notNullable();
+      t.string("externalId").notNullable();
+      t.specificType("emails", "text[]");
+      t.uuid("orgId").nullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.UserAliases);
+
+  await knex.schema.alterTable(TableName.Users, (t) => {
+    t.string("username").unique();
+    t.string("email").nullable().alter();
+    t.dropUnique(["email"]);
+  });
+
+  await knex(TableName.Users).update("username", knex.ref("email"));
+
+  await knex.schema.alterTable(TableName.Users, (t) => {
+    t.string("username").notNullable().alter();
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.LdapConfig);
+  await knex.schema.dropTableIfExists(TableName.UserAliases);
+  await knex.schema.alterTable(TableName.Users, (t) => {
+    t.dropColumn("username");
+    // t.string("email").notNullable().alter();
+  });
+  await dropOnUpdateTrigger(knex, TableName.LdapConfig);
+}

backend/src/db/migrations/20240312162549_temp-roles.ts

@@ -0,0 +1,50 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesTableExist = await knex.schema.hasTable(TableName.ProjectUserMembershipRole);
+  if (!doesTableExist) {
+    await knex.schema.createTable(TableName.ProjectUserMembershipRole, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("role").notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+      // until role is changed/removed the role should not deleted
+      t.uuid("customRoleId");
+      t.foreign("customRoleId").references("id").inTable(TableName.ProjectRoles);
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.ProjectUserMembershipRole);
+
+  const projectMemberships = await knex(TableName.ProjectMembership).select(
+    "id",
+    "role",
+    "createdAt",
+    "updatedAt",
+    knex.ref("roleId").withSchema(TableName.ProjectMembership).as("customRoleId")
+  );
+  if (projectMemberships.length)
+    await knex.batchInsert(
+      TableName.ProjectUserMembershipRole,
+      projectMemberships.map((data) => ({ ...data, projectMembershipId: data.id }))
+    );
+  // will be dropped later
+  // await knex.schema.alterTable(TableName.ProjectMembership, (t) => {
+  //   t.dropColumn("roleId");
+  //   t.dropColumn("role");
+  // });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.ProjectUserMembershipRole);
+  await dropOnUpdateTrigger(knex, TableName.ProjectUserMembershipRole);
+}

backend/src/db/migrations/20240312162556_temp-role-identity.ts

@@ -0,0 +1,52 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesTableExist = await knex.schema.hasTable(TableName.IdentityProjectMembershipRole);
+  if (!doesTableExist) {
+    await knex.schema.createTable(TableName.IdentityProjectMembershipRole, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("role").notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId")
+        .references("id")
+        .inTable(TableName.IdentityProjectMembership)
+        .onDelete("CASCADE");
+      // until role is changed/removed the role should not deleted
+      t.uuid("customRoleId");
+      t.foreign("customRoleId").references("id").inTable(TableName.ProjectRoles);
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityProjectMembershipRole);
+
+  const identityMemberships = await knex(TableName.IdentityProjectMembership).select(
+    "id",
+    "role",
+    "createdAt",
+    "updatedAt",
+    knex.ref("roleId").withSchema(TableName.IdentityProjectMembership).as("customRoleId")
+  );
+  if (identityMemberships.length)
+    await knex.batchInsert(
+      TableName.IdentityProjectMembershipRole,
+      identityMemberships.map((data) => ({ ...data, projectMembershipId: data.id }))
+    );
+  // await knex.schema.alterTable(TableName.IdentityProjectMembership, (t) => {
+  //   t.dropColumn("roleId");
+  //   t.dropColumn("role");
+  // });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityProjectMembershipRole);
+  await dropOnUpdateTrigger(knex, TableName.IdentityProjectMembershipRole);
+}

backend/src/db/migrations/20240318164718_dynamic-secret.ts

@@ -0,0 +1,58 @@
+import { Knex } from "knex";
+
+import { SecretEncryptionAlgo, SecretKeyEncoding, TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesTableExist = await knex.schema.hasTable(TableName.DynamicSecret);
+  if (!doesTableExist) {
+    await knex.schema.createTable(TableName.DynamicSecret, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("name").notNullable();
+      t.integer("version").notNullable();
+      t.string("type").notNullable();
+      t.string("defaultTTL").notNullable();
+      t.string("maxTTL");
+      t.string("inputIV").notNullable();
+      t.text("inputCiphertext").notNullable();
+      t.string("inputTag").notNullable();
+      t.string("algorithm").notNullable().defaultTo(SecretEncryptionAlgo.AES_256_GCM);
+      t.string("keyEncoding").notNullable().defaultTo(SecretKeyEncoding.UTF8);
+      t.uuid("folderId").notNullable();
+      // for background process communication
+      t.string("status");
+      t.string("statusDetails");
+      t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE");
+      t.unique(["name", "folderId"]);
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.DynamicSecret);
+
+  const doesTableDynamicSecretLease = await knex.schema.hasTable(TableName.DynamicSecretLease);
+  if (!doesTableDynamicSecretLease) {
+    await knex.schema.createTable(TableName.DynamicSecretLease, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.integer("version").notNullable();
+      t.string("externalEntityId").notNullable();
+      t.datetime("expireAt").notNullable();
+      // for background process communication
+      t.string("status");
+      t.string("statusDetails");
+      t.uuid("dynamicSecretId").notNullable();
+      t.foreign("dynamicSecretId").references("id").inTable(TableName.DynamicSecret).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.DynamicSecretLease);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.DynamicSecretLease);
+  await knex.schema.dropTableIfExists(TableName.DynamicSecretLease);
+
+  await dropOnUpdateTrigger(knex, TableName.DynamicSecret);
+  await knex.schema.dropTableIfExists(TableName.DynamicSecret);
+}

backend/src/db/migrations/20240326172010_project-user-additional-privilege.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.ProjectUserAdditionalPrivilege))) {
+    await knex.schema.createTable(TableName.ProjectUserAdditionalPrivilege, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("slug", 60).notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.jsonb("permissions").notNullable();
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.ProjectUserAdditionalPrivilege);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.ProjectUserAdditionalPrivilege);
+  await knex.schema.dropTableIfExists(TableName.ProjectUserAdditionalPrivilege);
+}

backend/src/db/migrations/20240326172011_machine-identity-additional-privilege.ts

@@ -0,0 +1,32 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityProjectAdditionalPrivilege))) {
+    await knex.schema.createTable(TableName.IdentityProjectAdditionalPrivilege, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("slug", 60).notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId")
+        .references("id")
+        .inTable(TableName.IdentityProjectMembership)
+        .onDelete("CASCADE");
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.jsonb("permissions").notNullable();
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityProjectAdditionalPrivilege);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.IdentityProjectAdditionalPrivilege);
+  await knex.schema.dropTableIfExists(TableName.IdentityProjectAdditionalPrivilege);
+}

backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts

@@ -0,0 +1,111 @@
+import { Knex } from "knex";
+import { z } from "zod";
+
+import { TableName, TOrgMemberships } from "../schemas";
+
+const validateOrgMembership = (membershipToValidate: TOrgMemberships, firstMembership: TOrgMemberships) => {
+  const firstOrgId = firstMembership.orgId;
+  const firstUserId = firstMembership.userId;
+
+  if (membershipToValidate.id === firstMembership.id) {
+    return;
+  }
+
+  if (membershipToValidate.inviteEmail !== firstMembership.inviteEmail) {
+    throw new Error(`Invite emails are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.orgId !== firstMembership.orgId) {
+    throw new Error(`OrgIds are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.role !== firstMembership.role) {
+    throw new Error(`Roles are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.roleId !== firstMembership.roleId) {
+    throw new Error(`RoleIds are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.status !== firstMembership.status) {
+    throw new Error(`Statuses are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.userId !== firstMembership.userId) {
+    throw new Error(`UserIds are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+};
+
+export async function up(knex: Knex): Promise<void> {
+  const RowSchema = z.object({
+    userId: z.string(),
+    orgId: z.string(),
+    cnt: z.string()
+  });
+
+  // Transactional find and delete duplicate rows
+  await knex.transaction(async (tx) => {
+    const duplicateRows = await tx(TableName.OrgMembership)
+      .select("userId", "orgId") // Select the userId and orgId so we can group by them
+      .count("* as cnt") // Count the number of rows for each userId and orgId, so we can make sure there are more than 1 row (a duplicate)
+      .groupBy("userId", "orgId")
+      .havingRaw("count(*) > ?", [1]); // Using havingRaw for direct SQL expressions
+
+    // Parse the rows to ensure they are in the correct format, and for type safety
+    const parsedRows = RowSchema.array().parse(duplicateRows);
+
+    // For each of the duplicate rows, loop through and find the actual memberships to delete
+    for (const row of parsedRows) {
+      const count = Number(row.cnt);
+
+      // An extra check to ensure that the count is actually a number, and the number is greater than 2
+      if (typeof count !== "number" || count < 2) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      // Find all the organization memberships that have the same userId and orgId
+      // eslint-disable-next-line no-await-in-loop
+      const rowsToDelete = await tx(TableName.OrgMembership).where({
+        userId: row.userId,
+        orgId: row.orgId
+      });
+
+      // Ensure that all the rows have exactly the same value, except id, createdAt, updatedAt
+      for (const rowToDelete of rowsToDelete) {
+        validateOrgMembership(rowToDelete, rowsToDelete[0]);
+      }
+
+      // Find the row with the latest createdAt, which we will keep
+
+      let lowestCreatedAt: number | null = null;
+      let latestCreatedRow: TOrgMemberships | null = null;
+
+      for (const rowToDelete of rowsToDelete) {
+        if (lowestCreatedAt === null || rowToDelete.createdAt.getTime() < lowestCreatedAt) {
+          lowestCreatedAt = rowToDelete.createdAt.getTime();
+          latestCreatedRow = rowToDelete;
+        }
+      }
+      if (!latestCreatedRow) {
+        throw new Error("Failed to find last created membership");
+      }
+
+      // Filter out the latest row from the rows to delete
+      const membershipIdsToDelete = rowsToDelete.map((r) => r.id).filter((id) => id !== latestCreatedRow!.id);
+
+      // eslint-disable-next-line no-await-in-loop
+      const numberOfRowsDeleted = await tx(TableName.OrgMembership).whereIn("id", membershipIdsToDelete).delete();
+
+      // eslint-disable-next-line no-console
+      console.log(
+        `Deleted ${numberOfRowsDeleted} duplicate organization memberships for ${row.userId} and ${row.orgId}`
+      );
+    }
+  });
+
+  await knex.schema.alterTable(TableName.OrgMembership, (table) => {
+    table.unique(["userId", "orgId"]);
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.OrgMembership, (table) => {
+    table.dropUnique(["userId", "orgId"]);
+  });
+}

backend/src/db/schemas/api-keys.ts

@@ -19,5 +19,5 @@ export const ApiKeysSchema = z.object({
 });
 
 export type TApiKeys = z.infer<typeof ApiKeysSchema>;
-export type TApiKeysInsert = Omit<TApiKeys, TImmutableDBKeys>;
+export type TApiKeysInsert = Omit<z.input<typeof ApiKeysSchema>, TImmutableDBKeys>;
-export type TApiKeysUpdate = Partial<Omit<TApiKeys, TImmutableDBKeys>>;
+export type TApiKeysUpdate = Partial<Omit<z.input<typeof ApiKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/audit-logs.ts

@@ -24,5 +24,5 @@ export const AuditLogsSchema = z.object({
 });
 
 export type TAuditLogs = z.infer<typeof AuditLogsSchema>;
-export type TAuditLogsInsert = Omit<TAuditLogs, TImmutableDBKeys>;
+export type TAuditLogsInsert = Omit<z.input<typeof AuditLogsSchema>, TImmutableDBKeys>;
-export type TAuditLogsUpdate = Partial<Omit<TAuditLogs, TImmutableDBKeys>>;
+export type TAuditLogsUpdate = Partial<Omit<z.input<typeof AuditLogsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/auth-token-sessions.ts

@@ -20,5 +20,5 @@ export const AuthTokenSessionsSchema = z.object({
 });
 
 export type TAuthTokenSessions = z.infer<typeof AuthTokenSessionsSchema>;
-export type TAuthTokenSessionsInsert = Omit<TAuthTokenSessions, TImmutableDBKeys>;
+export type TAuthTokenSessionsInsert = Omit<z.input<typeof AuthTokenSessionsSchema>, TImmutableDBKeys>;
-export type TAuthTokenSessionsUpdate = Partial<Omit<TAuthTokenSessions, TImmutableDBKeys>>;
+export type TAuthTokenSessionsUpdate = Partial<Omit<z.input<typeof AuthTokenSessionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/auth-tokens.ts

@@ -21,5 +21,5 @@ export const AuthTokensSchema = z.object({
 });
 
 export type TAuthTokens = z.infer<typeof AuthTokensSchema>;
-export type TAuthTokensInsert = Omit<TAuthTokens, TImmutableDBKeys>;
+export type TAuthTokensInsert = Omit<z.input<typeof AuthTokensSchema>, TImmutableDBKeys>;
-export type TAuthTokensUpdate = Partial<Omit<TAuthTokens, TImmutableDBKeys>>;
+export type TAuthTokensUpdate = Partial<Omit<z.input<typeof AuthTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/backup-private-key.ts

@@ -22,5 +22,5 @@ export const BackupPrivateKeySchema = z.object({
 });
 
 export type TBackupPrivateKey = z.infer<typeof BackupPrivateKeySchema>;
-export type TBackupPrivateKeyInsert = Omit<TBackupPrivateKey, TImmutableDBKeys>;
+export type TBackupPrivateKeyInsert = Omit<z.input<typeof BackupPrivateKeySchema>, TImmutableDBKeys>;
-export type TBackupPrivateKeyUpdate = Partial<Omit<TBackupPrivateKey, TImmutableDBKeys>>;
+export type TBackupPrivateKeyUpdate = Partial<Omit<z.input<typeof BackupPrivateKeySchema>, TImmutableDBKeys>>;

backend/src/db/schemas/dynamic-secret-leases.ts

@@ -0,0 +1,24 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const DynamicSecretLeasesSchema = z.object({
+  id: z.string().uuid(),
+  version: z.number(),
+  externalEntityId: z.string(),
+  expireAt: z.date(),
+  status: z.string().nullable().optional(),
+  statusDetails: z.string().nullable().optional(),
+  dynamicSecretId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TDynamicSecretLeases = z.infer<typeof DynamicSecretLeasesSchema>;
+export type TDynamicSecretLeasesInsert = Omit<z.input<typeof DynamicSecretLeasesSchema>, TImmutableDBKeys>;
+export type TDynamicSecretLeasesUpdate = Partial<Omit<z.input<typeof DynamicSecretLeasesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/dynamic-secrets.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const DynamicSecretsSchema = z.object({
+  id: z.string().uuid(),
+  name: z.string(),
+  version: z.number(),
+  type: z.string(),
+  defaultTTL: z.string(),
+  maxTTL: z.string().nullable().optional(),
+  inputIV: z.string(),
+  inputCiphertext: z.string(),
+  inputTag: z.string(),
+  algorithm: z.string().default("aes-256-gcm"),
+  keyEncoding: z.string().default("utf8"),
+  folderId: z.string().uuid(),
+  status: z.string().nullable().optional(),
+  statusDetails: z.string().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TDynamicSecrets = z.infer<typeof DynamicSecretsSchema>;
+export type TDynamicSecretsInsert = Omit<z.input<typeof DynamicSecretsSchema>, TImmutableDBKeys>;
+export type TDynamicSecretsUpdate = Partial<Omit<z.input<typeof DynamicSecretsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/git-app-install-sessions.ts

@@ -17,5 +17,5 @@ export const GitAppInstallSessionsSchema = z.object({
 });
 
 export type TGitAppInstallSessions = z.infer<typeof GitAppInstallSessionsSchema>;
-export type TGitAppInstallSessionsInsert = Omit<TGitAppInstallSessions, TImmutableDBKeys>;
+export type TGitAppInstallSessionsInsert = Omit<z.input<typeof GitAppInstallSessionsSchema>, TImmutableDBKeys>;
-export type TGitAppInstallSessionsUpdate = Partial<Omit<TGitAppInstallSessions, TImmutableDBKeys>>;
+export type TGitAppInstallSessionsUpdate = Partial<Omit<z.input<typeof GitAppInstallSessionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/git-app-org.ts

@@ -17,5 +17,5 @@ export const GitAppOrgSchema = z.object({
 });
 
 export type TGitAppOrg = z.infer<typeof GitAppOrgSchema>;
-export type TGitAppOrgInsert = Omit<TGitAppOrg, TImmutableDBKeys>;
+export type TGitAppOrgInsert = Omit<z.input<typeof GitAppOrgSchema>, TImmutableDBKeys>;
-export type TGitAppOrgUpdate = Partial<Omit<TGitAppOrg, TImmutableDBKeys>>;
+export type TGitAppOrgUpdate = Partial<Omit<z.input<typeof GitAppOrgSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/identities.ts

@@ -16,5 +16,5 @@ export const IdentitiesSchema = z.object({
 });
 
 export type TIdentities = z.infer<typeof IdentitiesSchema>;
-export type TIdentitiesInsert = Omit<TIdentities, TImmutableDBKeys>;
+export type TIdentitiesInsert = Omit<z.input<typeof IdentitiesSchema>, TImmutableDBKeys>;
-export type TIdentitiesUpdate = Partial<Omit<TIdentities, TImmutableDBKeys>>;
+export type TIdentitiesUpdate = Partial<Omit<z.input<typeof IdentitiesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/identity-access-tokens.ts

@@ -23,5 +23,5 @@ export const IdentityAccessTokensSchema = z.object({
 });
 
 export type TIdentityAccessTokens = z.infer<typeof IdentityAccessTokensSchema>;
-export type TIdentityAccessTokensInsert = Omit<TIdentityAccessTokens, TImmutableDBKeys>;
+export type TIdentityAccessTokensInsert = Omit<z.input<typeof IdentityAccessTokensSchema>, TImmutableDBKeys>;
-export type TIdentityAccessTokensUpdate = Partial<Omit<TIdentityAccessTokens, TImmutableDBKeys>>;
+export type TIdentityAccessTokensUpdate = Partial<Omit<z.input<typeof IdentityAccessTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/identity-org-memberships.ts

@@ -18,5 +18,7 @@ export const IdentityOrgMembershipsSchema = z.object({
 });
 
 export type TIdentityOrgMemberships = z.infer<typeof IdentityOrgMembershipsSchema>;
-export type TIdentityOrgMembershipsInsert = Omit<TIdentityOrgMemberships, TImmutableDBKeys>;
+export type TIdentityOrgMembershipsInsert = Omit<z.input<typeof IdentityOrgMembershipsSchema>, TImmutableDBKeys>;
-export type TIdentityOrgMembershipsUpdate = Partial<Omit<TIdentityOrgMemberships, TImmutableDBKeys>>;
+export type TIdentityOrgMembershipsUpdate = Partial<
+  Omit<z.input<typeof IdentityOrgMembershipsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-project-additional-privilege.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const IdentityProjectAdditionalPrivilegeSchema = z.object({
+  id: z.string().uuid(),
+  slug: z.string(),
+  projectMembershipId: z.string().uuid(),
+  isTemporary: z.boolean().default(false),
+  temporaryMode: z.string().nullable().optional(),
+  temporaryRange: z.string().nullable().optional(),
+  temporaryAccessStartTime: z.date().nullable().optional(),
+  temporaryAccessEndTime: z.date().nullable().optional(),
+  permissions: z.unknown(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TIdentityProjectAdditionalPrivilege = z.infer<typeof IdentityProjectAdditionalPrivilegeSchema>;
+export type TIdentityProjectAdditionalPrivilegeInsert = Omit<
+  z.input<typeof IdentityProjectAdditionalPrivilegeSchema>,
+  TImmutableDBKeys
+>;
+export type TIdentityProjectAdditionalPrivilegeUpdate = Partial<
+  Omit<z.input<typeof IdentityProjectAdditionalPrivilegeSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-project-membership-role.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const IdentityProjectMembershipRoleSchema = z.object({
+  id: z.string().uuid(),
+  role: z.string(),
+  projectMembershipId: z.string().uuid(),
+  customRoleId: z.string().uuid().nullable().optional(),
+  isTemporary: z.boolean().default(false),
+  temporaryMode: z.string().nullable().optional(),
+  temporaryRange: z.string().nullable().optional(),
+  temporaryAccessStartTime: z.date().nullable().optional(),
+  temporaryAccessEndTime: z.date().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TIdentityProjectMembershipRole = z.infer<typeof IdentityProjectMembershipRoleSchema>;
+export type TIdentityProjectMembershipRoleInsert = Omit<
+  z.input<typeof IdentityProjectMembershipRoleSchema>,
+  TImmutableDBKeys
+>;
+export type TIdentityProjectMembershipRoleUpdate = Partial<
+  Omit<z.input<typeof IdentityProjectMembershipRoleSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-project-memberships.ts

@@ -18,5 +18,10 @@ export const IdentityProjectMembershipsSchema = z.object({
 });
 
 export type TIdentityProjectMemberships = z.infer<typeof IdentityProjectMembershipsSchema>;
-export type TIdentityProjectMembershipsInsert = Omit<TIdentityProjectMemberships, TImmutableDBKeys>;
+export type TIdentityProjectMembershipsInsert = Omit<
-export type TIdentityProjectMembershipsUpdate = Partial<Omit<TIdentityProjectMemberships, TImmutableDBKeys>>;
+  z.input<typeof IdentityProjectMembershipsSchema>,
+  TImmutableDBKeys
+>;
+export type TIdentityProjectMembershipsUpdate = Partial<
+  Omit<z.input<typeof IdentityProjectMembershipsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-ua-client-secrets.ts

@@ -23,5 +23,7 @@ export const IdentityUaClientSecretsSchema = z.object({
 });
 
 export type TIdentityUaClientSecrets = z.infer<typeof IdentityUaClientSecretsSchema>;
-export type TIdentityUaClientSecretsInsert = Omit<TIdentityUaClientSecrets, TImmutableDBKeys>;
+export type TIdentityUaClientSecretsInsert = Omit<z.input<typeof IdentityUaClientSecretsSchema>, TImmutableDBKeys>;
-export type TIdentityUaClientSecretsUpdate = Partial<Omit<TIdentityUaClientSecrets, TImmutableDBKeys>>;
+export type TIdentityUaClientSecretsUpdate = Partial<
+  Omit<z.input<typeof IdentityUaClientSecretsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/identity-universal-auths.ts

@@ -21,5 +21,7 @@ export const IdentityUniversalAuthsSchema = z.object({
 });
 
 export type TIdentityUniversalAuths = z.infer<typeof IdentityUniversalAuthsSchema>;
-export type TIdentityUniversalAuthsInsert = Omit<TIdentityUniversalAuths, TImmutableDBKeys>;
+export type TIdentityUniversalAuthsInsert = Omit<z.input<typeof IdentityUniversalAuthsSchema>, TImmutableDBKeys>;
-export type TIdentityUniversalAuthsUpdate = Partial<Omit<TIdentityUniversalAuths, TImmutableDBKeys>>;
+export type TIdentityUniversalAuthsUpdate = Partial<
+  Omit<z.input<typeof IdentityUniversalAuthsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/incident-contacts.ts

@@ -16,5 +16,5 @@ export const IncidentContactsSchema = z.object({
 });
 
 export type TIncidentContacts = z.infer<typeof IncidentContactsSchema>;
-export type TIncidentContactsInsert = Omit<TIncidentContacts, TImmutableDBKeys>;
+export type TIncidentContactsInsert = Omit<z.input<typeof IncidentContactsSchema>, TImmutableDBKeys>;
-export type TIncidentContactsUpdate = Partial<Omit<TIncidentContacts, TImmutableDBKeys>>;
+export type TIncidentContactsUpdate = Partial<Omit<z.input<typeof IncidentContactsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/index.ts

@@ -3,17 +3,22 @@ export * from "./audit-logs";
 export * from "./auth-token-sessions";
 export * from "./auth-tokens";
 export * from "./backup-private-key";
+export * from "./dynamic-secret-leases";
+export * from "./dynamic-secrets";
 export * from "./git-app-install-sessions";
 export * from "./git-app-org";
 export * from "./identities";
 export * from "./identity-access-tokens";
 export * from "./identity-org-memberships";
+export * from "./identity-project-additional-privilege";
+export * from "./identity-project-membership-role";
 export * from "./identity-project-memberships";
 export * from "./identity-ua-client-secrets";
 export * from "./identity-universal-auths";
 export * from "./incident-contacts";
 export * from "./integration-auths";
 export * from "./integrations";
+export * from "./ldap-configs";
 export * from "./models";
 export * from "./org-bots";
 export * from "./org-memberships";
@@ -24,8 +29,11 @@ export * from "./project-environments";
 export * from "./project-keys";
 export * from "./project-memberships";
 export * from "./project-roles";
+export * from "./project-user-additional-privilege";
+export * from "./project-user-membership-roles";
 export * from "./projects";
 export * from "./saml-configs";
+export * from "./scim-tokens";
 export * from "./secret-approval-policies";
 export * from "./secret-approval-policies-approvers";
 export * from "./secret-approval-request-secret-tags";
@@ -51,6 +59,7 @@ export * from "./service-tokens";
 export * from "./super-admin";
 export * from "./trusted-ips";
 export * from "./user-actions";
+export * from "./user-aliases";
 export * from "./user-encryption-keys";
 export * from "./users";
 export * from "./webhooks";

backend/src/db/schemas/integration-auths.ts

@@ -33,5 +33,5 @@ export const IntegrationAuthsSchema = z.object({
 });
 
 export type TIntegrationAuths = z.infer<typeof IntegrationAuthsSchema>;
-export type TIntegrationAuthsInsert = Omit<TIntegrationAuths, TImmutableDBKeys>;
+export type TIntegrationAuthsInsert = Omit<z.input<typeof IntegrationAuthsSchema>, TImmutableDBKeys>;
-export type TIntegrationAuthsUpdate = Partial<Omit<TIntegrationAuths, TImmutableDBKeys>>;
+export type TIntegrationAuthsUpdate = Partial<Omit<z.input<typeof IntegrationAuthsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/integrations.ts

@@ -27,9 +27,10 @@ export const IntegrationsSchema = z.object({
   envId: z.string().uuid(),
   secretPath: z.string().default("/"),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  lastUsed: z.date().nullable().optional()
 });
 
 export type TIntegrations = z.infer<typeof IntegrationsSchema>;
-export type TIntegrationsInsert = Omit<TIntegrations, TImmutableDBKeys>;
+export type TIntegrationsInsert = Omit<z.input<typeof IntegrationsSchema>, TImmutableDBKeys>;
-export type TIntegrationsUpdate = Partial<Omit<TIntegrations, TImmutableDBKeys>>;
+export type TIntegrationsUpdate = Partial<Omit<z.input<typeof IntegrationsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/ldap-configs.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const LdapConfigsSchema = z.object({
+  id: z.string().uuid(),
+  orgId: z.string().uuid(),
+  isActive: z.boolean(),
+  url: z.string(),
+  encryptedBindDN: z.string(),
+  bindDNIV: z.string(),
+  bindDNTag: z.string(),
+  encryptedBindPass: z.string(),
+  bindPassIV: z.string(),
+  bindPassTag: z.string(),
+  searchBase: z.string(),
+  encryptedCACert: z.string(),
+  caCertIV: z.string(),
+  caCertTag: z.string(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TLdapConfigs = z.infer<typeof LdapConfigsSchema>;
+export type TLdapConfigsInsert = Omit<z.input<typeof LdapConfigsSchema>, TImmutableDBKeys>;
+export type TLdapConfigsUpdate = Partial<Omit<z.input<typeof LdapConfigsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/models.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 
 export enum TableName {
   Users = "users",
+  UserAliases = "user_aliases",
   UserEncryptionKey = "user_encryption_keys",
   AuthTokens = "auth_tokens",
   AuthTokenSession = "auth_token_sessions",
@@ -19,6 +20,8 @@ export enum TableName {
   Environment = "project_environments",
   ProjectMembership = "project_memberships",
   ProjectRoles = "project_roles",
+  ProjectUserAdditionalPrivilege = "project_user_additional_privilege",
+  ProjectUserMembershipRole = "project_user_membership_roles",
   ProjectKeys = "project_keys",
   Secret = "secrets",
   SecretBlindIndex = "secret_blind_indexes",
@@ -40,6 +43,9 @@ export enum TableName {
   IdentityUaClientSecret = "identity_ua_client_secrets",
   IdentityOrgMembership = "identity_org_memberships",
   IdentityProjectMembership = "identity_project_memberships",
+  IdentityProjectMembershipRole = "identity_project_membership_role",
+  IdentityProjectAdditionalPrivilege = "identity_project_additional_privilege",
+  ScimToken = "scim_tokens",
   SecretApprovalPolicy = "secret_approval_policies",
   SecretApprovalPolicyApprover = "secret_approval_policies_approvers",
   SecretApprovalRequest = "secret_approval_requests",
@@ -49,11 +55,14 @@ export enum TableName {
   SecretRotation = "secret_rotations",
   SecretRotationOutput = "secret_rotation_outputs",
   SamlConfig = "saml_configs",
+  LdapConfig = "ldap_configs",
   AuditLog = "audit_logs",
   GitAppInstallSession = "git_app_install_sessions",
   GitAppOrg = "git_app_org",
   SecretScanningGitRisk = "secret_scanning_git_risks",
   TrustedIps = "trusted_ips",
+  DynamicSecret = "dynamic_secrets",
+  DynamicSecretLease = "dynamic_secret_leases",
   // junction tables with tags
   JnSecretTag = "secret_tag_junction",
   SecretVersionTag = "secret_version_tag_junction"
@@ -111,6 +120,17 @@ export enum SecretType {
   Personal = "personal"
 }
 
+export enum ProjectVersion {
+  V1 = 1,
+  V2 = 2
+}
+
+export enum ProjectUpgradeStatus {
+  InProgress = "IN_PROGRESS",
+  // Completed -> Will be null if completed. So a completed status is not needed
+  Failed = "FAILED"
+}
+
 export enum IdentityAuthMethod {
   Univeral = "universal-auth"
 }

backend/src/db/schemas/org-bots.ts

@@ -27,5 +27,5 @@ export const OrgBotsSchema = z.object({
 });
 
 export type TOrgBots = z.infer<typeof OrgBotsSchema>;
-export type TOrgBotsInsert = Omit<TOrgBots, TImmutableDBKeys>;
+export type TOrgBotsInsert = Omit<z.input<typeof OrgBotsSchema>, TImmutableDBKeys>;
-export type TOrgBotsUpdate = Partial<Omit<TOrgBots, TImmutableDBKeys>>;
+export type TOrgBotsUpdate = Partial<Omit<z.input<typeof OrgBotsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/org-memberships.ts

@@ -20,5 +20,5 @@ export const OrgMembershipsSchema = z.object({
 });
 
 export type TOrgMemberships = z.infer<typeof OrgMembershipsSchema>;
-export type TOrgMembershipsInsert = Omit<TOrgMemberships, TImmutableDBKeys>;
+export type TOrgMembershipsInsert = Omit<z.input<typeof OrgMembershipsSchema>, TImmutableDBKeys>;
-export type TOrgMembershipsUpdate = Partial<Omit<TOrgMemberships, TImmutableDBKeys>>;
+export type TOrgMembershipsUpdate = Partial<Omit<z.input<typeof OrgMembershipsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/org-roles.ts

@@ -19,5 +19,5 @@ export const OrgRolesSchema = z.object({
 });
 
 export type TOrgRoles = z.infer<typeof OrgRolesSchema>;
-export type TOrgRolesInsert = Omit<TOrgRoles, TImmutableDBKeys>;
+export type TOrgRolesInsert = Omit<z.input<typeof OrgRolesSchema>, TImmutableDBKeys>;
-export type TOrgRolesUpdate = Partial<Omit<TOrgRoles, TImmutableDBKeys>>;
+export type TOrgRolesUpdate = Partial<Omit<z.input<typeof OrgRolesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/organizations.ts

@@ -13,9 +13,11 @@ export const OrganizationsSchema = z.object({
   customerId: z.string().nullable().optional(),
   slug: z.string(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  authEnforced: z.boolean().default(false).nullable().optional(),
+  scimEnabled: z.boolean().default(false).nullable().optional()
 });
 
 export type TOrganizations = z.infer<typeof OrganizationsSchema>;
-export type TOrganizationsInsert = Omit<TOrganizations, TImmutableDBKeys>;
+export type TOrganizationsInsert = Omit<z.input<typeof OrganizationsSchema>, TImmutableDBKeys>;
-export type TOrganizationsUpdate = Partial<Omit<TOrganizations, TImmutableDBKeys>>;
+export type TOrganizationsUpdate = Partial<Omit<z.input<typeof OrganizationsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-bots.ts

@@ -26,5 +26,5 @@ export const ProjectBotsSchema = z.object({
 });
 
 export type TProjectBots = z.infer<typeof ProjectBotsSchema>;
-export type TProjectBotsInsert = Omit<TProjectBots, TImmutableDBKeys>;
+export type TProjectBotsInsert = Omit<z.input<typeof ProjectBotsSchema>, TImmutableDBKeys>;
-export type TProjectBotsUpdate = Partial<Omit<TProjectBots, TImmutableDBKeys>>;
+export type TProjectBotsUpdate = Partial<Omit<z.input<typeof ProjectBotsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-environments.ts

@@ -18,5 +18,5 @@ export const ProjectEnvironmentsSchema = z.object({
 });
 
 export type TProjectEnvironments = z.infer<typeof ProjectEnvironmentsSchema>;
-export type TProjectEnvironmentsInsert = Omit<TProjectEnvironments, TImmutableDBKeys>;
+export type TProjectEnvironmentsInsert = Omit<z.input<typeof ProjectEnvironmentsSchema>, TImmutableDBKeys>;
-export type TProjectEnvironmentsUpdate = Partial<Omit<TProjectEnvironments, TImmutableDBKeys>>;
+export type TProjectEnvironmentsUpdate = Partial<Omit<z.input<typeof ProjectEnvironmentsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-keys.ts

@@ -19,5 +19,5 @@ export const ProjectKeysSchema = z.object({
 });
 
 export type TProjectKeys = z.infer<typeof ProjectKeysSchema>;
-export type TProjectKeysInsert = Omit<TProjectKeys, TImmutableDBKeys>;
+export type TProjectKeysInsert = Omit<z.input<typeof ProjectKeysSchema>, TImmutableDBKeys>;
-export type TProjectKeysUpdate = Partial<Omit<TProjectKeys, TImmutableDBKeys>>;
+export type TProjectKeysUpdate = Partial<Omit<z.input<typeof ProjectKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-memberships.ts

@@ -18,5 +18,5 @@ export const ProjectMembershipsSchema = z.object({
 });
 
 export type TProjectMemberships = z.infer<typeof ProjectMembershipsSchema>;
-export type TProjectMembershipsInsert = Omit<TProjectMemberships, TImmutableDBKeys>;
+export type TProjectMembershipsInsert = Omit<z.input<typeof ProjectMembershipsSchema>, TImmutableDBKeys>;
-export type TProjectMembershipsUpdate = Partial<Omit<TProjectMemberships, TImmutableDBKeys>>;
+export type TProjectMembershipsUpdate = Partial<Omit<z.input<typeof ProjectMembershipsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-roles.ts

@@ -19,5 +19,5 @@ export const ProjectRolesSchema = z.object({
 });
 
 export type TProjectRoles = z.infer<typeof ProjectRolesSchema>;
-export type TProjectRolesInsert = Omit<TProjectRoles, TImmutableDBKeys>;
+export type TProjectRolesInsert = Omit<z.input<typeof ProjectRolesSchema>, TImmutableDBKeys>;
-export type TProjectRolesUpdate = Partial<Omit<TProjectRoles, TImmutableDBKeys>>;
+export type TProjectRolesUpdate = Partial<Omit<z.input<typeof ProjectRolesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/project-user-additional-privilege.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const ProjectUserAdditionalPrivilegeSchema = z.object({
+  id: z.string().uuid(),
+  slug: z.string(),
+  projectMembershipId: z.string().uuid(),
+  isTemporary: z.boolean().default(false),
+  temporaryMode: z.string().nullable().optional(),
+  temporaryRange: z.string().nullable().optional(),
+  temporaryAccessStartTime: z.date().nullable().optional(),
+  temporaryAccessEndTime: z.date().nullable().optional(),
+  permissions: z.unknown(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TProjectUserAdditionalPrivilege = z.infer<typeof ProjectUserAdditionalPrivilegeSchema>;
+export type TProjectUserAdditionalPrivilegeInsert = Omit<
+  z.input<typeof ProjectUserAdditionalPrivilegeSchema>,
+  TImmutableDBKeys
+>;
+export type TProjectUserAdditionalPrivilegeUpdate = Partial<
+  Omit<z.input<typeof ProjectUserAdditionalPrivilegeSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/project-user-membership-roles.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const ProjectUserMembershipRolesSchema = z.object({
+  id: z.string().uuid(),
+  role: z.string(),
+  projectMembershipId: z.string().uuid(),
+  customRoleId: z.string().uuid().nullable().optional(),
+  isTemporary: z.boolean().default(false),
+  temporaryMode: z.string().nullable().optional(),
+  temporaryRange: z.string().nullable().optional(),
+  temporaryAccessStartTime: z.date().nullable().optional(),
+  temporaryAccessEndTime: z.date().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TProjectUserMembershipRoles = z.infer<typeof ProjectUserMembershipRolesSchema>;
+export type TProjectUserMembershipRolesInsert = Omit<
+  z.input<typeof ProjectUserMembershipRolesSchema>,
+  TImmutableDBKeys
+>;
+export type TProjectUserMembershipRolesUpdate = Partial<
+  Omit<z.input<typeof ProjectUserMembershipRolesSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/projects.ts

@@ -14,9 +14,11 @@ export const ProjectsSchema = z.object({
   autoCapitalization: z.boolean().default(true).nullable().optional(),
   orgId: z.string().uuid(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  version: z.number().default(1),
+  upgradeStatus: z.string().nullable().optional()
 });
 
 export type TProjects = z.infer<typeof ProjectsSchema>;
-export type TProjectsInsert = Omit<TProjects, TImmutableDBKeys>;
+export type TProjectsInsert = Omit<z.input<typeof ProjectsSchema>, TImmutableDBKeys>;
-export type TProjectsUpdate = Partial<Omit<TProjects, TImmutableDBKeys>>;
+export type TProjectsUpdate = Partial<Omit<z.input<typeof ProjectsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/saml-configs.ts

@@ -22,9 +22,10 @@ export const SamlConfigsSchema = z.object({
   certTag: z.string().nullable().optional(),
   createdAt: z.date(),
   updatedAt: z.date(),
-  orgId: z.string().uuid()
+  orgId: z.string().uuid(),
+  lastUsed: z.date().nullable().optional()
 });
 
 export type TSamlConfigs = z.infer<typeof SamlConfigsSchema>;
-export type TSamlConfigsInsert = Omit<TSamlConfigs, TImmutableDBKeys>;
+export type TSamlConfigsInsert = Omit<z.input<typeof SamlConfigsSchema>, TImmutableDBKeys>;
-export type TSamlConfigsUpdate = Partial<Omit<TSamlConfigs, TImmutableDBKeys>>;
+export type TSamlConfigsUpdate = Partial<Omit<z.input<typeof SamlConfigsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/scim-tokens.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const ScimTokensSchema = z.object({
+  id: z.string(),
+  ttlDays: z.coerce.number().default(365),
+  description: z.string(),
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TScimTokens = z.infer<typeof ScimTokensSchema>;
+export type TScimTokensInsert = Omit<z.input<typeof ScimTokensSchema>, TImmutableDBKeys>;
+export type TScimTokensUpdate = Partial<Omit<z.input<typeof ScimTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/secret-approval-policies-approvers.ts

@@ -16,5 +16,10 @@ export const SecretApprovalPoliciesApproversSchema = z.object({
 });
 
 export type TSecretApprovalPoliciesApprovers = z.infer<typeof SecretApprovalPoliciesApproversSchema>;
-export type TSecretApprovalPoliciesApproversInsert = Omit<TSecretApprovalPoliciesApprovers, TImmutableDBKeys>;
+export type TSecretApprovalPoliciesApproversInsert = Omit<
-export type TSecretApprovalPoliciesApproversUpdate = Partial<Omit<TSecretApprovalPoliciesApprovers, TImmutableDBKeys>>;
+  z.input<typeof SecretApprovalPoliciesApproversSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalPoliciesApproversUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalPoliciesApproversSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-policies.ts

@@ -18,5 +18,7 @@ export const SecretApprovalPoliciesSchema = z.object({
 });
 
 export type TSecretApprovalPolicies = z.infer<typeof SecretApprovalPoliciesSchema>;
-export type TSecretApprovalPoliciesInsert = Omit<TSecretApprovalPolicies, TImmutableDBKeys>;
+export type TSecretApprovalPoliciesInsert = Omit<z.input<typeof SecretApprovalPoliciesSchema>, TImmutableDBKeys>;
-export type TSecretApprovalPoliciesUpdate = Partial<Omit<TSecretApprovalPolicies, TImmutableDBKeys>>;
+export type TSecretApprovalPoliciesUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalPoliciesSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-request-secret-tags.ts

@@ -16,5 +16,10 @@ export const SecretApprovalRequestSecretTagsSchema = z.object({
 });
 
 export type TSecretApprovalRequestSecretTags = z.infer<typeof SecretApprovalRequestSecretTagsSchema>;
-export type TSecretApprovalRequestSecretTagsInsert = Omit<TSecretApprovalRequestSecretTags, TImmutableDBKeys>;
+export type TSecretApprovalRequestSecretTagsInsert = Omit<
-export type TSecretApprovalRequestSecretTagsUpdate = Partial<Omit<TSecretApprovalRequestSecretTags, TImmutableDBKeys>>;
+  z.input<typeof SecretApprovalRequestSecretTagsSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalRequestSecretTagsUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestSecretTagsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-requests-reviewers.ts

@@ -17,5 +17,10 @@ export const SecretApprovalRequestsReviewersSchema = z.object({
 });
 
 export type TSecretApprovalRequestsReviewers = z.infer<typeof SecretApprovalRequestsReviewersSchema>;
-export type TSecretApprovalRequestsReviewersInsert = Omit<TSecretApprovalRequestsReviewers, TImmutableDBKeys>;
+export type TSecretApprovalRequestsReviewersInsert = Omit<
-export type TSecretApprovalRequestsReviewersUpdate = Partial<Omit<TSecretApprovalRequestsReviewers, TImmutableDBKeys>>;
+  z.input<typeof SecretApprovalRequestsReviewersSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalRequestsReviewersUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestsReviewersSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-requests-secrets.ts

@@ -35,5 +35,10 @@ export const SecretApprovalRequestsSecretsSchema = z.object({
 });
 
 export type TSecretApprovalRequestsSecrets = z.infer<typeof SecretApprovalRequestsSecretsSchema>;
-export type TSecretApprovalRequestsSecretsInsert = Omit<TSecretApprovalRequestsSecrets, TImmutableDBKeys>;
+export type TSecretApprovalRequestsSecretsInsert = Omit<
-export type TSecretApprovalRequestsSecretsUpdate = Partial<Omit<TSecretApprovalRequestsSecrets, TImmutableDBKeys>>;
+  z.input<typeof SecretApprovalRequestsSecretsSchema>,
+  TImmutableDBKeys
+>;
+export type TSecretApprovalRequestsSecretsUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestsSecretsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/secret-approval-requests.ts

@@ -22,5 +22,7 @@ export const SecretApprovalRequestsSchema = z.object({
 });
 
 export type TSecretApprovalRequests = z.infer<typeof SecretApprovalRequestsSchema>;
-export type TSecretApprovalRequestsInsert = Omit<TSecretApprovalRequests, TImmutableDBKeys>;
+export type TSecretApprovalRequestsInsert = Omit<z.input<typeof SecretApprovalRequestsSchema>, TImmutableDBKeys>;
-export type TSecretApprovalRequestsUpdate = Partial<Omit<TSecretApprovalRequests, TImmutableDBKeys>>;
+export type TSecretApprovalRequestsUpdate = Partial<
+  Omit<z.input<typeof SecretApprovalRequestsSchema>, TImmutableDBKeys>
+>;