Merge pull request #1869 from Infisical/misc/made-aws-sm-mapping-plaintext-one-to-one

misc: made aws sm mapping one to one plaintext

.env.example

@@ -8,19 +8,17 @@ ENCRYPTION_KEY=6c1fe4e407b8911c104518103505b218
 # THIS IS A SAMPLE AUTH_SECRET KEY AND SHOULD NEVER BE USED FOR PRODUCTION
 AUTH_SECRET=5lrMXKKWCVocS/uerPsl7V+TX/aaUaI7iDkgl3tSmLE=
 
-# MongoDB
-# Backend will connect to the MongoDB instance at connection string MONGO_URL which can either be a ref
-# to the MongoDB container instance or Mongo Cloud
+# Postgres creds
+POSTGRES_PASSWORD=infisical
+POSTGRES_USER=infisical
+POSTGRES_DB=infisical
+
 # Required
-MONGO_URL=mongodb://root:example@mongo:27017/?authSource=admin
+DB_CONNECTION_URI=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
 
 # Redis
 REDIS_URL=redis://redis:6379
 
-# Optional credentials for MongoDB container instance and Mongo-Express
-MONGO_USERNAME=root
-MONGO_PASSWORD=example
-
 # Website URL
 # Required
 SITE_URL=http://localhost:8080

.env.migration.example

@@ -0,0 +1 @@
+DB_CONNECTION_URI=

.env.test.example

@@ -0,0 +1,4 @@
+REDIS_URL=redis://localhost:6379
+DB_CONNECTION_URI=postgres://infisical:infisical@localhost/infisical?sslmode=disable
+AUTH_SECRET=4bnfe4e407b8921c104518903515b218
+ENCRYPTION_KEY=4bnfe4e407b8921c104518903515b218

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 # Description 📣
 
-<!-- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. -->
+<!-- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. Here's how we expect a pull request to be : https://infisical.com/docs/contributing/getting-started/pull-requests -->
 
 ## Type ✨
 
@@ -19,4 +19,6 @@
 
 ---
 
-- [ ] I have read the [contributing guide](https://infisical.com/docs/contributing/overview), agreed and acknowledged the [code of conduct](https://infisical.com/docs/contributing/code-of-conduct). 📝
+- [ ] I have read the [contributing guide](https://infisical.com/docs/contributing/getting-started/overview), agreed and acknowledged the [code of conduct](https://infisical.com/docs/contributing/getting-started/code-of-conduct). 📝
+
+<!-- If you have any questions regarding contribution, here's the FAQ : https://infisical.com/docs/contributing/getting-started/faq -->

.github/resources/changelog-generator.py

@@ -0,0 +1,190 @@
+# inspired by https://www.photoroom.com/inside-photoroom/how-we-automated-our-changelog-thanks-to-chatgpt
+import os
+import requests
+import re
+from openai import OpenAI
+import subprocess
+from datetime import datetime
+
+import uuid
+
+# Constants
+REPO_OWNER = "infisical"
+REPO_NAME = "infisical"
+TOKEN = os.environ["GITHUB_TOKEN"]
+SLACK_WEBHOOK_URL = os.environ["SLACK_WEBHOOK_URL"]
+OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
+SLACK_MSG_COLOR = "#36a64f"
+
+headers = {
+    "Authorization": f"Bearer {TOKEN}",
+    "Accept": "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
+
+
+def set_multiline_output(name, value):
+    with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+        delimiter = uuid.uuid1()
+        print(f'{name}<<{delimiter}', file=fh)
+        print(value, file=fh)
+        print(delimiter, file=fh)
+
+def post_changelog_to_slack(changelog, tag):
+    slack_payload = {
+        "text": "Hey team, it's changelog time! :wave:",
+        "attachments": [
+            {
+                "color": SLACK_MSG_COLOR,
+                "title": f"🗓️Infisical Changelog - {tag}",
+                "text": changelog,
+            }
+        ],
+    }
+
+    response = requests.post(SLACK_WEBHOOK_URL, json=slack_payload)
+
+    if response.status_code != 200:
+        raise Exception("Failed to post changelog to Slack.")
+
+def find_previous_release_tag(release_tag:str):
+    previous_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0", f"{release_tag}^"]).decode("utf-8").strip()
+    while not(previous_tag.startswith("infisical/")):
+        previous_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0", f"{previous_tag}^"]).decode("utf-8").strip()
+    return previous_tag
+
+def get_tag_creation_date(tag_name):
+    url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/git/refs/tags/{tag_name}"
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    commit_sha = response.json()['object']['sha']
+    
+    commit_url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/commits/{commit_sha}"
+    commit_response = requests.get(commit_url, headers=headers)
+    commit_response.raise_for_status()
+    creation_date = commit_response.json()['commit']['author']['date']
+
+    return datetime.strptime(creation_date, '%Y-%m-%dT%H:%M:%SZ')
+
+
+def fetch_prs_between_tags(previous_tag_date:datetime, release_tag_date:datetime):
+    # Use GitHub API to fetch PRs merged between the commits
+    url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/pulls?state=closed&merged=true"
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception("Error fetching PRs from GitHub API!")
+
+    prs = []
+    for pr in response.json():
+        # the idea is as tags happen recently we get last 100 closed PRs and then filter by tag creation date
+        if pr["merged_at"] and datetime.strptime(pr["merged_at"],'%Y-%m-%dT%H:%M:%SZ') < release_tag_date and  datetime.strptime(pr["merged_at"],'%Y-%m-%dT%H:%M:%SZ') > previous_tag_date:
+            prs.append(pr)
+
+    return prs
+
+
+def extract_commit_details_from_prs(prs):
+    commit_details = []
+    for pr in prs:
+        commit_message = pr["title"]
+        commit_url = pr["html_url"]
+        pr_number = pr["number"]
+        branch_name = pr["head"]["ref"]
+        issue_numbers = re.findall(r"(www-\d+|web-\d+)", branch_name)
+
+        # If no issue numbers are found, add the PR details without issue numbers and URLs
+        if not issue_numbers:
+            commit_details.append(
+                {
+                    "message": commit_message,
+                    "pr_number": pr_number,
+                    "pr_url": commit_url,
+                    "issue_number": None,
+                    "issue_url": None,
+                }
+            )
+            continue
+
+        for issue in issue_numbers:
+            commit_details.append(
+                {
+                    "message": commit_message,
+                    "pr_number": pr_number,
+                    "pr_url": commit_url,
+                    "issue_number": issue,
+                }
+            )
+
+    return commit_details
+
+# Function to generate changelog using OpenAI
+def generate_changelog_with_openai(commit_details):
+    commit_messages = []
+    for details in commit_details:
+        base_message = f"{details['pr_url']} - {details['message']}"
+        # Add the issue URL if available
+        # if details["issue_url"]:
+        #     base_message += f" (Linear Issue: {details['issue_url']})"
+        commit_messages.append(base_message)
+
+    commit_list = "\n".join(commit_messages)
+    prompt = """
+Generate a changelog for Infisical, opensource secretops
+The changelog should:
+1. Be Informative: Using the provided list of GitHub commits, break them down into categories such as Features, Fixes & Improvements, and Technical Updates. Summarize each commit concisely, ensuring the key points are highlighted.
+2. Have a Professional yet Friendly tone: The tone should be balanced, not too corporate or too informal.
+3. Celebratory Introduction and Conclusion: Start the changelog with a celebratory note acknowledging the team's hard work and progress. End with a shoutout to the team and wishes for a pleasant weekend.
+4. Formatting: you cannot use Markdown formatting, and you can only use emojis for the introductory paragraph or the conclusion paragraph, nowhere else.
+5. Links: the syntax to create links is the following: `<http://www.example.com|This message is a link>`.
+6. Linear Links: note that the Linear link is optional, include it only if provided.
+7. Do not wrap your answer in a codeblock. Just output the text, nothing else
+Here's a good example to follow, please try to match the formatting as closely as possible, only changing the content of the changelog and have some liberty with the introduction. Notice the importance of the formatting of a changelog item:
+- <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>))
+And here's an example of the full changelog:
+
+*Features*
+• <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>)
+*Fixes & Improvements*
+• <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>)
+*Technical Updates*
+• <https://github.com/facebook/react/pull/27304/%7C#27304>: We optimize our ci to strip comments and minify production builds. (<https://linear.app/example/issue/WEB-1234/%7CWEB-1234>)
+
+Stay tuned for more exciting updates coming soon!
+And here are the commits:
+{}
+    """.format(
+        commit_list
+    )
+
+    client  = OpenAI(api_key=OPENAI_API_KEY)
+    messages = [{"role": "user", "content": prompt}]
+    response = client.chat.completions.create(model="gpt-3.5-turbo", messages=messages)
+
+    if "error" in response.choices[0].message:
+        raise Exception("Error generating changelog with OpenAI!")
+
+    return response.choices[0].message.content.strip()
+
+
+if __name__ == "__main__":
+    try:
+        # Get the latest and previous release tags
+        latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"]).decode("utf-8").strip()
+        previous_tag = find_previous_release_tag(latest_tag)
+
+        latest_tag_date = get_tag_creation_date(latest_tag)
+        previous_tag_date = get_tag_creation_date(previous_tag)
+
+        prs = fetch_prs_between_tags(previous_tag_date,latest_tag_date)
+        pr_details = extract_commit_details_from_prs(prs)
+
+        # Generate changelog
+        changelog = generate_changelog_with_openai(pr_details)
+
+        post_changelog_to_slack(changelog,latest_tag)
+        # Print or post changelog to Slack
+        # set_multiline_output("changelog", changelog)
+
+    except Exception as e:
+        print(str(e))

.github/resources/rename_migration_files.py

@@ -0,0 +1,26 @@
+import os
+from datetime import datetime, timedelta
+
+def rename_migrations():
+    migration_folder = "./backend/src/db/migrations"
+    with open("added_files.txt", "r") as file:
+        changed_files = file.readlines()
+    
+    # Find the latest file among the changed files
+    latest_timestamp = datetime.now() # utc time
+    for file_path in changed_files:
+        file_path = file_path.strip()
+        # each new file bump by 1s
+        latest_timestamp = latest_timestamp + timedelta(seconds=1)
+
+        new_filename = os.path.join(migration_folder, latest_timestamp.strftime("%Y%m%d%H%M%S") + f"_{file_path.split('_')[1]}")
+        old_filename = os.path.join(migration_folder, file_path)
+        os.rename(old_filename, new_filename)
+        print(f"Renamed {old_filename} to {new_filename}")
+
+    if len(changed_files) == 0:
+        print("No new files added to migration folder")
+
+if __name__ == "__main__":
+    rename_migrations()
+

.github/values.yaml

@@ -13,22 +13,21 @@ fullnameOverride: ""
 ##
 
 infisical:
-  ## @param backend.enabled Enable backend
-  ##
+  autoDatabaseSchemaMigration: false
+  
   enabled: false
-  ## @param backend.name Backend name
-  ##
+
   name: infisical
-  replicaCount: 2
+  replicaCount: 3
   image:
-    repository: infisical/infisical
-    tag: "latest-postgres"
-    pullPolicy: IfNotPresent
+    repository: infisical/staging_infisical
+    tag: "latest"
+    pullPolicy: Always
 
   deploymentAnnotations:
     secrets.infisical.com/auto-reload: "true"
 
-  kubeSecretRef: "infisical-gamma-secrets"
+  kubeSecretRef: "managed-secret"
 
 ingress:
   ## @param ingress.enabled Enable ingress
@@ -50,3 +49,9 @@ ingress:
     - secretName: letsencrypt-prod
       hosts:
         - gamma.infisical.com
+
+postgresql:
+  enabled: false
+
+redis:
+  enabled: false

.github/workflows/build-docker-image-to-prod.yml

@@ -41,6 +41,7 @@ jobs:
           load: true
           context: backend
           tags: infisical/infisical:test
+          platforms: linux/amd64,linux/arm64
       - name: ⏻ Spawn backend container and dependencies
         run: |
           docker compose -f .github/resources/docker-compose.be-test.yml up --wait --quiet-pull
@@ -92,6 +93,7 @@ jobs:
           project: 64mmf0n610
           context: frontend
           tags: infisical/frontend:test
+          platforms: linux/amd64,linux/arm64
           build-args: |
             POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
             NEXT_INFISICAL_PLATFORM_VERSION=${{ steps.extract_version.outputs.version }}

.github/workflows/build-patroni-docker-img.yml

@@ -0,0 +1,38 @@
+name: Build patroni 
+on: [workflow_dispatch]
+
+jobs:
+  patroni-image:
+    name: Build patroni
+    runs-on: ubuntu-latest
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+        with:
+          repository: 'zalando/patroni'
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: 🔧 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: 🐋 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
+      - name: 🏗️ Build backend and push to docker hub
+        uses: depot/build-push-action@v1
+        with:
+          project: 64mmf0n610
+          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
+          push: true
+          context: .
+          file: Dockerfile
+          tags: |
+            infisical/patroni:${{ steps.commit.outputs.short }}
+            infisical/patroni:latest
+          platforms: linux/amd64,linux/arm64
+    
+  

.github/workflows/build-staging-and-deploy-aws.yml

@@ -0,0 +1,140 @@
+name: Deployment pipeline
+on: [workflow_dispatch]
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  infisical-image:
+    name: Build backend image
+    runs-on: ubuntu-latest
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+      - name: 📦 Install dependencies to test all dependencies
+        run: npm ci --only-production
+        working-directory: backend
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: 🔧 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: 🐋 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
+      - name: 🏗️ Build backend and push to docker hub
+        uses: depot/build-push-action@v1
+        with:
+          project: 64mmf0n610
+          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
+          push: true
+          context: .
+          file: Dockerfile.standalone-infisical
+          tags: |
+            infisical/staging_infisical:${{ steps.commit.outputs.short }}
+            infisical/staging_infisical:latest
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
+            INFISICAL_PLATFORM_VERSION=${{ steps.commit.outputs.short }}
+    
+  gamma-deployment:
+    name: Deploy to gamma
+    runs-on: ubuntu-latest
+    needs: [infisical-image]
+    environment:
+      name: Gamma
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v2
+        with:
+          node-version: "20"
+      - name: Change directory to backend and install dependencies
+        env:
+          DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
+        run: |
+          cd backend
+          npm install
+          npm run migration:latest
+      - name: Configure AWS Credentials 
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: us-east-1
+          role-to-assume: arn:aws:iam::905418227878:role/deploy-new-ecs-img
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: Download task definition
+        run: |
+          aws ecs describe-task-definition --task-definition infisical-core-platform --query taskDefinition > task-definition.json
+      - name: Render Amazon ECS task definition
+        id: render-web-container
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: infisical-core-platform
+          image: infisical/staging_infisical:${{ steps.commit.outputs.short }}
+          environment-variables: "LOG_LEVEL=info"
+      - name: Deploy to Amazon ECS service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.render-web-container.outputs.task-definition }}
+          service: infisical-core-platform
+          cluster: infisical-core-platform
+          wait-for-service-stability: true
+
+  production-postgres-deployment:
+    name: Deploy to production
+    runs-on: ubuntu-latest
+    needs: [gamma-deployment]
+    environment:
+      name: Production
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v2
+        with:
+          node-version: "20"
+      - name: Change directory to backend and install dependencies
+        env:
+          DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
+        run: |
+          cd backend
+          npm install
+          npm run migration:latest
+      - name: Configure AWS Credentials 
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: us-east-1
+          role-to-assume: arn:aws:iam::381492033652:role/gha-make-prod-deployment
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: Download task definition
+        run: |
+          aws ecs describe-task-definition --task-definition infisical-core-platform --query taskDefinition > task-definition.json
+      - name: Render Amazon ECS task definition
+        id: render-web-container
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: infisical-core-platform
+          image: infisical/staging_infisical:${{ steps.commit.outputs.short }}
+          environment-variables: "LOG_LEVEL=info"
+      - name: Deploy to Amazon ECS service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.render-web-container.outputs.task-definition }}
+          service: infisical-core-platform
+          cluster: infisical-core-platform
+          wait-for-service-stability: true

.github/workflows/build-staging-and-deploy.yml

@@ -1,120 +0,0 @@
-name: Build, Publish and Deploy to Gamma
-on: [workflow_dispatch]
-
-jobs:
-  infisical-image:
-    name: Build backend image
-    runs-on: ubuntu-latest
-    steps:
-      - name: ☁️ Checkout source
-        uses: actions/checkout@v3
-      - name: 📦 Install dependencies to test all dependencies
-        run: npm ci --only-production
-        working-directory: backend
-      # - name: 🧪 Run tests
-      #   run: npm run test:ci
-      #   working-directory: backend
-      - name: Save commit hashes for tag
-        id: commit
-        uses: pr-mpt/actions-commit-hash@v2
-      - name: 🔧 Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: 🐋 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Set up Depot CLI
-        uses: depot/setup-action@v1
-      - name: 📦 Build backend and export to Docker
-        uses: depot/build-push-action@v1
-        with:
-          project: 64mmf0n610
-          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
-          load: true
-          context: .
-          file: Dockerfile.standalone-infisical
-          tags: infisical/infisical:test
-      # - name: ⏻ Spawn backend container and dependencies
-      #   run: |
-      #     docker compose -f .github/resources/docker-compose.be-test.yml up --wait --quiet-pull
-      # - name: 🧪 Test backend image
-      #   run: |
-      #     ./.github/resources/healthcheck.sh infisical-backend-test
-      # - name: ⏻ Shut down backend container and dependencies
-      #   run: |
-      #     docker compose -f .github/resources/docker-compose.be-test.yml down
-      - name: 🏗️ Build backend and push
-        uses: depot/build-push-action@v1
-        with:
-          project: 64mmf0n610
-          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
-          push: true
-          context: .
-          file: Dockerfile.standalone-infisical
-          tags: |
-            infisical/staging_infisical:${{ steps.commit.outputs.short }}
-            infisical/staging_infisical:latest
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
-            INFISICAL_PLATFORM_VERSION=${{ steps.extract_version.outputs.version }}
-  postgres-migration:
-    name: Run latest migration files
-    runs-on: ubuntu-latest
-    needs: [infisical-image]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v2
-        with:
-          node-version: "20"
-      - name: Change directory to backend and install dependencies
-        env:
-          DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
-        run: |
-          cd backend
-          npm install
-          npm run migration:latest
-      # - name: Run postgres DB migration files
-      #   env:
-      #     DB_CONNECTION_URI: ${{ secrets.DB_CONNECTION_URI }}
-      #   run: npm run migration:latest
-  gamma-deployment:
-    name: Deploy to gamma
-    runs-on: ubuntu-latest
-    needs: [postgres-migration]
-    steps:
-      - name: ☁️ Checkout source
-        uses: actions/checkout@v3
-      - name: Install Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.10.0
-      - name: Install infisical helm chart
-        run: |
-          helm repo add infisical-helm-charts 'https://dl.cloudsmith.io/public/infisical/helm-charts/helm/charts/'
-          helm repo update
-      - name: Install kubectl
-        uses: azure/setup-kubectl@v3
-      - name: Install doctl
-        uses: digitalocean/action-doctl@v2
-        with:
-          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-      - name: Save DigitalOcean kubeconfig with short-lived credentials
-        run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 infisical-gamma-postgres
-      - name: switch to gamma namespace
-        run: kubectl config set-context --current --namespace=gamma
-      - name: test kubectl
-        run: kubectl get ingress
-      - name: Download helm values to file and upgrade gamma deploy
-        run: |
-          wget https://raw.githubusercontent.com/Infisical/infisical/main/.github/values.yaml
-          helm upgrade infisical infisical-helm-charts/infisical-standalone  --values values.yaml --wait --install
-          if [[ $(helm status infisical) == *"FAILED"* ]]; then
-            echo "Helm upgrade failed"
-            exit 1
-          else
-            echo "Helm upgrade was successful"
-          fi

.github/workflows/check-api-for-breaking-changes.yml

@@ -0,0 +1,76 @@
+name: "Check API For Breaking Changes"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "backend/src/server/routes/**"
+      - "backend/src/ee/routes/**"
+
+jobs:
+  check-be-api-changes:
+    name: Check API Changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+      # - name: Setup Node 20
+      #   uses: actions/setup-node@v3
+      #   with:
+      #     node-version: "20"
+      # uncomment this when testing locally using nektos/act
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        if: ${{ env.ACT }}
+        name: Install `docker-compose` for local simulations
+        with:
+          version: "2.14.2"
+      - name: 📦Build the latest image
+        run: docker build --tag infisical-api .
+        working-directory: backend
+      - name: Start postgres and redis
+        run: touch .env && docker-compose -f docker-compose.dev.yml up -d db redis
+      - name: Start the server
+        run: |
+            echo "SECRET_SCANNING_GIT_APP_ID=793712" >> .env
+            echo "SECRET_SCANNING_PRIVATE_KEY=some-random" >> .env
+            echo "SECRET_SCANNING_WEBHOOK_SECRET=some-random" >> .env
+            docker run --name infisical-api -d -p 4000:4000 -e DB_CONNECTION_URI=$DB_CONNECTION_URI -e REDIS_URL=$REDIS_URL -e JWT_AUTH_SECRET=$JWT_AUTH_SECRET --env-file .env --entrypoint '/bin/sh' infisical-api -c "npm run migration:latest && ls && node dist/main.mjs"
+        env:
+          REDIS_URL: redis://172.17.0.1:6379
+          DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable
+          JWT_AUTH_SECRET: something-random
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.5'
+      - name: Wait for container to be stable and check logs
+        run: |
+          SECONDS=0
+          HEALTHY=0
+          while [ $SECONDS -lt 60 ]; do
+            if docker ps | grep infisical-api | grep -q healthy; then
+              echo "Container is healthy."
+              HEALTHY=1
+              break
+            fi
+            echo "Waiting for container to be healthy... ($SECONDS seconds elapsed)"
+
+            docker logs infisical-api
+
+            sleep 2
+            SECONDS=$((SECONDS+2))
+          done
+      
+          if [ $HEALTHY -ne 1 ]; then
+            echo "Container did not become healthy in time"
+            exit 1
+          fi
+      - name: Install openapi-diff
+        run: go install github.com/tufin/oasdiff@latest
+      - name: Running OpenAPI Spec diff action
+        run: oasdiff breaking https://app.infisical.com/api/docs/json http://localhost:4000/api/docs/json --fail-on ERR
+      - name: cleanup
+        run: |
+          docker-compose -f "docker-compose.dev.yml" down
+          docker stop infisical-api
+          docker remove infisical-api

.github/workflows/check-be-pull-request.yml

@@ -1,43 +0,0 @@
-name: "Check Backend Pull Request"
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-    paths:
-      - "backend/**"
-      - "!backend/README.md"
-      - "!backend/.*"
-      - "backend/.eslintrc.js"
-
-jobs:
-  check-be-pr:
-    name: Check
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: ☁️ Checkout source
-        uses: actions/checkout@v3
-      - name: 🔧 Setup Node 16
-        uses: actions/setup-node@v3
-        with:
-          node-version: "16"
-          cache: "npm"
-          cache-dependency-path: backend/package-lock.json
-      - name: 📦 Install dependencies
-        run: npm ci --only-production
-        working-directory: backend
-      # - name: 🧪 Run tests
-      #   run: npm run test:ci
-      #   working-directory: backend
-      # - name: 📁 Upload test results
-      #   uses: actions/upload-artifact@v3
-      #   if: always()
-      #   with:
-      #     name: be-test-results
-      #     path: |
-      #       ./backend/reports
-      #       ./backend/coverage
-      - name: 🏗️ Run build
-        run: npm run build
-        working-directory: backend

.github/workflows/check-be-ts-and-lint.yml

@@ -0,0 +1,35 @@
+name: "Check Backend PR types and lint"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "backend/**"
+      - "!backend/README.md"
+      - "!backend/.*"
+      - "backend/.eslintrc.js"
+
+jobs:
+  check-be-pr:
+    name: Check TS and Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+      - name: 🔧 Setup Node 20
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: backend/package-lock.json
+      - name: Install dependencies
+        run: npm install
+        working-directory: backend
+      - name: Run type check
+        run: npm run type:check
+        working-directory: backend
+      - name: Run lint check
+        run: npm run lint
+        working-directory: backend

.github/workflows/check-fe-ts-and-lint.yml

@@ -1,4 +1,4 @@
-name: Check Frontend Pull Request
+name: Check Frontend Type and Lint check
 
 on:
   pull_request:
@@ -10,8 +10,8 @@ on:
       - "frontend/.eslintrc.js"
 
 jobs:
-  check-fe-pr:
-    name: Check
+  check-fe-ts-lint:
+    name: Check Frontend Type and Lint check
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
@@ -25,12 +25,11 @@ jobs:
           cache: "npm"
           cache-dependency-path: frontend/package-lock.json
       - name: 📦 Install dependencies
-        run: npm ci --only-production --ignore-scripts
+        run: npm install
         working-directory: frontend
-      # -
-      #   name: 🧪 Run tests
-      #   run: npm run test:ci
-      #   working-directory: frontend
-      - name: 🏗️ Run build
-        run: npm run build
+      - name: 🏗️ Run Type check
+        run: npm run type:check 
+        working-directory: frontend
+      - name: 🏗️ Run Link check
+        run: npm run lint:fix 
         working-directory: frontend

.github/workflows/generate-release-changelog.yml

@@ -0,0 +1,34 @@
+name: Generate Changelog
+permissions:
+  contents: write
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "infisical/v*.*.*-postgres"
+
+jobs:
+  generate_changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12.0"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests openai
+      - name: Generate Changelog and Post to Slack
+        id: gen-changelog
+        run: python .github/resources/changelog-generator.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/release-standalone-docker-img-postgres-offical.yml

@@ -5,9 +5,14 @@ on:
       - "infisical/v*.*.*-postgres"
 
 jobs:
+  infisical-tests:
+    name: Run tests before deployment
+    # https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
+    uses: ./.github/workflows/run-backend-tests.yml
   infisical-standalone:
     name: Build infisical standalone image postgres
     runs-on: ubuntu-latest
+    needs: [infisical-tests]
     steps:
       - name: Extract version from tag
         id: extract_version

.github/workflows/release_build_infisical_cli.yml

@@ -1,58 +1,72 @@
 name: Build and release CLI
 
 on:
-  push:
-    # run only against tags
-    tags:
-      - "infisical-cli/v*.*.*"
+    workflow_dispatch:
+
+    push:
+        # run only against tags
+        tags:
+            - "infisical-cli/v*.*.*"
 
 permissions:
-  contents: write
-  # packages: write
-  # issues: write
-
+    contents: write
+    # packages: write
+    # issues: write
 jobs:
-  goreleaser:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: 🐋 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - run: git fetch --force --tags
-      - run: echo "Ref name ${{github.ref_name}}"
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.19.3"
-          cache: true
-          cache-dependency-path: cli/go.sum
-      - name: libssl1.1 => libssl1.0-dev for OSXCross
-        run: |
-          echo 'deb http://security.ubuntu.com/ubuntu bionic-security main' | sudo tee -a /etc/apt/sources.list
-          sudo apt update && apt-cache policy libssl1.0-dev
-          sudo apt-get install libssl1.0-dev
-      - name: OSXCross for CGO Support
-        run: |
-          mkdir ../../osxcross
-          git clone https://github.com/plentico/osxcross-target.git ../../osxcross/target
-      - uses: goreleaser/goreleaser-action@v4
-        with:
-          distribution: goreleaser-pro
-          version: latest
-          args: release --clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GO_RELEASER_GITHUB_TOKEN }}
-          POSTHOG_API_KEY_FOR_CLI: ${{ secrets.POSTHOG_API_KEY_FOR_CLI }}
-          FURY_TOKEN: ${{ secrets.FURYPUSHTOKEN }}
-          AUR_KEY: ${{ secrets.AUR_KEY }}
-          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
-      - uses: actions/setup-python@v4
-      - run: pip install --upgrade cloudsmith-cli
-      - name: Publish to CloudSmith
-        run: sh cli/upload_to_cloudsmith.sh
-        env:
-          CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }}
+    cli-integration-tests:
+        name: Run tests before deployment
+        uses: ./.github/workflows/run-cli-tests.yml
+        secrets:
+            CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }}
+            CLI_TESTS_UA_CLIENT_SECRET: ${{ secrets.CLI_TESTS_UA_CLIENT_SECRET }}
+            CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }}
+            CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }}
+            CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }}
+
+    goreleaser:
+        runs-on: ubuntu-20.04
+        needs: [cli-integration-tests]
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  fetch-depth: 0
+            - name: 🐋 Login to Docker Hub
+              uses: docker/login-action@v2
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: 🔧 Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+            - run: git fetch --force --tags
+            - run: echo "Ref name ${{github.ref_name}}"
+            - uses: actions/setup-go@v3
+              with:
+                  go-version: ">=1.19.3"
+                  cache: true
+                  cache-dependency-path: cli/go.sum
+            - name: libssl1.1 => libssl1.0-dev for OSXCross
+              run: |
+                  echo 'deb http://security.ubuntu.com/ubuntu bionic-security main' | sudo tee -a /etc/apt/sources.list
+                  sudo apt update && apt-cache policy libssl1.0-dev
+                  sudo apt-get install libssl1.0-dev
+            - name: OSXCross for CGO Support
+              run: |
+                  mkdir ../../osxcross
+                  git clone https://github.com/plentico/osxcross-target.git ../../osxcross/target
+            - uses: goreleaser/goreleaser-action@v4
+              with:
+                  distribution: goreleaser-pro
+                  version: latest
+                  args: release --clean
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GO_RELEASER_GITHUB_TOKEN }}
+                  POSTHOG_API_KEY_FOR_CLI: ${{ secrets.POSTHOG_API_KEY_FOR_CLI }}
+                  FURY_TOKEN: ${{ secrets.FURYPUSHTOKEN }}
+                  AUR_KEY: ${{ secrets.AUR_KEY }}
+                  GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+            - uses: actions/setup-python@v4
+            - run: pip install --upgrade cloudsmith-cli
+            - name: Publish to CloudSmith
+              run: sh cli/upload_to_cloudsmith.sh
+              env:
+                  CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }}

.github/workflows/run-backend-tests.yml

@@ -0,0 +1,47 @@
+name: "Run backend tests"
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "backend/**"
+      - "!backend/README.md"
+      - "!backend/.*"
+      - "backend/.eslintrc.js"
+  workflow_call:
+
+jobs:
+  check-be-pr:
+    name: Run integration test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        if: ${{ env.ACT }}
+        name: Install `docker-compose` for local simulations
+        with:
+          version: "2.14.2"
+      - name: 🔧 Setup Node 20
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: backend/package-lock.json
+      - name: Install dependencies
+        run: npm install
+        working-directory: backend
+      - name: Start postgres and redis
+        run: touch .env && docker-compose -f docker-compose.dev.yml up -d db redis
+      - name: Start integration test
+        run: npm run test:e2e
+        working-directory: backend
+        env:
+          REDIS_URL: redis://172.17.0.1:6379
+          DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable
+          AUTH_SECRET: something-random
+          ENCRYPTION_KEY: 4bnfe4e407b8921c104518903515b218
+      - name: cleanup
+        run: |
+          docker-compose -f "docker-compose.dev.yml" down

.github/workflows/run-cli-tests.yml

@@ -0,0 +1,47 @@
+name: Go CLI Tests
+
+on:
+    pull_request:
+        types: [opened, synchronize]
+        paths:
+            - "cli/**"
+
+    workflow_dispatch:
+
+    workflow_call:
+        secrets:
+            CLI_TESTS_UA_CLIENT_ID:
+                required: true
+            CLI_TESTS_UA_CLIENT_SECRET:
+                required: true
+            CLI_TESTS_SERVICE_TOKEN:
+                required: true
+            CLI_TESTS_PROJECT_ID:
+                required: true
+            CLI_TESTS_ENV_SLUG:
+                required: true
+
+jobs:
+    test:
+        defaults:
+            run:
+                working-directory: ./cli
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v4
+            - name: Setup Go
+              uses: actions/setup-go@v4
+              with:
+                  go-version: "1.21.x"
+            - name: Install dependencies
+              run: go get .
+            - name: Test with the Go CLI
+              env:
+                  CLI_TESTS_UA_CLIENT_ID: ${{ secrets.CLI_TESTS_UA_CLIENT_ID }}
+                  CLI_TESTS_UA_CLIENT_SECRET: ${{ secrets.CLI_TESTS_UA_CLIENT_SECRET }}
+                  CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }}
+                  CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }}
+                  CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }}
+
+              run: go test -v -count=1 ./test

.github/workflows/update-be-new-migration-latest-timestamp.yml

@@ -0,0 +1,59 @@
+name: Rename Migrations
+
+on:
+  pull_request:
+    types: [closed]
+    paths:
+      - 'backend/src/db/migrations/**'
+
+jobs:
+  rename:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get list of newly added files in migration folder
+        run: |
+          git diff --name-status HEAD^ HEAD backend/src/db/migrations | grep '^A' | cut -f2 | xargs -n1 basename > added_files.txt
+          if [ ! -s added_files.txt ]; then
+            echo "No new files added. Skipping"
+            echo "SKIP_RENAME=true" >> $GITHUB_ENV
+          fi
+
+      - name: Script to rename migrations
+        if: env.SKIP_RENAME != 'true'
+        run: python .github/resources/rename_migration_files.py
+
+      - name: Commit and push changes
+        if: env.SKIP_RENAME != 'true'
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git add ./backend/src/db/migrations
+          rm added_files.txt
+          git commit -m "chore: renamed new migration files to latest timestamp (gh-action)"
+
+      - name: Get PR details
+        id: pr_details
+        run: |
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          PR_MERGER=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | jq -r '.merged_by.login')
+          
+          echo "PR Number: $PR_NUMBER"
+          echo "PR Merger: $PR_MERGER"
+          echo "pr_merger=$PR_MERGER" >> $GITHUB_OUTPUT
+
+      - name: Create Pull Request
+        if: env.SKIP_RENAME != 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: 'chore: renamed new migration files to latest UTC (gh-action)'
+          title: 'GH Action: rename new migration file timestamp'
+          branch-suffix: timestamp
+          reviewers: ${{ steps.pr_details.outputs.pr_merger }}

.gitignore

@@ -6,7 +6,7 @@ node_modules
 .env.gamma
 .env.prod
 .env.infisical
-
+.env.migration
 *~
 *.swp
 *.swo
@@ -59,7 +59,13 @@ yarn-error.log*
 # Infisical init
 .infisical.json
 
+.infisicalignore
+
 # Editor specific
 .vscode/*
 
 frontend-build
+
+*.tgz
+cli/infisical-merge
+cli/test/infisical-merge

.goreleaser.yaml

@@ -190,10 +190,34 @@ dockers:
   - dockerfile: docker/alpine
     goos: linux
     goarch: amd64
+    use: buildx
     ids:
       - all-other-builds
     image_templates:
-      - "infisical/cli:{{ .Version }}"
-      - "infisical/cli:{{ .Major }}.{{ .Minor }}"
-      - "infisical/cli:{{ .Major }}"
-      - "infisical/cli:latest"
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-amd64"
+      - "infisical/cli:latest-amd64"
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/amd64"
+  - dockerfile: docker/alpine
+    goos: linux
+    goarch: amd64
+    use: buildx
+    ids:
+      - all-other-builds
+    image_templates:
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-arm64"
+      - "infisical/cli:latest-arm64"
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/arm64"
+
+docker_manifests:
+  - name_template: "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
+    image_templates:
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-amd64"
+      - "infisical/cli:{{ .Major }}.{{ .Minor }}.{{ .Patch }}-arm64"
+  - name_template: "infisical/cli:latest"
+    image_templates:
+      - "infisical/cli:latest-amd64"
+      - "infisical/cli:latest-arm64"

.infisicalignore

@@ -1 +1,7 @@
 .github/resources/docker-compose.be-test.yml:generic-api-key:16
+frontend/src/views/Project/MembersPage/components/IdentityTab/components/IdentityRoleForm/IdentityRbacSection.tsx:generic-api-key:206
+frontend/src/views/Project/MembersPage/components/IdentityTab/components/IdentityRoleForm/SpecificPrivilegeSection.tsx:generic-api-key:304
+frontend/src/views/Project/MembersPage/components/MemberListTab/MemberRoleForm/MemberRbacSection.tsx:generic-api-key:206
+frontend/src/views/Project/MembersPage/components/MemberListTab/MemberRoleForm/SpecificPrivilegeSection.tsx:generic-api-key:292
+docs/self-hosting/configuration/envars.mdx:generic-api-key:106
+frontend/src/views/Project/MembersPage/components/MemberListTab/MemberRoleForm/SpecificPrivilegeSection.tsx:generic-api-key:451

CONTRIBUTING.md

@@ -2,6 +2,6 @@
 
 Thanks for taking the time to contribute! 😃 🚀
 
-Please refer to our [Contributing Guide](https://infisical.com/docs/contributing/overview) for instructions on how to contribute.
+Please refer to our [Contributing Guide](https://infisical.com/docs/contributing/getting-started/overview) for instructions on how to contribute.
 
 We also have some 🔥amazing🔥 merch for our contributors. Please reach out to tony@infisical.com for more info 👀

Dockerfile.standalone-infisical

@@ -1,6 +1,7 @@
 ARG POSTHOG_HOST=https://app.posthog.com
 ARG POSTHOG_API_KEY=posthog-api-key
 ARG INTERCOM_ID=intercom-id
+ARG SAML_ORG_SLUG=saml-org-slug-default
 
 FROM  node:20-alpine AS base
 
@@ -35,6 +36,8 @@ ARG INTERCOM_ID
 ENV NEXT_PUBLIC_INTERCOM_ID $INTERCOM_ID
 ARG INFISICAL_PLATFORM_VERSION
 ENV NEXT_PUBLIC_INFISICAL_PLATFORM_VERSION $INFISICAL_PLATFORM_VERSION
+ARG SAML_ORG_SLUG
+ENV NEXT_PUBLIC_SAML_ORG_SLUG=$SAML_ORG_SLUG 
 
 # Build
 RUN npm run build
@@ -100,11 +103,13 @@ ENV NEXT_PUBLIC_POSTHOG_API_KEY=$POSTHOG_API_KEY \
 ARG INTERCOM_ID=intercom-id
 ENV NEXT_PUBLIC_INTERCOM_ID=$INTERCOM_ID \
   BAKED_NEXT_PUBLIC_INTERCOM_ID=$INTERCOM_ID
+ARG SAML_ORG_SLUG
+ENV NEXT_PUBLIC_SAML_ORG_SLUG=$SAML_ORG_SLUG \
+  BAKED_NEXT_PUBLIC_SAML_ORG_SLUG=$SAML_ORG_SLUG
 
 WORKDIR / 
 
 COPY --from=backend-runner /app /backend
-COPY --from=backend-runner /app/dist/services/smtp/templates /backend/dist/templates
 
 COPY --from=frontend-runner /app ./backend/frontend-build
 
@@ -119,9 +124,6 @@ WORKDIR /backend
 
 ENV TELEMETRY_ENABLED true
 
-HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \  
-  CMD node healthcheck.js
-
 EXPOSE 8080
 EXPOSE 443
 

Makefile

@@ -5,16 +5,13 @@ push:
 	docker-compose -f docker-compose.yml push
 
 up-dev:
-	docker-compose -f docker-compose.dev.yml up --build
+	docker compose -f docker-compose.dev.yml up --build
 
-up-pg-dev:
-	docker compose -f docker-compose.pg.yml up --build
-
-i-dev:
-	infisical run -- docker-compose -f docker-compose.dev.yml up --build
+up-dev-ldap:
+	docker compose -f docker-compose.dev.yml --profile ldap up --build
 
 up-prod:
-	docker-compose -f docker-compose.yml up --build
+	docker-compose -f docker-compose.prod.yml up --build
 
 down:
-	docker-compose down
+	docker compose -f docker-compose.dev.yml down

README.md

@@ -10,7 +10,8 @@
   <a href="https://infisical.com/">Infisical Cloud</a> |
   <a href="https://infisical.com/docs/self-hosting/overview">Self-Hosting</a> |
   <a href="https://infisical.com/docs/documentation/getting-started/introduction">Docs</a> |
-  <a href="https://www.infisical.com">Website</a>
+  <a href="https://www.infisical.com">Website</a> |
+  <a href="https://infisical.com/careers">Hiring (Remote/SF)</a>
 </h4>
 
 <p align="center">
@@ -33,7 +34,7 @@
     <img src="https://img.shields.io/github/commit-activity/m/infisical/infisical" alt="git commit activity" />
   </a>
   <a href="https://cloudsmith.io/~infisical/repos/">
-    <img src="https://img.shields.io/badge/Downloads-2.58M-orange" alt="Cloudsmith downloads" />
+    <img src="https://img.shields.io/badge/Downloads-6.95M-orange" alt="Cloudsmith downloads" />
   </a>
   <a href="https://infisical.com/slack">
     <img src="https://img.shields.io/badge/chat-on%20Slack-blueviolet" alt="Slack community channel" />
@@ -53,17 +54,19 @@ We're on a mission to make secret management more accessible to everyone, not ju
 
 ## Features
 
-- **[User-friendly dashboard](https://infisical.com/docs/documentation/platform/project)** to manage secrets across projects and environments (e.g. development, production, etc.)
-- **[Client SDKs](https://infisical.com/docs/sdks/overview)** to fetch secrets for your apps and infrastructure on demand
-- **[Infisical CLI](https://infisical.com/docs/cli/overview)** to fetch and inject secrets into any framework in local development
-- **[Native integrations](https://infisical.com/docs/integrations/overview)** with platforms like GitHub, Vercel, Netlify, and more
-- [**Automatic Kubernetes deployment secret reloads**](https://infisical.com/docs/documentation/getting-started/kubernetes)
-- **[Complete control over your data](https://infisical.com/docs/self-hosting/overview)** - host it yourself on any infrastructure
-- **[Secret versioning](https://infisical.com/docs/documentation/platform/secret-versioning)** and **[Point-in-Time Recovery]()** to version every secret and project state
-- **[Audit logs](https://infisical.com/docs/documentation/platform/audit-logs)** to record every action taken in a project
-- **Role-based Access Controls** per environment
-- [**Simple on-premise deployments** to AWS, Digital Ocean, and more](https://infisical.com/docs/self-hosting/overview)
-- [**Secret Scanning and Leak Prevention**](https://infisical.com/docs/cli/scanning-overview)
+- **[User-friendly dashboard](https://infisical.com/docs/documentation/platform/project)** to manage secrets across projects and environments (e.g. development, production, etc.). 
+- **[Client SDKs](https://infisical.com/docs/sdks/overview)** to fetch secrets for your apps and infrastructure on demand. 
+- **[Infisical CLI](https://infisical.com/docs/cli/overview)** to fetch and inject secrets into any framework in local development and CI/CD. 
+- **[Infisical API](https://infisical.com/docs/api-reference/overview/introduction)** to perform CRUD operation on secrets, users, projects, and any other resource in Infisical. 
+- **[Native integrations](https://infisical.com/docs/integrations/overview)** with platforms like [GitHub](https://infisical.com/docs/integrations/cicd/githubactions), [Vercel](https://infisical.com/docs/integrations/cloud/vercel), [AWS](https://infisical.com/docs/integrations/cloud/aws-secret-manager), and tools like [Terraform](https://infisical.com/docs/integrations/frameworks/terraform), [Ansible](https://infisical.com/docs/integrations/platforms/ansible), and more. 
+- **[Infisical Kubernetes operator](https://infisical.com/docs/documentation/getting-started/kubernetes)** to managed secrets in k8s, automatically reload deployments, and more.
+- **[Infisical Agent](https://infisical.com/docs/infisical-agent/overview)** to inject secrets into your applications without modifying any code logic. 
+- **[Self-hosting and on-prem](https://infisical.com/docs/self-hosting/overview)** to get complete control over your data.
+- **[Secret versioning](https://infisical.com/docs/documentation/platform/secret-versioning)** and **[Point-in-Time Recovery](https://infisical.com/docs/documentation/platform/pit-recovery)** to version every secret and project state. 
+- **[Audit logs](https://infisical.com/docs/documentation/platform/audit-logs)** to record every action taken in a project. 
+- **[Role-based Access Controls](https://infisical.com/docs/documentation/platform/role-based-access-controls)** to create permission sets on any resource in Infisica and assign those to user or machine identities. 
+- **[Simple on-premise deployments](https://infisical.com/docs/self-hosting/overview)** to AWS, Digital Ocean, and more.
+- **[Secret Scanning and Leak Prevention](https://infisical.com/docs/cli/scanning-overview)** to prevent secrets from leaking to git. 
 
 And much more.
 
@@ -73,7 +76,7 @@ Check out the [Quickstart Guides](https://infisical.com/docs/getting-started/int
 
 | Use Infisical Cloud                                                                                                                                     | Deploy Infisical on premise                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | ------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| The fastest and most reliable way to <br> get started with Infisical is signing up <br> for free to [Infisical Cloud](https://app.infisical.com/login). | <a href="https://infisical.com/docs/self-hosting/deployment-options/aws-ec2"><img src=".github/images/deploy-to-aws.png" width="150" width="300" /></a> <a href="https://infisical.com/docs/self-hosting/deployment-options/digital-ocean-marketplace" alt="Deploy to DigitalOcean"> <img width="217" alt="Deploy to DO" src="https://www.deploytodo.com/do-btn-blue.svg"/> </a> <br> View all [deployment options](https://infisical.com/docs/self-hosting/overview) |
+| The fastest and most reliable way to <br> get started with Infisical is signing up <br> for free to [Infisical Cloud](https://app.infisical.com/login). |  <br> View all [deployment options](https://infisical.com/docs/self-hosting/overview) |
 
 ### Run Infisical locally
 
@@ -82,13 +85,13 @@ To set up and run Infisical locally, make sure you have Git and Docker installed
 Linux/macOS:
 
 ```console
-git clone https://github.com/Infisical/infisical && cd "$(basename $_ .git)" && cp .env.example .env && docker-compose -f docker-compose.yml up
+git clone https://github.com/Infisical/infisical && cd "$(basename $_ .git)" && cp .env.example .env && docker-compose -f docker-compose.prod.yml up
 ```
 
 Windows Command Prompt:
 
 ```console
-git clone https://github.com/Infisical/infisical && cd infisical && copy .env.example .env && docker-compose -f docker-compose.yml up
+git clone https://github.com/Infisical/infisical && cd infisical && copy .env.example .env && docker-compose -f docker-compose.prod.yml up
 ```
 
 Create an account at `http://localhost:80`
@@ -115,9 +118,9 @@ Lean about Infisical's code scanning feature [here](https://infisical.com/docs/c
 
 This repo available under the [MIT expat license](https://github.com/Infisical/infisical/blob/main/LICENSE), with the exception of the `ee` directory which will contain premium enterprise features requiring a Infisical license.
 
-If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://cal.com/vmatsiiako/infisical-demo):
+If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://infisical.cal.com/vlad/infisical-demo):
 
-<a href="https://cal.com/vmatsiiako/infisical-demo"><img alt="Schedule a meeting" src="https://cal.com/book-with-cal-dark.svg" /></a>
+<a href="[https://infisical.cal.com/vlad/infisical-demo](https://infisical.cal.com/vlad/infisical-demo)"><img alt="Schedule a meeting" src="https://cal.com/book-with-cal-dark.svg" /></a>
 
 ## Security
 

backend/.eslintignore

@@ -1,2 +1,3 @@
 vitest-environment-infisical.ts
 vitest.config.ts
+vitest.e2e.config.ts

backend/.eslintrc.js

@@ -21,6 +21,19 @@ module.exports = {
     tsconfigRootDir: __dirname
   },
   root: true,
+  overrides: [
+    {
+      files: ["./e2e-test/**/*", "./src/db/migrations/**/*"],
+      rules: {
+        "@typescript-eslint/no-unsafe-member-access": "off",
+        "@typescript-eslint/no-unsafe-assignment": "off",
+        "@typescript-eslint/no-unsafe-argument": "off",
+        "@typescript-eslint/no-unsafe-return": "off",
+        "@typescript-eslint/no-unsafe-call": "off"
+      }
+    }
+  ],
+
   rules: {
     "@typescript-eslint/no-empty-function": "off",
     "@typescript-eslint/no-unsafe-enum-comparison": "off",

backend/e2e-test/mocks/keystore.ts

@@ -0,0 +1,30 @@
+import { TKeyStoreFactory } from "@app/keystore/keystore";
+
+export const mockKeyStore = (): TKeyStoreFactory => {
+  const store: Record<string, string | number | Buffer> = {};
+
+  return {
+    setItem: async (key, value) => {
+      store[key] = value;
+      return "OK";
+    },
+    setItemWithExpiry: async (key, value) => {
+      store[key] = value;
+      return "OK";
+    },
+    deleteItem: async (key) => {
+      delete store[key];
+      return 1;
+    },
+    getItem: async (key) => {
+      const value = store[key];
+      if (typeof value === "string") {
+        return value;
+      }
+      return null;
+    },
+    incrementBy: async () => {
+      return 1;
+    }
+  };
+};

backend/e2e-test/routes/v1/identity.spec.ts

@@ -0,0 +1,71 @@
+import { OrgMembershipRole } from "@app/db/schemas";
+import { seedData1 } from "@app/db/seed-data";
+
+export const createIdentity = async (name: string, role: string) => {
+  const createIdentityRes = await testServer.inject({
+    method: "POST",
+    url: "/api/v1/identities",
+    body: {
+      name,
+      role,
+      organizationId: seedData1.organization.id
+    },
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(createIdentityRes.statusCode).toBe(200);
+  return createIdentityRes.json().identity;
+};
+
+export const deleteIdentity = async (id: string) => {
+  const deleteIdentityRes = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/identities/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(deleteIdentityRes.statusCode).toBe(200);
+  return deleteIdentityRes.json().identity;
+};
+
+describe("Identity v1", async () => {
+  test("Create identity", async () => {
+    const newIdentity = await createIdentity("mac1", OrgMembershipRole.Admin);
+    expect(newIdentity.name).toBe("mac1");
+    expect(newIdentity.authMethod).toBeNull();
+
+    await deleteIdentity(newIdentity.id);
+  });
+
+  test("Update identity", async () => {
+    const newIdentity = await createIdentity("mac1", OrgMembershipRole.Admin);
+    expect(newIdentity.name).toBe("mac1");
+    expect(newIdentity.authMethod).toBeNull();
+
+    const updatedIdentity = await testServer.inject({
+      method: "PATCH",
+      url: `/api/v1/identities/${newIdentity.id}`,
+      headers: {
+        authorization: `Bearer ${jwtAuthToken}`
+      },
+      body: {
+        name: "updated-mac-1",
+        role: OrgMembershipRole.Member
+      }
+    });
+
+    expect(updatedIdentity.statusCode).toBe(200);
+    expect(updatedIdentity.json().identity.name).toBe("updated-mac-1");
+
+    await deleteIdentity(newIdentity.id);
+  });
+
+  test("Delete Identity", async () => {
+    const newIdentity = await createIdentity("mac1", OrgMembershipRole.Admin);
+
+    const deletedIdentity = await deleteIdentity(newIdentity.id);
+    expect(deletedIdentity.name).toBe("mac1");
+  });
+});

backend/e2e-test/routes/v1/login.spec.ts

@@ -1,6 +1,7 @@
-import { seedData1 } from "@app/db/seed-data";
 import jsrp from "jsrp";
 
+import { seedData1 } from "@app/db/seed-data";
+
 describe("Login V1 Router", async () => {
   // eslint-disable-next-line
   const client = new jsrp.client();

backend/e2e-test/routes/v1/project-env.spec.ts

@@ -1,6 +1,40 @@
 import { seedData1 } from "@app/db/seed-data";
 import { DEFAULT_PROJECT_ENVS } from "@app/db/seeds/3-project";
 
+const createProjectEnvironment = async (name: string, slug: string) => {
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/workspace/${seedData1.project.id}/environments`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      name,
+      slug
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("environment");
+  return payload.environment;
+};
+
+const deleteProjectEnvironment = async (envId: string) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/workspace/${seedData1.project.id}/environments/${envId}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("environment");
+  return payload.environment;
+};
+
 describe("Project Environment Router", async () => {
   test("Get default environments", async () => {
     const res = await testServer.inject({
@@ -31,24 +65,10 @@ describe("Project Environment Router", async () => {
     expect(payload.workspace.environments.length).toBe(3);
   });
 
-  const mockProjectEnv = { name: "temp", slug: "temp", id: "" }; // id will be filled in create op
+  const mockProjectEnv = { name: "temp", slug: "temp" }; // id will be filled in create op
   test("Create environment", async () => {
-    const res = await testServer.inject({
-      method: "POST",
-      url: `/api/v1/workspace/${seedData1.project.id}/environments`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        name: mockProjectEnv.name,
-        slug: mockProjectEnv.slug
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("environment");
-    expect(payload.environment).toEqual(
+    const newEnvironment = await createProjectEnvironment(mockProjectEnv.name, mockProjectEnv.slug);
+    expect(newEnvironment).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         name: mockProjectEnv.name,
@@ -59,14 +79,15 @@ describe("Project Environment Router", async () => {
         updatedAt: expect.any(String)
       })
     );
-    mockProjectEnv.id = payload.environment.id;
+    await deleteProjectEnvironment(newEnvironment.id);
   });
 
   test("Update environment", async () => {
+    const newEnvironment = await createProjectEnvironment(mockProjectEnv.name, mockProjectEnv.slug);
     const updatedName = { name: "temp#2", slug: "temp2" };
     const res = await testServer.inject({
       method: "PATCH",
-      url: `/api/v1/workspace/${seedData1.project.id}/environments/${mockProjectEnv.id}`,
+      url: `/api/v1/workspace/${seedData1.project.id}/environments/${newEnvironment.id}`,
       headers: {
         authorization: `Bearer ${jwtAuthToken}`
       },
@@ -82,7 +103,7 @@ describe("Project Environment Router", async () => {
     expect(payload).toHaveProperty("environment");
     expect(payload.environment).toEqual(
       expect.objectContaining({
-        id: expect.any(String),
+        id: newEnvironment.id,
         name: updatedName.name,
         slug: updatedName.slug,
         projectId: seedData1.project.id,
@@ -91,61 +112,21 @@ describe("Project Environment Router", async () => {
         updatedAt: expect.any(String)
       })
     );
-    mockProjectEnv.name = updatedName.name;
-    mockProjectEnv.slug = updatedName.slug;
+    await deleteProjectEnvironment(newEnvironment.id);
   });
 
   test("Delete environment", async () => {
-    const res = await testServer.inject({
-      method: "DELETE",
-      url: `/api/v1/workspace/${seedData1.project.id}/environments/${mockProjectEnv.id}`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("environment");
-    expect(payload.environment).toEqual(
+    const newEnvironment = await createProjectEnvironment(mockProjectEnv.name, mockProjectEnv.slug);
+    const deletedProjectEnvironment = await deleteProjectEnvironment(newEnvironment.id);
+    expect(deletedProjectEnvironment).toEqual(
       expect.objectContaining({
-        id: expect.any(String),
+        id: deletedProjectEnvironment.id,
         name: mockProjectEnv.name,
         slug: mockProjectEnv.slug,
-        position: 1,
+        position: 4,
         createdAt: expect.any(String),
         updatedAt: expect.any(String)
       })
     );
   });
-
-  // after all these opreations the list of environment should be still same
-  test("Default list of environment", async () => {
-    const res = await testServer.inject({
-      method: "GET",
-      url: `/api/v1/workspace/${seedData1.project.id}`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("workspace");
-    // check for default environments
-    expect(payload).toEqual({
-      workspace: expect.objectContaining({
-        name: seedData1.project.name,
-        id: seedData1.project.id,
-        slug: seedData1.project.slug,
-        environments: expect.arrayContaining([
-          expect.objectContaining(DEFAULT_PROJECT_ENVS[0]),
-          expect.objectContaining(DEFAULT_PROJECT_ENVS[1]),
-          expect.objectContaining(DEFAULT_PROJECT_ENVS[2])
-        ])
-      })
-    });
-    // ensure only two default environments exist
-    expect(payload.workspace.environments.length).toBe(3);
-  });
 });

backend/e2e-test/routes/v1/secret-folder.spec.ts

@@ -1,5 +1,40 @@
 import { seedData1 } from "@app/db/seed-data";
 
+const createFolder = async (dto: { path: string; name: string }) => {
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/folders`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      name: dto.name,
+      path: dto.path
+    }
+  });
+  expect(res.statusCode).toBe(200);
+  return res.json().folder;
+};
+
+const deleteFolder = async (dto: { path: string; id: string }) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/folders/${dto.id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      path: dto.path
+    }
+  });
+  expect(res.statusCode).toBe(200);
+  return res.json().folder;
+};
+
 describe("Secret Folder Router", async () => {
   test.each([
     { name: "folder1", path: "/" }, // one in root
@@ -7,30 +42,15 @@ describe("Secret Folder Router", async () => {
     { name: "folder2", path: "/" },
     { name: "folder1", path: "/level1/level2" } // this should not create folder return same thing
   ])("Create folder $name in $path", async ({ name, path }) => {
-    const res = await testServer.inject({
-      method: "POST",
-      url: `/api/v1/folders`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        name,
-        path
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("folder");
+    const createdFolder = await createFolder({ path, name });
     // check for default environments
-    expect(payload).toEqual({
-      folder: expect.objectContaining({
+    expect(createdFolder).toEqual(
+      expect.objectContaining({
         name,
         id: expect.any(String)
       })
-    });
+    );
+    await deleteFolder({ path, id: createdFolder.id });
   });
 
   test.each([
@@ -43,6 +63,8 @@ describe("Secret Folder Router", async () => {
     },
     { path: "/level1/level2", expected: { folders: [{ name: "folder1" }], length: 1 } }
   ])("Get folders $path", async ({ path, expected }) => {
+    const newFolders = await Promise.all(expected.folders.map(({ name }) => createFolder({ name, path })));
+
     const res = await testServer.inject({
       method: "GET",
       url: `/api/v1/folders`,
@@ -59,36 +81,22 @@ describe("Secret Folder Router", async () => {
     expect(res.statusCode).toBe(200);
     const payload = JSON.parse(res.payload);
     expect(payload).toHaveProperty("folders");
-    expect(payload.folders.length).toBe(expected.length);
-    expect(payload).toEqual({ folders: expected.folders.map((el) => expect.objectContaining(el)) });
-  });
-
-  let toBeDeleteFolderId = "";
-  test("Update a deep folder", async () => {
-    const res = await testServer.inject({
-      method: "PATCH",
-      url: `/api/v1/folders/folder1`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        name: "folder-updated",
-        path: "/level1/level2"
-      }
+    expect(payload.folders.length >= expected.folders.length).toBeTruthy();
+    expect(payload).toEqual({
+      folders: expect.arrayContaining(expected.folders.map((el) => expect.objectContaining(el)))
     });
 
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("folder");
-    expect(payload.folder).toEqual(
+    await Promise.all(newFolders.map(({ id }) => deleteFolder({ path, id })));
+  });
+
+  test("Update a deep folder", async () => {
+    const newFolder = await createFolder({ name: "folder-updated", path: "/level1/level2" });
+    expect(newFolder).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         name: "folder-updated"
       })
     );
-    toBeDeleteFolderId = payload.folder.id;
 
     const resUpdatedFolders = await testServer.inject({
       method: "GET",
@@ -106,14 +114,16 @@ describe("Secret Folder Router", async () => {
     expect(resUpdatedFolders.statusCode).toBe(200);
     const updatedFolderList = JSON.parse(resUpdatedFolders.payload);
     expect(updatedFolderList).toHaveProperty("folders");
-    expect(updatedFolderList.folders.length).toEqual(1);
     expect(updatedFolderList.folders[0].name).toEqual("folder-updated");
+
+    await deleteFolder({ path: "/level1/level2", id: newFolder.id });
   });
 
   test("Delete a deep folder", async () => {
+    const newFolder = await createFolder({ name: "folder-updated", path: "/level1/level2" });
     const res = await testServer.inject({
       method: "DELETE",
-      url: `/api/v1/folders/${toBeDeleteFolderId}`,
+      url: `/api/v1/folders/${newFolder.id}`,
       headers: {
         authorization: `Bearer ${jwtAuthToken}`
       },

backend/e2e-test/routes/v1/secret-import.spec.ts

@@ -1,32 +1,57 @@
 import { seedData1 } from "@app/db/seed-data";
 
-describe("Secret Folder Router", async () => {
+const createSecretImport = async (importPath: string, importEnv: string) => {
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/secret-imports`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      path: "/",
+      import: {
+        environment: importEnv,
+        path: importPath
+      }
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("secretImport");
+  return payload.secretImport;
+};
+
+const deleteSecretImport = async (id: string) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/secret-imports/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      path: "/"
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+  const payload = JSON.parse(res.payload);
+  expect(payload).toHaveProperty("secretImport");
+  return payload.secretImport;
+};
+
+describe("Secret Import Router", async () => {
   test.each([
-    { importEnv: "dev", importPath: "/" }, // one in root
+    { importEnv: "prod", importPath: "/" }, // one in root
     { importEnv: "staging", importPath: "/" } // then create a deep one creating intermediate ones
   ])("Create secret import $importEnv with path $importPath", async ({ importPath, importEnv }) => {
-    const res = await testServer.inject({
-      method: "POST",
-      url: `/api/v1/secret-imports`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        path: "/",
-        import: {
-          environment: importEnv,
-          path: importPath
-        }
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("secretImport");
     // check for default environments
-    expect(payload.secretImport).toEqual(
+    const payload = await createSecretImport(importPath, importEnv);
+    expect(payload).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         importPath: expect.any(String),
@@ -37,10 +62,12 @@ describe("Secret Folder Router", async () => {
         })
       })
     );
+    await deleteSecretImport(payload.id);
   });
 
-  let testSecretImportId = "";
   test("Get secret imports", async () => {
+    const createdImport1 = await createSecretImport("/", "prod");
+    const createdImport2 = await createSecretImport("/", "staging");
     const res = await testServer.inject({
       method: "GET",
       url: `/api/v1/secret-imports`,
@@ -58,7 +85,6 @@ describe("Secret Folder Router", async () => {
     const payload = JSON.parse(res.payload);
     expect(payload).toHaveProperty("secretImports");
     expect(payload.secretImports.length).toBe(2);
-    testSecretImportId = payload.secretImports[0].id;
     expect(payload.secretImports).toEqual(
       expect.arrayContaining([
         expect.objectContaining({
@@ -72,12 +98,20 @@ describe("Secret Folder Router", async () => {
         })
       ])
     );
+    await deleteSecretImport(createdImport1.id);
+    await deleteSecretImport(createdImport2.id);
   });
 
   test("Update secret import position", async () => {
-    const res = await testServer.inject({
+    const prodImportDetails = { path: "/", envSlug: "prod" };
+    const stagingImportDetails = { path: "/", envSlug: "staging" };
+
+    const createdImport1 = await createSecretImport(prodImportDetails.path, prodImportDetails.envSlug);
+    const createdImport2 = await createSecretImport(stagingImportDetails.path, stagingImportDetails.envSlug);
+
+    const updateImportRes = await testServer.inject({
       method: "PATCH",
-      url: `/api/v1/secret-imports/${testSecretImportId}`,
+      url: `/api/v1/secret-imports/${createdImport1.id}`,
       headers: {
         authorization: `Bearer ${jwtAuthToken}`
       },
@@ -91,8 +125,8 @@ describe("Secret Folder Router", async () => {
       }
     });
 
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
+    expect(updateImportRes.statusCode).toBe(200);
+    const payload = JSON.parse(updateImportRes.payload);
     expect(payload).toHaveProperty("secretImport");
     // check for default environments
     expect(payload.secretImport).toEqual(
@@ -102,7 +136,7 @@ describe("Secret Folder Router", async () => {
         position: 2,
         importEnv: expect.objectContaining({
           name: expect.any(String),
-          slug: expect.any(String),
+          slug: expect.stringMatching(prodImportDetails.envSlug),
           id: expect.any(String)
         })
       })
@@ -124,28 +158,19 @@ describe("Secret Folder Router", async () => {
     expect(secretImportsListRes.statusCode).toBe(200);
     const secretImportList = JSON.parse(secretImportsListRes.payload);
     expect(secretImportList).toHaveProperty("secretImports");
-    expect(secretImportList.secretImports[1].id).toEqual(testSecretImportId);
+    expect(secretImportList.secretImports[1].id).toEqual(createdImport1.id);
+    expect(secretImportList.secretImports[0].id).toEqual(createdImport2.id);
+
+    await deleteSecretImport(createdImport1.id);
+    await deleteSecretImport(createdImport2.id);
   });
 
   test("Delete secret import position", async () => {
-    const res = await testServer.inject({
-      method: "DELETE",
-      url: `/api/v1/secret-imports/${testSecretImportId}`,
-      headers: {
-        authorization: `Bearer ${jwtAuthToken}`
-      },
-      body: {
-        workspaceId: seedData1.project.id,
-        environment: seedData1.environment.slug,
-        path: "/"
-      }
-    });
-
-    expect(res.statusCode).toBe(200);
-    const payload = JSON.parse(res.payload);
-    expect(payload).toHaveProperty("secretImport");
+    const createdImport1 = await createSecretImport("/", "prod");
+    const createdImport2 = await createSecretImport("/", "staging");
+    const deletedImport = await deleteSecretImport(createdImport1.id);
     // check for default environments
-    expect(payload.secretImport).toEqual(
+    expect(deletedImport).toEqual(
       expect.objectContaining({
         id: expect.any(String),
         importPath: expect.any(String),
@@ -175,5 +200,7 @@ describe("Secret Folder Router", async () => {
     expect(secretImportList).toHaveProperty("secretImports");
     expect(secretImportList.secretImports.length).toEqual(1);
     expect(secretImportList.secretImports[0].position).toEqual(1);
+
+    await deleteSecretImport(createdImport2.id);
   });
 });

backend/e2e-test/routes/v2/service-token.spec.ts

@@ -0,0 +1,579 @@
+import crypto from "node:crypto";
+
+import { SecretType, TSecrets } from "@app/db/schemas";
+import { decryptSecret, encryptSecret, getUserPrivateKey, seedData1 } from "@app/db/seed-data";
+import { decryptAsymmetric, decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+
+const createServiceToken = async (
+  scopes: { environment: string; secretPath: string }[],
+  permissions: ("read" | "write")[]
+) => {
+  const projectKeyRes = await testServer.inject({
+    method: "GET",
+    url: `/api/v2/workspace/${seedData1.project.id}/encrypted-key`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  const projectKeyEnc = JSON.parse(projectKeyRes.payload);
+
+  const userInfoRes = await testServer.inject({
+    method: "GET",
+    url: "/api/v2/users/me",
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  const { user: userInfo } = JSON.parse(userInfoRes.payload);
+  const privateKey = await getUserPrivateKey(seedData1.password, userInfo);
+  const projectKey = decryptAsymmetric({
+    ciphertext: projectKeyEnc.encryptedKey,
+    nonce: projectKeyEnc.nonce,
+    publicKey: projectKeyEnc.sender.publicKey,
+    privateKey
+  });
+
+  const randomBytes = crypto.randomBytes(16).toString("hex");
+  const { ciphertext, iv, tag } = encryptSymmetric128BitHexKeyUTF8(projectKey, randomBytes);
+  const serviceTokenRes = await testServer.inject({
+    method: "POST",
+    url: "/api/v2/service-token",
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: {
+      name: "test-token",
+      workspaceId: seedData1.project.id,
+      scopes,
+      encryptedKey: ciphertext,
+      iv,
+      tag,
+      permissions,
+      expiresIn: null
+    }
+  });
+  expect(serviceTokenRes.statusCode).toBe(200);
+  const serviceTokenInfo = serviceTokenRes.json();
+  expect(serviceTokenInfo).toHaveProperty("serviceToken");
+  expect(serviceTokenInfo).toHaveProperty("serviceTokenData");
+  return `${serviceTokenInfo.serviceToken}.${randomBytes}`;
+};
+
+const deleteServiceToken = async () => {
+  const serviceTokenListRes = await testServer.inject({
+    method: "GET",
+    url: `/api/v1/workspace/${seedData1.project.id}/service-token-data`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(serviceTokenListRes.statusCode).toBe(200);
+  const serviceTokens = JSON.parse(serviceTokenListRes.payload).serviceTokenData as { name: string; id: string }[];
+  expect(serviceTokens.length).toBeGreaterThan(0);
+  const serviceTokenInfo = serviceTokens.find(({ name }) => name === "test-token");
+  expect(serviceTokenInfo).toBeDefined();
+
+  const deleteTokenRes = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v2/service-token/${serviceTokenInfo?.id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+  expect(deleteTokenRes.statusCode).toBe(200);
+};
+
+const createSecret = async (dto: {
+  projectKey: string;
+  path: string;
+  key: string;
+  value: string;
+  comment: string;
+  type?: SecretType;
+  token: string;
+}) => {
+  const createSecretReqBody = {
+    workspaceId: seedData1.project.id,
+    environment: seedData1.environment.slug,
+    type: dto.type || SecretType.Shared,
+    secretPath: dto.path,
+    ...encryptSecret(dto.projectKey, dto.key, dto.value, dto.comment)
+  };
+  const createSecRes = await testServer.inject({
+    method: "POST",
+    url: `/api/v3/secrets/${dto.key}`,
+    headers: {
+      authorization: `Bearer ${dto.token}`
+    },
+    body: createSecretReqBody
+  });
+  expect(createSecRes.statusCode).toBe(200);
+  const createdSecretPayload = JSON.parse(createSecRes.payload);
+  expect(createdSecretPayload).toHaveProperty("secret");
+  return createdSecretPayload.secret;
+};
+
+const deleteSecret = async (dto: { path: string; key: string; token: string }) => {
+  const deleteSecRes = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v3/secrets/${dto.key}`,
+    headers: {
+      authorization: `Bearer ${dto.token}`
+    },
+    body: {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      secretPath: dto.path
+    }
+  });
+  expect(deleteSecRes.statusCode).toBe(200);
+  const updatedSecretPayload = JSON.parse(deleteSecRes.payload);
+  expect(updatedSecretPayload).toHaveProperty("secret");
+  return updatedSecretPayload.secret;
+};
+
+describe("Service token secret ops", async () => {
+  let serviceToken = "";
+  let projectKey = "";
+  let folderId = "";
+  beforeAll(async () => {
+    serviceToken = await createServiceToken(
+      [{ secretPath: "/**", environment: seedData1.environment.slug }],
+      ["read", "write"]
+    );
+
+    // this is ensure cli service token decryptiong working fine
+    const serviceTokenInfoRes = await testServer.inject({
+      method: "GET",
+      url: "/api/v2/service-token",
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(serviceTokenInfoRes.statusCode).toBe(200);
+    const serviceTokenInfo = serviceTokenInfoRes.json();
+    const serviceTokenParts = serviceToken.split(".");
+    projectKey = decryptSymmetric128BitHexKeyUTF8({
+      key: serviceTokenParts[3],
+      tag: serviceTokenInfo.tag,
+      ciphertext: serviceTokenInfo.encryptedKey,
+      iv: serviceTokenInfo.iv
+    });
+
+    // create a deep folder
+    const folderCreate = await testServer.inject({
+      method: "POST",
+      url: `/api/v1/folders`,
+      headers: {
+        authorization: `Bearer ${jwtAuthToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        name: "folder",
+        path: "/nested1/nested2"
+      }
+    });
+    expect(folderCreate.statusCode).toBe(200);
+    folderId = folderCreate.json().folder.id;
+  });
+
+  afterAll(async () => {
+    await deleteServiceToken();
+
+    // create a deep folder
+    const deleteFolder = await testServer.inject({
+      method: "DELETE",
+      url: `/api/v1/folders/${folderId}`,
+      headers: {
+        authorization: `Bearer ${jwtAuthToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        path: "/nested1/nested2"
+      }
+    });
+    expect(deleteFolder.statusCode).toBe(200);
+  });
+
+  const testSecrets = [
+    {
+      path: "/",
+      secret: {
+        key: "ST-SEC",
+        value: "something-secret",
+        comment: "some comment"
+      }
+    },
+    {
+      path: "/nested1/nested2/folder",
+      secret: {
+        key: "NESTED-ST-SEC",
+        value: "something-secret",
+        comment: "some comment"
+      }
+    }
+  ];
+
+  const getSecrets = async (environment: string, secretPath = "/") => {
+    const res = await testServer.inject({
+      method: "GET",
+      url: `/api/v3/secrets`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      query: {
+        secretPath,
+        environment,
+        workspaceId: seedData1.project.id
+      }
+    });
+    const secrets: TSecrets[] = JSON.parse(res.payload).secrets || [];
+    return secrets.map((el) => ({ ...decryptSecret(projectKey, el), type: el.type }));
+  };
+
+  test.each(testSecrets)("Create secret in path $path", async ({ secret, path }) => {
+    const createdSecret = await createSecret({ projectKey, path, ...secret, token: serviceToken });
+    const decryptedSecret = decryptSecret(projectKey, createdSecret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+    expect(decryptedSecret.value).toEqual(secret.value);
+    expect(decryptedSecret.comment).toEqual(secret.comment);
+    expect(decryptedSecret.version).toEqual(1);
+
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          key: secret.key,
+          value: secret.value,
+          type: SecretType.Shared
+        })
+      ])
+    );
+    await deleteSecret({ path, key: secret.key, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Get secret by name in path $path", async ({ secret, path }) => {
+    await createSecret({ projectKey, path, ...secret, token: serviceToken });
+
+    const getSecByNameRes = await testServer.inject({
+      method: "GET",
+      url: `/api/v3/secrets/${secret.key}`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      query: {
+        secretPath: path,
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug
+      }
+    });
+    expect(getSecByNameRes.statusCode).toBe(200);
+    const getSecretByNamePayload = JSON.parse(getSecByNameRes.payload);
+    expect(getSecretByNamePayload).toHaveProperty("secret");
+    const decryptedSecret = decryptSecret(projectKey, getSecretByNamePayload.secret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+    expect(decryptedSecret.value).toEqual(secret.value);
+    expect(decryptedSecret.comment).toEqual(secret.comment);
+
+    await deleteSecret({ path, key: secret.key, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Update secret in path $path", async ({ path, secret }) => {
+    await createSecret({ projectKey, path, ...secret, token: serviceToken });
+    const updateSecretReqBody = {
+      workspaceId: seedData1.project.id,
+      environment: seedData1.environment.slug,
+      type: SecretType.Shared,
+      secretPath: path,
+      ...encryptSecret(projectKey, secret.key, "new-value", secret.comment)
+    };
+    const updateSecRes = await testServer.inject({
+      method: "PATCH",
+      url: `/api/v3/secrets/${secret.key}`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: updateSecretReqBody
+    });
+    expect(updateSecRes.statusCode).toBe(200);
+    const updatedSecretPayload = JSON.parse(updateSecRes.payload);
+    expect(updatedSecretPayload).toHaveProperty("secret");
+    const decryptedSecret = decryptSecret(projectKey, updatedSecretPayload.secret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+    expect(decryptedSecret.value).toEqual("new-value");
+    expect(decryptedSecret.comment).toEqual(secret.comment);
+
+    // list secret should have updated value
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          key: secret.key,
+          value: "new-value",
+          type: SecretType.Shared
+        })
+      ])
+    );
+
+    await deleteSecret({ path, key: secret.key, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Delete secret in path $path", async ({ secret, path }) => {
+    await createSecret({ projectKey, path, ...secret, token: serviceToken });
+    const deletedSecret = await deleteSecret({ path, key: secret.key, token: serviceToken });
+    const decryptedSecret = decryptSecret(projectKey, deletedSecret);
+    expect(decryptedSecret.key).toEqual(secret.key);
+
+    // shared secret deletion should delete personal ones also
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.not.arrayContaining([
+        expect.objectContaining({
+          key: secret.key,
+          type: SecretType.Shared
+        })
+      ])
+    );
+  });
+
+  test.each(testSecrets)("Bulk create secrets in path $path", async ({ secret, path }) => {
+    const createSharedSecRes = await testServer.inject({
+      method: "POST",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`,
+          ...encryptSecret(projectKey, `BULK-${secret.key}-${i + 1}`, secret.value, secret.comment)
+        }))
+      }
+    });
+    expect(createSharedSecRes.statusCode).toBe(200);
+    const createSharedSecPayload = JSON.parse(createSharedSecRes.payload);
+    expect(createSharedSecPayload).toHaveProperty("secrets");
+
+    // bulk ones should exist
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining(
+        Array.from(Array(5)).map((_e, i) =>
+          expect.objectContaining({
+            key: `BULK-${secret.key}-${i + 1}`,
+            type: SecretType.Shared
+          })
+        )
+      )
+    );
+
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        deleteSecret({ path, token: serviceToken, key: `BULK-${secret.key}-${i + 1}` })
+      )
+    );
+  });
+
+  test.each(testSecrets)("Bulk create fail on existing secret in path $path", async ({ secret, path }) => {
+    await createSecret({ projectKey, ...secret, key: `BULK-${secret.key}-1`, path, token: serviceToken });
+
+    const createSharedSecRes = await testServer.inject({
+      method: "POST",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`,
+          ...encryptSecret(projectKey, `BULK-${secret.key}-${i + 1}`, secret.value, secret.comment)
+        }))
+      }
+    });
+    expect(createSharedSecRes.statusCode).toBe(400);
+
+    await deleteSecret({ path, key: `BULK-${secret.key}-1`, token: serviceToken });
+  });
+
+  test.each(testSecrets)("Bulk update secrets in path $path", async ({ secret, path }) => {
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        createSecret({ projectKey, token: serviceToken, ...secret, key: `BULK-${secret.key}-${i + 1}`, path })
+      )
+    );
+
+    const updateSharedSecRes = await testServer.inject({
+      method: "PATCH",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`,
+          ...encryptSecret(projectKey, `BULK-${secret.key}-${i + 1}`, "update-value", secret.comment)
+        }))
+      }
+    });
+    expect(updateSharedSecRes.statusCode).toBe(200);
+    const updateSharedSecPayload = JSON.parse(updateSharedSecRes.payload);
+    expect(updateSharedSecPayload).toHaveProperty("secrets");
+
+    // bulk ones should exist
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.arrayContaining(
+        Array.from(Array(5)).map((_e, i) =>
+          expect.objectContaining({
+            key: `BULK-${secret.key}-${i + 1}`,
+            value: "update-value",
+            type: SecretType.Shared
+          })
+        )
+      )
+    );
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        deleteSecret({ path, key: `BULK-${secret.key}-${i + 1}`, token: serviceToken })
+      )
+    );
+  });
+
+  test.each(testSecrets)("Bulk delete secrets in path $path", async ({ secret, path }) => {
+    await Promise.all(
+      Array.from(Array(5)).map((_e, i) =>
+        createSecret({ projectKey, token: serviceToken, ...secret, key: `BULK-${secret.key}-${i + 1}`, path })
+      )
+    );
+
+    const deletedSharedSecRes = await testServer.inject({
+      method: "DELETE",
+      url: `/api/v3/secrets/batch`,
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      },
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: path,
+        secrets: Array.from(Array(5)).map((_e, i) => ({
+          secretName: `BULK-${secret.key}-${i + 1}`
+        }))
+      }
+    });
+
+    expect(deletedSharedSecRes.statusCode).toBe(200);
+    const deletedSecretPayload = JSON.parse(deletedSharedSecRes.payload);
+    expect(deletedSecretPayload).toHaveProperty("secrets");
+
+    // bulk ones should exist
+    const secrets = await getSecrets(seedData1.environment.slug, path);
+    expect(secrets).toEqual(
+      expect.not.arrayContaining(
+        Array.from(Array(5)).map((_e, i) =>
+          expect.objectContaining({
+            key: `BULK-${secret.value}-${i + 1}`,
+            type: SecretType.Shared
+          })
+        )
+      )
+    );
+  });
+});
+
+describe("Service token fail cases", async () => {
+  test("Unauthorized secret path access", async () => {
+    const serviceToken = await createServiceToken(
+      [{ secretPath: "/", environment: seedData1.environment.slug }],
+      ["read", "write"]
+    );
+    const fetchSecrets = await testServer.inject({
+      method: "GET",
+      url: "/api/v3/secrets",
+      query: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: "/nested/deep"
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(fetchSecrets.statusCode).toBe(401);
+    expect(fetchSecrets.json().error).toBe("PermissionDenied");
+    await deleteServiceToken();
+  });
+
+  test("Unauthorized secret environment access", async () => {
+    const serviceToken = await createServiceToken(
+      [{ secretPath: "/", environment: seedData1.environment.slug }],
+      ["read", "write"]
+    );
+    const fetchSecrets = await testServer.inject({
+      method: "GET",
+      url: "/api/v3/secrets",
+      query: {
+        workspaceId: seedData1.project.id,
+        environment: "prod",
+        secretPath: "/"
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(fetchSecrets.statusCode).toBe(401);
+    expect(fetchSecrets.json().error).toBe("PermissionDenied");
+    await deleteServiceToken();
+  });
+
+  test("Unauthorized write operation", async () => {
+    const serviceToken = await createServiceToken(
+      [{ secretPath: "/", environment: seedData1.environment.slug }],
+      ["read"]
+    );
+    const writeSecrets = await testServer.inject({
+      method: "POST",
+      url: `/api/v3/secrets/NEW`,
+      body: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        type: SecretType.Shared,
+        secretPath: "/",
+        // doesn't matter project key because this will fail before that due to read only access
+        ...encryptSecret(crypto.randomBytes(16).toString("hex"), "NEW", "value", "")
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(writeSecrets.statusCode).toBe(401);
+    expect(writeSecrets.json().error).toBe("PermissionDenied");
+
+    // but read access should still work fine
+    const fetchSecrets = await testServer.inject({
+      method: "GET",
+      url: "/api/v3/secrets",
+      query: {
+        workspaceId: seedData1.project.id,
+        environment: seedData1.environment.slug,
+        secretPath: "/"
+      },
+      headers: {
+        authorization: `Bearer ${serviceToken}`
+      }
+    });
+    expect(fetchSecrets.statusCode).toBe(200);
+    await deleteServiceToken();
+  });
+});

backend/e2e-test/vitest-environment-knex.ts

@@ -1,26 +1,31 @@
-// import { main } from "@app/server/app";
-import { initEnvConfig } from "@app/lib/config/env";
+// eslint-disable-next-line
+import "ts-node/register";
+
 import dotenv from "dotenv";
+import jwt from "jsonwebtoken";
 import knex from "knex";
 import path from "path";
-import { mockSmtpServer } from "./mocks/smtp";
-import { initLogger } from "@app/lib/logger";
-import jwt from "jsonwebtoken";
 
-import "ts-node/register";
-import { main } from "@app/server/app";
-import { mockQueue } from "./mocks/queue";
-import { AuthTokenType } from "@app/services/auth/auth-type";
 import { seedData1 } from "@app/db/seed-data";
+import { initEnvConfig } from "@app/lib/config/env";
+import { initLogger } from "@app/lib/logger";
+import { main } from "@app/server/app";
+import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type";
 
-dotenv.config({ path: path.join(__dirname, "../.env.test") });
+import { mockQueue } from "./mocks/queue";
+import { mockSmtpServer } from "./mocks/smtp";
+import { mockKeyStore } from "./mocks/keystore";
+
+dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true });
 export default {
   name: "knex-env",
   transformMode: "ssr",
   async setup() {
+    const logger = await initLogger();
+    const cfg = initEnvConfig(logger);
     const db = knex({
       client: "pg",
-      connection: process.env.DB_CONNECTION_URI,
+      connection: cfg.DB_CONNECTION_URI,
       migrations: {
         directory: path.join(__dirname, "../src/db/migrations"),
         extension: "ts",
@@ -37,9 +42,8 @@ export default {
       await db.seed.run();
       const smtp = mockSmtpServer();
       const queue = mockQueue();
-      const logger = await initLogger();
-      const cfg = initEnvConfig(logger);
-      const server = await main({ db, smtp, logger, queue });
+      const keyStore = mockKeyStore();
+      const server = await main({ db, smtp, logger, queue, keyStore });
       // @ts-expect-error type
       globalThis.testServer = server;
       // @ts-expect-error type
@@ -48,12 +52,15 @@ export default {
           authTokenType: AuthTokenType.ACCESS_TOKEN,
           userId: seedData1.id,
           tokenVersionId: seedData1.token.id,
+          authMethod: AuthMethod.EMAIL,
+          organizationId: seedData1.organization.id,
           accessVersion: 1
         },
         cfg.AUTH_SECRET,
         { expiresIn: cfg.JWT_AUTH_LIFETIME }
       );
     } catch (error) {
+      console.log("[TEST] Error setting up environment", error);
       await db.destroy();
       throw error;
     }

backend/package.json

@@ -2,12 +2,12 @@
   "name": "backend",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "main": "./dist/main.mjs",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "dev": "tsx watch --clear-screen=false ./src/main.ts  | pino-pretty --colorize --colorizeObjects --singleLine",
     "dev:docker": "nodemon",
-    "build": "rimraf dist && tsup && cp -R ./src/lib/validator/disposable_emails.txt ./dist && cp -R ./src/services/smtp/templates ./dist",
+    "build": "tsup",
     "start": "node dist/main.mjs",
     "type:check": "tsc --noEmit",
     "lint:fix": "eslint --fix --ext js,ts ./src",
@@ -24,8 +24,8 @@
     "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest",
     "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback",
     "seed:new": "tsx ./scripts/create-seed-file.ts",
-    "seed:run": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
-    "db:reset": "npm run migration:rollback -- --all && npm run migration:latest && npm run seed:run"
+    "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
+    "db:reset": "npm run migration:rollback -- --all && npm run migration:latest"
   },
   "keywords": [],
   "author": "",
@@ -44,7 +44,13 @@
     "@types/pg": "^8.10.9",
     "@types/picomatch": "^2.3.3",
     "@types/prompt-sync": "^4.2.3",
+    "@types/resolve": "^1.20.6",
     "@types/uuid": "^9.0.7",
+    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/parser": "^6.20.0",
+    "eslint": "^8.56.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-airbnb-typescript": "^17.1.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-import": "^2.29.1",
@@ -55,72 +61,77 @@
     "prompt-sync": "^4.2.0",
     "rimraf": "^5.0.5",
     "ts-node": "^10.9.1",
+    "tsc-alias": "^1.8.8",
     "tsconfig-paths": "^4.2.0",
     "tsup": "^8.0.1",
     "tsx": "^4.4.0",
     "typescript": "^5.3.2",
     "vite-tsconfig-paths": "^4.2.2",
-    "vitest": "^1.0.4"
+    "vitest": "^1.2.2"
   },
   "dependencies": {
-    "@aws-sdk/client-secrets-manager": "^3.485.0",
+    "@aws-sdk/client-iam": "^3.525.0",
+    "@aws-sdk/client-secrets-manager": "^3.504.0",
     "@casl/ability": "^6.5.0",
-    "@fastify/cookie": "^9.2.0",
-    "@fastify/cors": "^8.4.1",
+    "@fastify/cookie": "^9.3.1",
+    "@fastify/cors": "^8.5.0",
+    "@fastify/etag": "^5.1.0",
     "@fastify/formbody": "^7.4.0",
     "@fastify/helmet": "^11.1.1",
     "@fastify/passport": "^2.4.0",
     "@fastify/rate-limit": "^9.0.0",
     "@fastify/session": "^10.7.0",
-    "@fastify/swagger": "^8.12.0",
-    "@fastify/swagger-ui": "^1.10.1",
+    "@fastify/swagger": "^8.14.0",
+    "@fastify/swagger-ui": "^2.1.0",
     "@node-saml/passport-saml": "^4.0.4",
     "@octokit/rest": "^20.0.2",
     "@octokit/webhooks-types": "^7.3.1",
     "@serdnam/pino-cloudwatch-transport": "^1.0.4",
     "@sindresorhus/slugify": "^2.2.1",
-    "@typescript-eslint/eslint-plugin": "^6.20.0",
-    "@typescript-eslint/parser": "^6.20.0",
     "@ucast/mongo2js": "^1.3.4",
     "ajv": "^8.12.0",
     "argon2": "^0.31.2",
-    "aws-sdk": "^2.1532.0",
-    "axios": "^1.6.2",
+    "aws-sdk": "^2.1553.0",
+    "axios": "^1.6.7",
     "axios-retry": "^4.0.0",
     "bcrypt": "^5.1.1",
-    "bullmq": "^5.1.1",
-    "dotenv": "^16.3.1",
-    "eslint": "^8.56.0",
-    "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-config-airbnb-typescript": "^17.1.0",
-    "fastify": "^4.24.3",
+    "bullmq": "^5.4.2",
+    "cassandra-driver": "^4.7.2",
+    "dotenv": "^16.4.1",
+    "fastify": "^4.26.0",
     "fastify-plugin": "^4.5.1",
+    "google-auth-library": "^9.9.0",
+    "googleapis": "^137.1.0",
     "handlebars": "^4.7.8",
     "ioredis": "^5.3.2",
     "jmespath": "^0.16.0",
     "jsonwebtoken": "^9.0.2",
     "jsrp": "^0.2.4",
     "knex": "^3.0.1",
+    "ldapjs": "^3.0.7",
     "libsodium-wrappers": "^0.7.13",
     "lodash.isequal": "^4.5.0",
-    "mysql2": "^3.6.5",
+    "ms": "^2.1.3",
+    "mysql2": "^3.9.7",
     "nanoid": "^5.0.4",
-    "node-cache": "^5.1.2",
-    "nodemailer": "^6.9.7",
+    "nodemailer": "^6.9.9",
     "ora": "^7.0.1",
+    "oracledb": "^6.4.0",
     "passport-github": "^1.1.0",
     "passport-gitlab2": "^5.0.0",
     "passport-google-oauth20": "^2.0.0",
+    "passport-ldapauth": "^3.0.1",
     "pg": "^8.11.3",
+    "pg-query-stream": "^4.5.3",
     "picomatch": "^3.0.1",
     "pino": "^8.16.2",
-    "posthog-node": "^3.6.0",
-    "probot": "^12.3.3",
+    "posthog-node": "^3.6.2",
+    "probot": "^13.0.0",
     "smee-client": "^2.0.0",
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
     "uuid": "^9.0.1",
     "zod": "^3.22.4",
-    "zod-to-json-schema": "^3.22.0"
+    "zod-to-json-schema": "^3.22.4"
   }
 }

backend/scripts/create-backend-file.ts

@@ -103,11 +103,15 @@ export const ${dalName} = (db: TDbClient) => {
     `import { z } from "zod";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
+import { readLimit } from "@app/server/config/rateLimiter";
 
 export const register${pascalCase}Router = async (server: FastifyZodProvider) => {
   server.route({
-    url: "/",
     method: "GET",
+    url: "/",
+    config: {
+      rateLimit: readLimit
+    },
     schema: {
       params: z.object({}),
       response: {

backend/scripts/create-migration.ts

@@ -7,10 +7,10 @@ const prompt = promptSync({ sigint: true });
 
 const migrationName = prompt("Enter name for migration: ");
 
+// Remove spaces from migration name and replace with hyphens
+const formattedMigrationName = migrationName.replace(/\s+/g, "-");
+
 execSync(
-  `npx knex migrate:make --knexfile ${path.join(
-    __dirname,
-    "../src/db/knexfile.ts"
-  )} -x ts ${migrationName}`,
+  `npx knex migrate:make --knexfile ${path.join(__dirname, "../src/db/knexfile.ts")} -x ts ${formattedMigrationName}`,
   { stdio: "inherit" }
 );

backend/scripts/create-seed-file.ts

@@ -7,11 +7,10 @@ import promptSync from "prompt-sync";
 const prompt = promptSync({ sigint: true });
 
 const migrationName = prompt("Enter name for seedfile: ");
-const fileCounter = readdirSync(path.join(__dirname, "../src/db/seed")).length || 1;
+const fileCounter = readdirSync(path.join(__dirname, "../src/db/seeds")).length || 1;
 execSync(
-  `npx knex seed:make --knexfile ${path.join(
-    __dirname,
-    "../src/db/knexfile.ts"
-  )} -x ts ${fileCounter}-${migrationName}`,
+  `npx knex seed:make --knexfile ${path.join(__dirname, "../src/db/knexfile.ts")} -x ts ${
+    fileCounter + 1
+  }-${migrationName}`,
   { stdio: "inherit" }
 );

backend/scripts/generate-schema-types.ts

@@ -3,13 +3,9 @@ import dotenv from "dotenv";
 import path from "path";
 import knex from "knex";
 import { writeFileSync } from "fs";
-import promptSync from "prompt-sync";
-
-const prompt = promptSync({ sigint: true });
 
 dotenv.config({
-  path: path.join(__dirname, "../.env"),
-  debug: true
+  path: path.join(__dirname, "../../.env.migration")
 });
 
 const db = knex({
@@ -48,7 +44,7 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob
   if (!value || value === "null") return;
   switch (type) {
     case "uuid":
-      return;
+      return `.default("00000000-0000-0000-0000-000000000000")`;
     case "character varying": {
       if (value === "gen_random_uuid()") return;
       if (typeof value === "string" && value.includes("::")) {
@@ -94,17 +90,7 @@ const main = async () => {
       .orderBy("table_name")
   ).filter((el) => !el.tableName.includes("_migrations"));
 
-  console.log("Select a table to generate schema");
-  console.table(tables);
-  console.log("all: all tables");
-  const selectedTables = prompt("Type table numbers comma seperated: ");
-  const tableNumbers =
-    selectedTables !== "all" ? selectedTables.split(",").map((el) => Number(el)) : [];
-
   for (let i = 0; i < tables.length; i += 1) {
-    // skip if not desired table
-    if (selectedTables !== "all" && !tableNumbers.includes(i)) continue;
-
     const { tableName } = tables[i];
     const columns = await db(tableName).columnInfo();
     const columnNames = Object.keys(columns);
@@ -114,7 +100,8 @@ const main = async () => {
       const columnName = columnNames[colNum];
       const colInfo = columns[columnName];
       let ztype = getZodPrimitiveType(colInfo.type);
-      if (colInfo.defaultValue) {
+      // don't put optional on id
+      if (colInfo.defaultValue && columnName !== "id") {
         const { defaultValue } = colInfo;
         const zSchema = getZodDefaultValue(colInfo.type, defaultValue);
         if (zSchema) {
@@ -124,16 +111,17 @@ const main = async () => {
       if (colInfo.nullable) {
         ztype = ztype.concat(".nullable().optional()");
       }
-      schema = schema.concat(`${!schema ? "\n" : ""}  ${columnName}: ${ztype},\n`);
+      schema = schema.concat(
+        `${!schema ? "\n" : ""}  ${columnName}: ${ztype}${colNum === columnNames.length - 1 ? "" : ","}\n`
+      );
     }
 
     const dashcase = tableName.split("_").join("-");
     const pascalCase = tableName
       .split("_")
-      .reduce(
-        (prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`,
-        ""
-      );
+      .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, "");
+
+    // the insert and update are changed to zod input type to use default cases
     writeFileSync(
       path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`),
       `// Code generated by automation script, DO NOT EDIT.
@@ -148,19 +136,10 @@ import { TImmutableDBKeys } from "./models";
 export const ${pascalCase}Schema = z.object({${schema}});
 
 export type T${pascalCase} = z.infer<typeof ${pascalCase}Schema>;
-export type T${pascalCase}Insert = Omit<T${pascalCase}, TImmutableDBKeys>;
-export type T${pascalCase}Update = Partial<Omit<T${pascalCase}, TImmutableDBKeys>>;
+export type T${pascalCase}Insert = Omit<z.input<typeof ${pascalCase}Schema>, TImmutableDBKeys>;
+export type T${pascalCase}Update = Partial<Omit<z.input<typeof ${pascalCase}Schema>, TImmutableDBKeys>>;
 `
     );
-
-    // const file = readFileSync(path.join(__dirname, "../src/db/schemas/index.ts"), "utf8");
-    // if (!file.includes(`export * from "./${dashcase};"`)) {
-    //   appendFileSync(
-    //     path.join(__dirname, "../src/db/schemas/index.ts"),
-    //     `\nexport * from "./${dashcase}";`,
-    //     "utf8"
-    //   );
-    // }
   }
 
   process.exit(0);

backend/src/@types/fastify.d.ts

@@ -1,11 +1,21 @@
 import "fastify";
 
 import { TUsers } from "@app/db/schemas";
+import { TAccessApprovalPolicyServiceFactory } from "@app/ee/services/access-approval-policy/access-approval-policy-service";
+import { TAccessApprovalRequestServiceFactory } from "@app/ee/services/access-approval-request/access-approval-request-service";
 import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
+import { TAuditLogStreamServiceFactory } from "@app/ee/services/audit-log-stream/audit-log-stream-service";
+import { TDynamicSecretServiceFactory } from "@app/ee/services/dynamic-secret/dynamic-secret-service";
+import { TDynamicSecretLeaseServiceFactory } from "@app/ee/services/dynamic-secret-lease/dynamic-secret-lease-service";
+import { TGroupServiceFactory } from "@app/ee/services/group/group-service";
+import { TIdentityProjectAdditionalPrivilegeServiceFactory } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-service";
+import { TLdapConfigServiceFactory } from "@app/ee/services/ldap-config/ldap-config-service";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
+import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service";
 import { TSamlConfigServiceFactory } from "@app/ee/services/saml-config/saml-config-service";
+import { TScimServiceFactory } from "@app/ee/services/scim/scim-service";
 import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
 import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
 import { TSecretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
@@ -17,10 +27,14 @@ import { TApiKeyServiceFactory } from "@app/services/api-key/api-key-service";
 import { TAuthLoginFactory } from "@app/services/auth/auth-login-service";
 import { TAuthPasswordFactory } from "@app/services/auth/auth-password-service";
 import { TAuthSignupFactory } from "@app/services/auth/auth-signup-service";
-import { ActorType } from "@app/services/auth/auth-type";
+import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type";
 import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-service";
+import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service";
 import { TIdentityServiceFactory } from "@app/services/identity/identity-service";
 import { TIdentityAccessTokenServiceFactory } from "@app/services/identity-access-token/identity-access-token-service";
+import { TIdentityAwsAuthServiceFactory } from "@app/services/identity-aws-auth/identity-aws-auth-service";
+import { TIdentityGcpAuthServiceFactory } from "@app/services/identity-gcp-auth/identity-gcp-auth-service";
+import { TIdentityKubernetesAuthServiceFactory } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-service";
 import { TIdentityProjectServiceFactory } from "@app/services/identity-project/identity-project-service";
 import { TIdentityUaServiceFactory } from "@app/services/identity-ua/identity-ua-service";
 import { TIntegrationServiceFactory } from "@app/services/integration/integration-service";
@@ -51,13 +65,16 @@ declare module "fastify" {
     // used for mfa session authentication
     mfa: {
       userId: string;
+      orgId?: string;
       user: TUsers;
     };
     // identity injection. depending on which kinda of token the information is filled in auth
     auth: TAuthMode;
     permission: {
+      authMethod: ActorAuthMethod;
       type: ActorType;
       id: string;
+      orgId: string;
     };
     // passport data
     passportUser: {
@@ -66,6 +83,7 @@ declare module "fastify" {
     };
     auditLogInfo: Pick<TCreateAuditLogDTO, "userAgent" | "userAgentType" | "ipAddress" | "actor">;
     ssoConfig: Awaited<ReturnType<TSamlConfigServiceFactory["getSaml"]>>;
+    ldapConfig: Awaited<ReturnType<TLdapConfigServiceFactory["getLdapCfg"]>>;
   }
 
   interface FastifyInstance {
@@ -79,6 +97,8 @@ declare module "fastify" {
       orgRole: TOrgRoleServiceFactory;
       superAdmin: TSuperAdminServiceFactory;
       user: TUserServiceFactory;
+      group: TGroupServiceFactory;
+      groupProject: TGroupProjectServiceFactory;
       apiKey: TApiKeyServiceFactory;
       project: TProjectServiceFactory;
       projectMembership: TProjectMembershipServiceFactory;
@@ -98,17 +118,29 @@ declare module "fastify" {
       identityAccessToken: TIdentityAccessTokenServiceFactory;
       identityProject: TIdentityProjectServiceFactory;
       identityUa: TIdentityUaServiceFactory;
+      identityKubernetesAuth: TIdentityKubernetesAuthServiceFactory;
+      identityGcpAuth: TIdentityGcpAuthServiceFactory;
+      identityAwsAuth: TIdentityAwsAuthServiceFactory;
+      accessApprovalPolicy: TAccessApprovalPolicyServiceFactory;
+      accessApprovalRequest: TAccessApprovalRequestServiceFactory;
       secretApprovalPolicy: TSecretApprovalPolicyServiceFactory;
       secretApprovalRequest: TSecretApprovalRequestServiceFactory;
       secretRotation: TSecretRotationServiceFactory;
       snapshot: TSecretSnapshotServiceFactory;
       saml: TSamlConfigServiceFactory;
+      scim: TScimServiceFactory;
+      ldap: TLdapConfigServiceFactory;
       auditLog: TAuditLogServiceFactory;
+      auditLogStream: TAuditLogStreamServiceFactory;
       secretScanning: TSecretScanningServiceFactory;
       license: TLicenseServiceFactory;
       trustedIp: TTrustedIpServiceFactory;
       secretBlindIndex: TSecretBlindIndexServiceFactory;
       telemetry: TTelemetryServiceFactory;
+      dynamicSecret: TDynamicSecretServiceFactory;
+      dynamicSecretLease: TDynamicSecretLeaseServiceFactory;
+      projectUserAdditionalPrivilege: TProjectUserAdditionalPrivilegeServiceFactory;
+      identityProjectAdditionalPrivilege: TIdentityProjectAdditionalPrivilegeServiceFactory;
     };
     // this is exclusive use for middlewares in which we need to inject data
     // everywhere else access using service layer

backend/src/@types/knex.d.ts

@@ -2,11 +2,26 @@ import { Knex } from "knex";
 
 import {
   TableName,
+  TAccessApprovalPolicies,
+  TAccessApprovalPoliciesApprovers,
+  TAccessApprovalPoliciesApproversInsert,
+  TAccessApprovalPoliciesApproversUpdate,
+  TAccessApprovalPoliciesInsert,
+  TAccessApprovalPoliciesUpdate,
+  TAccessApprovalRequests,
+  TAccessApprovalRequestsInsert,
+  TAccessApprovalRequestsReviewers,
+  TAccessApprovalRequestsReviewersInsert,
+  TAccessApprovalRequestsReviewersUpdate,
+  TAccessApprovalRequestsUpdate,
   TApiKeys,
   TApiKeysInsert,
   TApiKeysUpdate,
   TAuditLogs,
   TAuditLogsInsert,
+  TAuditLogStreams,
+  TAuditLogStreamsInsert,
+  TAuditLogStreamsUpdate,
   TAuditLogsUpdate,
   TAuthTokens,
   TAuthTokenSessions,
@@ -17,21 +32,51 @@ import {
   TBackupPrivateKey,
   TBackupPrivateKeyInsert,
   TBackupPrivateKeyUpdate,
+  TDynamicSecretLeases,
+  TDynamicSecretLeasesInsert,
+  TDynamicSecretLeasesUpdate,
+  TDynamicSecrets,
+  TDynamicSecretsInsert,
+  TDynamicSecretsUpdate,
   TGitAppInstallSessions,
   TGitAppInstallSessionsInsert,
   TGitAppInstallSessionsUpdate,
   TGitAppOrg,
   TGitAppOrgInsert,
   TGitAppOrgUpdate,
+  TGroupProjectMembershipRoles,
+  TGroupProjectMembershipRolesInsert,
+  TGroupProjectMembershipRolesUpdate,
+  TGroupProjectMemberships,
+  TGroupProjectMembershipsInsert,
+  TGroupProjectMembershipsUpdate,
+  TGroups,
+  TGroupsInsert,
+  TGroupsUpdate,
   TIdentities,
   TIdentitiesInsert,
   TIdentitiesUpdate,
   TIdentityAccessTokens,
   TIdentityAccessTokensInsert,
   TIdentityAccessTokensUpdate,
+  TIdentityAwsAuths,
+  TIdentityAwsAuthsInsert,
+  TIdentityAwsAuthsUpdate,
+  TIdentityGcpAuths,
+  TIdentityGcpAuthsInsert,
+  TIdentityGcpAuthsUpdate,
+  TIdentityKubernetesAuths,
+  TIdentityKubernetesAuthsInsert,
+  TIdentityKubernetesAuthsUpdate,
   TIdentityOrgMemberships,
   TIdentityOrgMembershipsInsert,
   TIdentityOrgMembershipsUpdate,
+  TIdentityProjectAdditionalPrivilege,
+  TIdentityProjectAdditionalPrivilegeInsert,
+  TIdentityProjectAdditionalPrivilegeUpdate,
+  TIdentityProjectMembershipRole,
+  TIdentityProjectMembershipRoleInsert,
+  TIdentityProjectMembershipRoleUpdate,
   TIdentityProjectMemberships,
   TIdentityProjectMembershipsInsert,
   TIdentityProjectMembershipsUpdate,
@@ -50,6 +95,12 @@ import {
   TIntegrations,
   TIntegrationsInsert,
   TIntegrationsUpdate,
+  TLdapConfigs,
+  TLdapConfigsInsert,
+  TLdapConfigsUpdate,
+  TLdapGroupMaps,
+  TLdapGroupMapsInsert,
+  TLdapGroupMapsUpdate,
   TOrganizations,
   TOrganizationsInsert,
   TOrganizationsUpdate,
@@ -80,9 +131,18 @@ import {
   TProjects,
   TProjectsInsert,
   TProjectsUpdate,
+  TProjectUserAdditionalPrivilege,
+  TProjectUserAdditionalPrivilegeInsert,
+  TProjectUserAdditionalPrivilegeUpdate,
+  TProjectUserMembershipRoles,
+  TProjectUserMembershipRolesInsert,
+  TProjectUserMembershipRolesUpdate,
   TSamlConfigs,
   TSamlConfigsInsert,
   TSamlConfigsUpdate,
+  TScimTokens,
+  TScimTokensInsert,
+  TScimTokensUpdate,
   TSecretApprovalPolicies,
   TSecretApprovalPoliciesApprovers,
   TSecretApprovalPoliciesApproversInsert,
@@ -158,9 +218,15 @@ import {
   TUserActions,
   TUserActionsInsert,
   TUserActionsUpdate,
+  TUserAliases,
+  TUserAliasesInsert,
+  TUserAliasesUpdate,
   TUserEncryptionKeys,
   TUserEncryptionKeysInsert,
   TUserEncryptionKeysUpdate,
+  TUserGroupMembership,
+  TUserGroupMembershipInsert,
+  TUserGroupMembershipUpdate,
   TUsers,
   TUsersInsert,
   TUsersUpdate,
@@ -168,10 +234,28 @@ import {
   TWebhooksInsert,
   TWebhooksUpdate
 } from "@app/db/schemas";
+import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references";
 
 declare module "knex/types/tables" {
   interface Tables {
     [TableName.Users]: Knex.CompositeTableType<TUsers, TUsersInsert, TUsersUpdate>;
+    [TableName.Groups]: Knex.CompositeTableType<TGroups, TGroupsInsert, TGroupsUpdate>;
+    [TableName.UserGroupMembership]: Knex.CompositeTableType<
+      TUserGroupMembership,
+      TUserGroupMembershipInsert,
+      TUserGroupMembershipUpdate
+    >;
+    [TableName.GroupProjectMembership]: Knex.CompositeTableType<
+      TGroupProjectMemberships,
+      TGroupProjectMembershipsInsert,
+      TGroupProjectMembershipsUpdate
+    >;
+    [TableName.GroupProjectMembershipRole]: Knex.CompositeTableType<
+      TGroupProjectMembershipRoles,
+      TGroupProjectMembershipRolesInsert,
+      TGroupProjectMembershipRolesUpdate
+    >;
+    [TableName.UserAliases]: Knex.CompositeTableType<TUserAliases, TUserAliasesInsert, TUserAliasesUpdate>;
     [TableName.UserEncryptionKey]: Knex.CompositeTableType<
       TUserEncryptionKeys,
       TUserEncryptionKeysInsert,
@@ -211,9 +295,24 @@ declare module "knex/types/tables" {
       TProjectEnvironmentsUpdate
     >;
     [TableName.ProjectBot]: Knex.CompositeTableType<TProjectBots, TProjectBotsInsert, TProjectBotsUpdate>;
+    [TableName.ProjectUserMembershipRole]: Knex.CompositeTableType<
+      TProjectUserMembershipRoles,
+      TProjectUserMembershipRolesInsert,
+      TProjectUserMembershipRolesUpdate
+    >;
     [TableName.ProjectRoles]: Knex.CompositeTableType<TProjectRoles, TProjectRolesInsert, TProjectRolesUpdate>;
+    [TableName.ProjectUserAdditionalPrivilege]: Knex.CompositeTableType<
+      TProjectUserAdditionalPrivilege,
+      TProjectUserAdditionalPrivilegeInsert,
+      TProjectUserAdditionalPrivilegeUpdate
+    >;
     [TableName.ProjectKeys]: Knex.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
     [TableName.Secret]: Knex.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
+    [TableName.SecretReference]: Knex.CompositeTableType<
+      TSecretReferences,
+      TSecretReferencesInsert,
+      TSecretReferencesUpdate
+    >;
     [TableName.SecretBlindIndex]: Knex.CompositeTableType<
       TSecretBlindIndexes,
       TSecretBlindIndexesInsert,
@@ -242,6 +341,21 @@ declare module "knex/types/tables" {
       TIdentityUniversalAuthsInsert,
       TIdentityUniversalAuthsUpdate
     >;
+    [TableName.IdentityKubernetesAuth]: Knex.CompositeTableType<
+      TIdentityKubernetesAuths,
+      TIdentityKubernetesAuthsInsert,
+      TIdentityKubernetesAuthsUpdate
+    >;
+    [TableName.IdentityGcpAuth]: Knex.CompositeTableType<
+      TIdentityGcpAuths,
+      TIdentityGcpAuthsInsert,
+      TIdentityGcpAuthsUpdate
+    >;
+    [TableName.IdentityAwsAuth]: Knex.CompositeTableType<
+      TIdentityAwsAuths,
+      TIdentityAwsAuthsInsert,
+      TIdentityAwsAuthsUpdate
+    >;
     [TableName.IdentityUaClientSecret]: Knex.CompositeTableType<
       TIdentityUaClientSecrets,
       TIdentityUaClientSecretsInsert,
@@ -262,6 +376,42 @@ declare module "knex/types/tables" {
       TIdentityProjectMembershipsInsert,
       TIdentityProjectMembershipsUpdate
     >;
+    [TableName.IdentityProjectMembershipRole]: Knex.CompositeTableType<
+      TIdentityProjectMembershipRole,
+      TIdentityProjectMembershipRoleInsert,
+      TIdentityProjectMembershipRoleUpdate
+    >;
+    [TableName.IdentityProjectAdditionalPrivilege]: Knex.CompositeTableType<
+      TIdentityProjectAdditionalPrivilege,
+      TIdentityProjectAdditionalPrivilegeInsert,
+      TIdentityProjectAdditionalPrivilegeUpdate
+    >;
+
+    [TableName.AccessApprovalPolicy]: Knex.CompositeTableType<
+      TAccessApprovalPolicies,
+      TAccessApprovalPoliciesInsert,
+      TAccessApprovalPoliciesUpdate
+    >;
+
+    [TableName.AccessApprovalPolicyApprover]: Knex.CompositeTableType<
+      TAccessApprovalPoliciesApprovers,
+      TAccessApprovalPoliciesApproversInsert,
+      TAccessApprovalPoliciesApproversUpdate
+    >;
+
+    [TableName.AccessApprovalRequest]: Knex.CompositeTableType<
+      TAccessApprovalRequests,
+      TAccessApprovalRequestsInsert,
+      TAccessApprovalRequestsUpdate
+    >;
+
+    [TableName.AccessApprovalRequestReviewer]: Knex.CompositeTableType<
+      TAccessApprovalRequestsReviewers,
+      TAccessApprovalRequestsReviewersInsert,
+      TAccessApprovalRequestsReviewersUpdate
+    >;
+
+    [TableName.ScimToken]: Knex.CompositeTableType<TScimTokens, TScimTokensInsert, TScimTokensUpdate>;
     [TableName.SecretApprovalPolicy]: Knex.CompositeTableType<
       TSecretApprovalPolicies,
       TSecretApprovalPoliciesInsert,
@@ -313,9 +463,22 @@ declare module "knex/types/tables" {
       TSecretSnapshotFoldersInsert,
       TSecretSnapshotFoldersUpdate
     >;
+    [TableName.DynamicSecret]: Knex.CompositeTableType<TDynamicSecrets, TDynamicSecretsInsert, TDynamicSecretsUpdate>;
+    [TableName.DynamicSecretLease]: Knex.CompositeTableType<
+      TDynamicSecretLeases,
+      TDynamicSecretLeasesInsert,
+      TDynamicSecretLeasesUpdate
+    >;
     [TableName.SamlConfig]: Knex.CompositeTableType<TSamlConfigs, TSamlConfigsInsert, TSamlConfigsUpdate>;
+    [TableName.LdapConfig]: Knex.CompositeTableType<TLdapConfigs, TLdapConfigsInsert, TLdapConfigsUpdate>;
+    [TableName.LdapGroupMap]: Knex.CompositeTableType<TLdapGroupMaps, TLdapGroupMapsInsert, TLdapGroupMapsUpdate>;
     [TableName.OrgBot]: Knex.CompositeTableType<TOrgBots, TOrgBotsInsert, TOrgBotsUpdate>;
     [TableName.AuditLog]: Knex.CompositeTableType<TAuditLogs, TAuditLogsInsert, TAuditLogsUpdate>;
+    [TableName.AuditLogStream]: Knex.CompositeTableType<
+      TAuditLogStreams,
+      TAuditLogStreamsInsert,
+      TAuditLogStreamsUpdate
+    >;
     [TableName.GitAppInstallSession]: Knex.CompositeTableType<
       TGitAppInstallSessions,
       TGitAppInstallSessionsInsert,

backend/src/cache/redis.ts

@@ -1,6 +0,0 @@
-import Redis from "ioredis";
-
-export const initRedisConnection = (redisUrl: string) => {
-  const redis = new Redis(redisUrl);
-  return redis;
-};

backend/src/db/instance.ts

@@ -1,10 +1,25 @@
 import knex from "knex";
 
 export type TDbClient = ReturnType<typeof initDbConnection>;
-export const initDbConnection = (dbConnectionUri: string) => {
+export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnectionUri: string; dbRootCert?: string }) => {
   const db = knex({
     client: "pg",
-    connection: dbConnectionUri
+    connection: {
+      connectionString: dbConnectionUri,
+      host: process.env.DB_HOST,
+      // @ts-expect-error I have no clue why only for the port there is a type error
+      // eslint-disable-next-line
+      port: process.env.DB_PORT,
+      user: process.env.DB_USER,
+      database: process.env.DB_NAME,
+      password: process.env.DB_PASSWORD,
+      ssl: dbRootCert
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(dbRootCert, "base64").toString("ascii")
+          }
+        : false
+    }
   });
 
   return db;

backend/src/db/knexfile.ts

@@ -5,15 +5,31 @@ import dotenv from "dotenv";
 import type { Knex } from "knex";
 import path from "path";
 
-// Update with your config settings.
+// Update with your config settings. .
 dotenv.config({
-  path: path.join(__dirname, "../../.env"),
-  debug: true
+  path: path.join(__dirname, "../../../.env.migration")
 });
+dotenv.config({
+  path: path.join(__dirname, "../../../.env")
+});
+
 export default {
   development: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      host: process.env.DB_HOST,
+      port: process.env.DB_PORT,
+      user: process.env.DB_USER,
+      database: process.env.DB_NAME,
+      password: process.env.DB_PASSWORD,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10
@@ -27,7 +43,20 @@ export default {
   },
   production: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      host: process.env.DB_HOST,
+      port: process.env.DB_PORT,
+      user: process.env.DB_USER,
+      database: process.env.DB_NAME,
+      password: process.env.DB_PASSWORD,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10

backend/src/db/migrations/20240204171758_org-based-auth.ts

@@ -0,0 +1,25 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.boolean("authEnforced").defaultTo(false);
+    t.index("slug");
+  });
+
+  await knex.schema.alterTable(TableName.SamlConfig, (t) => {
+    t.datetime("lastUsed");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.dropColumn("authEnforced");
+    t.dropIndex("slug");
+  });
+
+  await knex.schema.alterTable(TableName.SamlConfig, (t) => {
+    t.dropColumn("lastUsed");
+  });
+}

backend/src/db/migrations/20240208234120_scim-token.ts

@@ -0,0 +1,31 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.ScimToken))) {
+    await knex.schema.createTable(TableName.ScimToken, (t) => {
+      t.string("id", 36).primary().defaultTo(knex.fn.uuid());
+      t.bigInteger("ttlDays").defaultTo(365).notNullable();
+      t.string("description").notNullable();
+      t.uuid("orgId").notNullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.boolean("scimEnabled").defaultTo(false);
+  });
+
+  await createOnUpdateTrigger(knex, TableName.ScimToken);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.ScimToken);
+  await dropOnUpdateTrigger(knex, TableName.ScimToken);
+  await knex.schema.alterTable(TableName.Organization, (t) => {
+    t.dropColumn("scimEnabled");
+  });
+}

backend/src/db/migrations/20240216154123_ghost_users.ts

@@ -0,0 +1,39 @@
+import { Knex } from "knex";
+
+import { ProjectVersion, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (!hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.boolean("isGhost").defaultTo(false).notNullable();
+    });
+  }
+
+  if (!hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.integer("version").defaultTo(ProjectVersion.V1).notNullable();
+      t.string("upgradeStatus").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.dropColumn("isGhost");
+    });
+  }
+
+  if (hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.dropColumn("version");
+      t.dropColumn("upgradeStatus");
+    });
+  }
+}

backend/src/db/migrations/20240222201806_admin-signup-control.ts

@@ -0,0 +1,20 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const isTablePresent = await knex.schema.hasTable(TableName.SuperAdmin);
+  if (isTablePresent) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.string("allowedSignUpDomain");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.SuperAdmin, "allowedSignUpDomain")) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.dropColumn("allowedSignUpDomain");
+    });
+  }
+}

backend/src/db/migrations/20240226094411_instance-id.ts

@@ -0,0 +1,25 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const ADMIN_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+    t.uuid("instanceId").notNullable().defaultTo(knex.fn.uuid());
+  });
+
+  const superUserConfigExists = await knex(TableName.SuperAdmin).where("id", ADMIN_CONFIG_UUID).first();
+  if (!superUserConfigExists) {
+    // eslint-disable-next-line
+    await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+    t.dropColumn("instanceId");
+  });
+}

backend/src/db/migrations/20240307232900_integration-last-used.ts

@@ -0,0 +1,15 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    t.datetime("lastUsed");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    t.dropColumn("lastUsed");
+  });
+}

backend/src/db/migrations/20240311210135_ldap-config.ts

@@ -0,0 +1,68 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.LdapConfig))) {
+    await knex.schema.createTable(TableName.LdapConfig, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("orgId").notNullable().unique();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.boolean("isActive").notNullable();
+      t.string("url").notNullable();
+      t.string("encryptedBindDN").notNullable();
+      t.string("bindDNIV").notNullable();
+      t.string("bindDNTag").notNullable();
+      t.string("encryptedBindPass").notNullable();
+      t.string("bindPassIV").notNullable();
+      t.string("bindPassTag").notNullable();
+      t.string("searchBase").notNullable();
+      t.text("encryptedCACert").notNullable();
+      t.string("caCertIV").notNullable();
+      t.string("caCertTag").notNullable();
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.LdapConfig);
+
+  if (!(await knex.schema.hasTable(TableName.UserAliases))) {
+    await knex.schema.createTable(TableName.UserAliases, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("userId").notNullable();
+      t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE");
+      t.string("username").notNullable();
+      t.string("aliasType").notNullable();
+      t.string("externalId").notNullable();
+      t.specificType("emails", "text[]");
+      t.uuid("orgId").nullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.UserAliases);
+
+  await knex.schema.alterTable(TableName.Users, (t) => {
+    t.string("username").unique();
+    t.string("email").nullable().alter();
+    t.dropUnique(["email"]);
+  });
+
+  await knex(TableName.Users).update("username", knex.ref("email"));
+
+  await knex.schema.alterTable(TableName.Users, (t) => {
+    t.string("username").notNullable().alter();
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.LdapConfig);
+  await knex.schema.dropTableIfExists(TableName.UserAliases);
+  await knex.schema.alterTable(TableName.Users, (t) => {
+    t.dropColumn("username");
+    // t.string("email").notNullable().alter();
+  });
+  await dropOnUpdateTrigger(knex, TableName.LdapConfig);
+}

backend/src/db/migrations/20240312162549_temp-roles.ts

@@ -0,0 +1,50 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesTableExist = await knex.schema.hasTable(TableName.ProjectUserMembershipRole);
+  if (!doesTableExist) {
+    await knex.schema.createTable(TableName.ProjectUserMembershipRole, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("role").notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+      // until role is changed/removed the role should not deleted
+      t.uuid("customRoleId");
+      t.foreign("customRoleId").references("id").inTable(TableName.ProjectRoles);
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.ProjectUserMembershipRole);
+
+  const projectMemberships = await knex(TableName.ProjectMembership).select(
+    "id",
+    "role",
+    "createdAt",
+    "updatedAt",
+    knex.ref("roleId").withSchema(TableName.ProjectMembership).as("customRoleId")
+  );
+  if (projectMemberships.length)
+    await knex.batchInsert(
+      TableName.ProjectUserMembershipRole,
+      projectMemberships.map((data) => ({ ...data, projectMembershipId: data.id }))
+    );
+  // will be dropped later
+  // await knex.schema.alterTable(TableName.ProjectMembership, (t) => {
+  //   t.dropColumn("roleId");
+  //   t.dropColumn("role");
+  // });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.ProjectUserMembershipRole);
+  await dropOnUpdateTrigger(knex, TableName.ProjectUserMembershipRole);
+}

backend/src/db/migrations/20240312162556_temp-role-identity.ts

@@ -0,0 +1,52 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesTableExist = await knex.schema.hasTable(TableName.IdentityProjectMembershipRole);
+  if (!doesTableExist) {
+    await knex.schema.createTable(TableName.IdentityProjectMembershipRole, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("role").notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId")
+        .references("id")
+        .inTable(TableName.IdentityProjectMembership)
+        .onDelete("CASCADE");
+      // until role is changed/removed the role should not deleted
+      t.uuid("customRoleId");
+      t.foreign("customRoleId").references("id").inTable(TableName.ProjectRoles);
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityProjectMembershipRole);
+
+  const identityMemberships = await knex(TableName.IdentityProjectMembership).select(
+    "id",
+    "role",
+    "createdAt",
+    "updatedAt",
+    knex.ref("roleId").withSchema(TableName.IdentityProjectMembership).as("customRoleId")
+  );
+  if (identityMemberships.length)
+    await knex.batchInsert(
+      TableName.IdentityProjectMembershipRole,
+      identityMemberships.map((data) => ({ ...data, projectMembershipId: data.id }))
+    );
+  // await knex.schema.alterTable(TableName.IdentityProjectMembership, (t) => {
+  //   t.dropColumn("roleId");
+  //   t.dropColumn("role");
+  // });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityProjectMembershipRole);
+  await dropOnUpdateTrigger(knex, TableName.IdentityProjectMembershipRole);
+}

backend/src/db/migrations/20240318164718_dynamic-secret.ts

@@ -0,0 +1,58 @@
+import { Knex } from "knex";
+
+import { SecretEncryptionAlgo, SecretKeyEncoding, TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesTableExist = await knex.schema.hasTable(TableName.DynamicSecret);
+  if (!doesTableExist) {
+    await knex.schema.createTable(TableName.DynamicSecret, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("name").notNullable();
+      t.integer("version").notNullable();
+      t.string("type").notNullable();
+      t.string("defaultTTL").notNullable();
+      t.string("maxTTL");
+      t.string("inputIV").notNullable();
+      t.text("inputCiphertext").notNullable();
+      t.string("inputTag").notNullable();
+      t.string("algorithm").notNullable().defaultTo(SecretEncryptionAlgo.AES_256_GCM);
+      t.string("keyEncoding").notNullable().defaultTo(SecretKeyEncoding.UTF8);
+      t.uuid("folderId").notNullable();
+      // for background process communication
+      t.string("status");
+      t.string("statusDetails");
+      t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE");
+      t.unique(["name", "folderId"]);
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.DynamicSecret);
+
+  const doesTableDynamicSecretLease = await knex.schema.hasTable(TableName.DynamicSecretLease);
+  if (!doesTableDynamicSecretLease) {
+    await knex.schema.createTable(TableName.DynamicSecretLease, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.integer("version").notNullable();
+      t.string("externalEntityId").notNullable();
+      t.datetime("expireAt").notNullable();
+      // for background process communication
+      t.string("status");
+      t.string("statusDetails");
+      t.uuid("dynamicSecretId").notNullable();
+      t.foreign("dynamicSecretId").references("id").inTable(TableName.DynamicSecret).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.DynamicSecretLease);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.DynamicSecretLease);
+  await knex.schema.dropTableIfExists(TableName.DynamicSecretLease);
+
+  await dropOnUpdateTrigger(knex, TableName.DynamicSecret);
+  await knex.schema.dropTableIfExists(TableName.DynamicSecret);
+}

backend/src/db/migrations/20240326172010_project-user-additional-privilege.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.ProjectUserAdditionalPrivilege))) {
+    await knex.schema.createTable(TableName.ProjectUserAdditionalPrivilege, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("slug", 60).notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.jsonb("permissions").notNullable();
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.ProjectUserAdditionalPrivilege);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.ProjectUserAdditionalPrivilege);
+  await knex.schema.dropTableIfExists(TableName.ProjectUserAdditionalPrivilege);
+}

backend/src/db/migrations/20240326172011_machine-identity-additional-privilege.ts

@@ -0,0 +1,32 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityProjectAdditionalPrivilege))) {
+    await knex.schema.createTable(TableName.IdentityProjectAdditionalPrivilege, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("slug", 60).notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId")
+        .references("id")
+        .inTable(TableName.IdentityProjectMembership)
+        .onDelete("CASCADE");
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.jsonb("permissions").notNullable();
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityProjectAdditionalPrivilege);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.IdentityProjectAdditionalPrivilege);
+  await knex.schema.dropTableIfExists(TableName.IdentityProjectAdditionalPrivilege);
+}

backend/src/db/migrations/20240405000045_org-memberships-unique-constraint.ts

@@ -0,0 +1,112 @@
+import { Knex } from "knex";
+import { z } from "zod";
+
+import { TableName, TOrgMemberships } from "../schemas";
+
+const validateOrgMembership = (membershipToValidate: TOrgMemberships, firstMembership: TOrgMemberships) => {
+  const firstOrgId = firstMembership.orgId;
+  const firstUserId = firstMembership.userId;
+
+  if (membershipToValidate.id === firstMembership.id) {
+    return;
+  }
+
+  if (membershipToValidate.inviteEmail !== firstMembership.inviteEmail) {
+    throw new Error(`Invite emails are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.orgId !== firstMembership.orgId) {
+    throw new Error(`OrgIds are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.role !== firstMembership.role) {
+    throw new Error(`Roles are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.roleId !== firstMembership.roleId) {
+    throw new Error(`RoleIds are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.status !== firstMembership.status) {
+    throw new Error(`Statuses are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+  if (membershipToValidate.userId !== firstMembership.userId) {
+    throw new Error(`UserIds are different for the same userId and orgId: ${firstUserId}, ${firstOrgId}`);
+  }
+};
+
+export async function up(knex: Knex): Promise<void> {
+  const RowSchema = z.object({
+    userId: z.string(),
+    orgId: z.string(),
+    cnt: z.string()
+  });
+
+  // Transactional find and delete duplicate rows
+  await knex.transaction(async (tx) => {
+    const duplicateRows = await tx(TableName.OrgMembership)
+      .select("userId", "orgId") // Select the userId and orgId so we can group by them
+      .whereNotNull("userId") // Ensure that the userId is not null
+      .count("* as cnt") // Count the number of rows for each userId and orgId, so we can make sure there are more than 1 row (a duplicate)
+      .groupBy("userId", "orgId")
+      .havingRaw("count(*) > ?", [1]); // Using havingRaw for direct SQL expressions
+
+    // Parse the rows to ensure they are in the correct format, and for type safety
+    const parsedRows = RowSchema.array().parse(duplicateRows);
+
+    // For each of the duplicate rows, loop through and find the actual memberships to delete
+    for (const row of parsedRows) {
+      const count = Number(row.cnt);
+
+      // An extra check to ensure that the count is actually a number, and the number is greater than 2
+      if (typeof count !== "number" || count < 2) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      // Find all the organization memberships that have the same userId and orgId
+      // eslint-disable-next-line no-await-in-loop
+      const rowsToDelete = await tx(TableName.OrgMembership).where({
+        userId: row.userId,
+        orgId: row.orgId
+      });
+
+      // Ensure that all the rows have exactly the same value, except id, createdAt, updatedAt
+      for (const rowToDelete of rowsToDelete) {
+        validateOrgMembership(rowToDelete, rowsToDelete[0]);
+      }
+
+      // Find the row with the latest createdAt, which we will keep
+
+      let lowestCreatedAt: number | null = null;
+      let latestCreatedRow: TOrgMemberships | null = null;
+
+      for (const rowToDelete of rowsToDelete) {
+        if (lowestCreatedAt === null || rowToDelete.createdAt.getTime() < lowestCreatedAt) {
+          lowestCreatedAt = rowToDelete.createdAt.getTime();
+          latestCreatedRow = rowToDelete;
+        }
+      }
+      if (!latestCreatedRow) {
+        throw new Error("Failed to find last created membership");
+      }
+
+      // Filter out the latest row from the rows to delete
+      const membershipIdsToDelete = rowsToDelete.map((r) => r.id).filter((id) => id !== latestCreatedRow!.id);
+
+      // eslint-disable-next-line no-await-in-loop
+      const numberOfRowsDeleted = await tx(TableName.OrgMembership).whereIn("id", membershipIdsToDelete).delete();
+
+      // eslint-disable-next-line no-console
+      console.log(
+        `Deleted ${numberOfRowsDeleted} duplicate organization memberships for ${row.userId} and ${row.orgId}`
+      );
+    }
+  });
+
+  await knex.schema.alterTable(TableName.OrgMembership, (table) => {
+    table.unique(["userId", "orgId"]);
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.OrgMembership, (table) => {
+    table.dropUnique(["userId", "orgId"]);
+  });
+}

backend/src/db/migrations/20240412174842_group.ts

@@ -0,0 +1,82 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.Groups))) {
+    await knex.schema.createTable(TableName.Groups, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("orgId").notNullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.string("name").notNullable();
+      t.string("slug").notNullable();
+      t.unique(["orgId", "slug"]);
+      t.string("role").notNullable();
+      t.uuid("roleId");
+      t.foreign("roleId").references("id").inTable(TableName.OrgRoles);
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.Groups);
+
+  if (!(await knex.schema.hasTable(TableName.UserGroupMembership))) {
+    await knex.schema.createTable(TableName.UserGroupMembership, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); // link to user and link to groups cascade on groups
+      t.uuid("userId").notNullable();
+      t.foreign("userId").references("id").inTable(TableName.Users).onDelete("CASCADE");
+      t.uuid("groupId").notNullable();
+      t.foreign("groupId").references("id").inTable(TableName.Groups).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.UserGroupMembership);
+
+  if (!(await knex.schema.hasTable(TableName.GroupProjectMembership))) {
+    await knex.schema.createTable(TableName.GroupProjectMembership, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("projectId").notNullable();
+      t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      t.uuid("groupId").notNullable();
+      t.foreign("groupId").references("id").inTable(TableName.Groups).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+  await createOnUpdateTrigger(knex, TableName.GroupProjectMembership);
+
+  if (!(await knex.schema.hasTable(TableName.GroupProjectMembershipRole))) {
+    await knex.schema.createTable(TableName.GroupProjectMembershipRole, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("role").notNullable();
+      t.uuid("projectMembershipId").notNullable();
+      t.foreign("projectMembershipId").references("id").inTable(TableName.GroupProjectMembership).onDelete("CASCADE");
+      // until role is changed/removed the role should not deleted
+      t.uuid("customRoleId");
+      t.foreign("customRoleId").references("id").inTable(TableName.ProjectRoles);
+      t.boolean("isTemporary").notNullable().defaultTo(false);
+      t.string("temporaryMode");
+      t.string("temporaryRange"); // could be cron or relative time like 1H or 1minute etc
+      t.datetime("temporaryAccessStartTime");
+      t.datetime("temporaryAccessEndTime");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.GroupProjectMembershipRole);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.GroupProjectMembershipRole);
+  await dropOnUpdateTrigger(knex, TableName.GroupProjectMembershipRole);
+
+  await knex.schema.dropTableIfExists(TableName.UserGroupMembership);
+  await dropOnUpdateTrigger(knex, TableName.UserGroupMembership);
+
+  await knex.schema.dropTableIfExists(TableName.GroupProjectMembership);
+  await dropOnUpdateTrigger(knex, TableName.GroupProjectMembership);
+
+  await knex.schema.dropTableIfExists(TableName.Groups);
+  await dropOnUpdateTrigger(knex, TableName.Groups);
+}

backend/src/db/migrations/20240414192520_drop-role-roleid-project-membership.ts

@@ -0,0 +1,47 @@
+import { Knex } from "knex";
+
+import { ProjectMembershipRole, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesProjectRoleFieldExist = await knex.schema.hasColumn(TableName.ProjectMembership, "role");
+  const doesProjectRoleIdFieldExist = await knex.schema.hasColumn(TableName.ProjectMembership, "roleId");
+  await knex.schema.alterTable(TableName.ProjectMembership, (t) => {
+    if (doesProjectRoleFieldExist) t.dropColumn("roleId");
+    if (doesProjectRoleIdFieldExist) t.dropColumn("role");
+  });
+
+  const doesIdentityProjectRoleFieldExist = await knex.schema.hasColumn(TableName.IdentityProjectMembership, "role");
+  const doesIdentityProjectRoleIdFieldExist = await knex.schema.hasColumn(
+    TableName.IdentityProjectMembership,
+    "roleId"
+  );
+  await knex.schema.alterTable(TableName.IdentityProjectMembership, (t) => {
+    if (doesIdentityProjectRoleFieldExist) t.dropColumn("roleId");
+    if (doesIdentityProjectRoleIdFieldExist) t.dropColumn("role");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesProjectRoleFieldExist = await knex.schema.hasColumn(TableName.ProjectMembership, "role");
+  const doesProjectRoleIdFieldExist = await knex.schema.hasColumn(TableName.ProjectMembership, "roleId");
+  await knex.schema.alterTable(TableName.ProjectMembership, (t) => {
+    if (!doesProjectRoleFieldExist) t.string("role").defaultTo(ProjectMembershipRole.Member);
+    if (!doesProjectRoleIdFieldExist) {
+      t.uuid("roleId");
+      t.foreign("roleId").references("id").inTable(TableName.ProjectRoles);
+    }
+  });
+
+  const doesIdentityProjectRoleFieldExist = await knex.schema.hasColumn(TableName.IdentityProjectMembership, "role");
+  const doesIdentityProjectRoleIdFieldExist = await knex.schema.hasColumn(
+    TableName.IdentityProjectMembership,
+    "roleId"
+  );
+  await knex.schema.alterTable(TableName.IdentityProjectMembership, (t) => {
+    if (!doesIdentityProjectRoleFieldExist) t.string("role").defaultTo(ProjectMembershipRole.Member);
+    if (!doesIdentityProjectRoleIdFieldExist) {
+      t.uuid("roleId");
+      t.foreign("roleId").references("id").inTable(TableName.ProjectRoles);
+    }
+  });
+}

backend/src/db/migrations/20240417032913_pending-group-addition.ts

@@ -0,0 +1,15 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.UserGroupMembership, (t) => {
+    t.boolean("isPending").notNullable().defaultTo(false);
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.UserGroupMembership, (t) => {
+    t.dropColumn("isPending");
+  });
+}

backend/src/db/migrations/20240423023203_ldap-config-groups.ts

@@ -0,0 +1,34 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.LdapGroupMap))) {
+    await knex.schema.createTable(TableName.LdapGroupMap, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("ldapConfigId").notNullable();
+      t.foreign("ldapConfigId").references("id").inTable(TableName.LdapConfig).onDelete("CASCADE");
+      t.string("ldapGroupCN").notNullable();
+      t.uuid("groupId").notNullable();
+      t.foreign("groupId").references("id").inTable(TableName.Groups).onDelete("CASCADE");
+      t.unique(["ldapGroupCN", "groupId", "ldapConfigId"]);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.LdapGroupMap);
+
+  await knex.schema.alterTable(TableName.LdapConfig, (t) => {
+    t.string("groupSearchBase").notNullable().defaultTo("");
+    t.string("groupSearchFilter").notNullable().defaultTo("");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.LdapGroupMap);
+  await dropOnUpdateTrigger(knex, TableName.LdapGroupMap);
+  await knex.schema.alterTable(TableName.LdapConfig, (t) => {
+    t.dropColumn("groupSearchBase");
+    t.dropColumn("groupSearchFilter");
+  });
+}

backend/src/db/migrations/20240424235842_user-search-filter.ts

@@ -0,0 +1,15 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.LdapConfig, (t) => {
+    t.string("searchFilter").notNullable().defaultTo("");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.LdapConfig, (t) => {
+    t.dropColumn("searchFilter");
+  });
+}

backend/src/db/migrations/20240429154610_audit-log-index.ts

@@ -0,0 +1,28 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesCreatedAtExist = await knex.schema.hasColumn(TableName.AuditLog, "createdAt");
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesProjectIdExist && doesCreatedAtExist) t.index(["projectId", "createdAt"]);
+      if (doesOrgIdExist && doesCreatedAtExist) t.index(["orgId", "createdAt"]);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesCreatedAtExist = await knex.schema.hasColumn(TableName.AuditLog, "createdAt");
+
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesProjectIdExist && doesCreatedAtExist) t.dropIndex(["projectId", "createdAt"]);
+      if (doesOrgIdExist && doesCreatedAtExist) t.dropIndex(["orgId", "createdAt"]);
+    });
+  }
+}

backend/src/db/migrations/20240503101144_audit-log-stream.ts

@@ -0,0 +1,28 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.AuditLogStream))) {
+    await knex.schema.createTable(TableName.AuditLogStream, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("url").notNullable();
+      t.text("encryptedHeadersCiphertext");
+      t.text("encryptedHeadersIV");
+      t.text("encryptedHeadersTag");
+      t.string("encryptedHeadersAlgorithm");
+      t.string("encryptedHeadersKeyEncoding");
+      t.uuid("orgId").notNullable();
+      t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.AuditLogStream);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await dropOnUpdateTrigger(knex, TableName.AuditLogStream);
+  await knex.schema.dropTableIfExists(TableName.AuditLogStream);
+}

backend/src/db/migrations/20240507032811_trusted-saml-ldap-emails.ts

@@ -0,0 +1,54 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const isUsersTablePresent = await knex.schema.hasTable(TableName.Users);
+  if (isUsersTablePresent) {
+    const hasIsEmailVerifiedColumn = await knex.schema.hasColumn(TableName.Users, "isEmailVerified");
+
+    if (!hasIsEmailVerifiedColumn) {
+      await knex.schema.alterTable(TableName.Users, (t) => {
+        t.boolean("isEmailVerified").defaultTo(false);
+      });
+    }
+
+    // Backfilling the isEmailVerified to true where isAccepted is true
+    await knex(TableName.Users).update({ isEmailVerified: true }).where("isAccepted", true);
+  }
+
+  const isUserAliasTablePresent = await knex.schema.hasTable(TableName.UserAliases);
+  if (isUserAliasTablePresent) {
+    await knex.schema.alterTable(TableName.UserAliases, (t) => {
+      t.string("username").nullable().alter();
+    });
+  }
+
+  const isSuperAdminTablePresent = await knex.schema.hasTable(TableName.SuperAdmin);
+  if (isSuperAdminTablePresent) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.boolean("trustSamlEmails").defaultTo(false);
+      t.boolean("trustLdapEmails").defaultTo(false);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.Users, "isEmailVerified")) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.dropColumn("isEmailVerified");
+    });
+  }
+
+  if (await knex.schema.hasColumn(TableName.SuperAdmin, "trustSamlEmails")) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.dropColumn("trustSamlEmails");
+    });
+  }
+
+  if (await knex.schema.hasColumn(TableName.SuperAdmin, "trustLdapEmails")) {
+    await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
+      t.dropColumn("trustLdapEmails");
+    });
+  }
+}

backend/src/db/migrations/20240507162140_access-approval-policy.ts

@@ -0,0 +1,41 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.AccessApprovalPolicy))) {
+    await knex.schema.createTable(TableName.AccessApprovalPolicy, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("name").notNullable();
+      t.integer("approvals").defaultTo(1).notNullable();
+      t.string("secretPath");
+
+      t.uuid("envId").notNullable();
+      t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+    await createOnUpdateTrigger(knex, TableName.AccessApprovalPolicy);
+  }
+
+  if (!(await knex.schema.hasTable(TableName.AccessApprovalPolicyApprover))) {
+    await knex.schema.createTable(TableName.AccessApprovalPolicyApprover, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("approverId").notNullable();
+      t.foreign("approverId").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+
+      t.uuid("policyId").notNullable();
+      t.foreign("policyId").references("id").inTable(TableName.AccessApprovalPolicy).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+    await createOnUpdateTrigger(knex, TableName.AccessApprovalPolicyApprover);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.AccessApprovalPolicyApprover);
+  await knex.schema.dropTableIfExists(TableName.AccessApprovalPolicy);
+
+  await dropOnUpdateTrigger(knex, TableName.AccessApprovalPolicyApprover);
+  await dropOnUpdateTrigger(knex, TableName.AccessApprovalPolicy);
+}

backend/src/db/migrations/20240507162141_access.ts

@@ -0,0 +1,51 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.AccessApprovalRequest))) {
+    await knex.schema.createTable(TableName.AccessApprovalRequest, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+
+      t.uuid("policyId").notNullable();
+      t.foreign("policyId").references("id").inTable(TableName.AccessApprovalPolicy).onDelete("CASCADE");
+
+      t.uuid("privilegeId").nullable();
+      t.foreign("privilegeId").references("id").inTable(TableName.ProjectUserAdditionalPrivilege).onDelete("CASCADE");
+
+      t.uuid("requestedBy").notNullable();
+      t.foreign("requestedBy").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+
+      // We use these values to create the actual privilege at a later point in time.
+      t.boolean("isTemporary").notNullable();
+      t.string("temporaryRange").nullable();
+
+      t.jsonb("permissions").notNullable();
+
+      t.timestamps(true, true, true);
+    });
+  }
+  await createOnUpdateTrigger(knex, TableName.AccessApprovalRequest);
+
+  if (!(await knex.schema.hasTable(TableName.AccessApprovalRequestReviewer))) {
+    await knex.schema.createTable(TableName.AccessApprovalRequestReviewer, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.uuid("member").notNullable();
+      t.foreign("member").references("id").inTable(TableName.ProjectMembership).onDelete("CASCADE");
+      t.string("status").notNullable();
+      t.uuid("requestId").notNullable();
+      t.foreign("requestId").references("id").inTable(TableName.AccessApprovalRequest).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+  }
+  await createOnUpdateTrigger(knex, TableName.AccessApprovalRequestReviewer);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.AccessApprovalRequestReviewer);
+  await knex.schema.dropTableIfExists(TableName.AccessApprovalRequest);
+
+  await dropOnUpdateTrigger(knex, TableName.AccessApprovalRequestReviewer);
+  await dropOnUpdateTrigger(knex, TableName.AccessApprovalRequest);
+}

backend/src/db/migrations/20240507210655_identity-aws-auth.ts

@@ -0,0 +1,30 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityAwsAuth))) {
+    await knex.schema.createTable(TableName.IdentityAwsAuth, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.bigInteger("accessTokenTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenMaxTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenNumUsesLimit").defaultTo(0).notNullable();
+      t.jsonb("accessTokenTrustedIps").notNullable();
+      t.timestamps(true, true, true);
+      t.uuid("identityId").notNullable().unique();
+      t.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE");
+      t.string("type").notNullable();
+      t.string("stsEndpoint").notNullable();
+      t.string("allowedPrincipalArns").notNullable();
+      t.string("allowedAccountIds").notNullable();
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityAwsAuth);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityAwsAuth);
+  await dropOnUpdateTrigger(knex, TableName.IdentityAwsAuth);
+}

backend/src/db/migrations/20240514041650_identity-gcp-auth.ts

@@ -0,0 +1,30 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityGcpAuth))) {
+    await knex.schema.createTable(TableName.IdentityGcpAuth, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.bigInteger("accessTokenTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenMaxTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenNumUsesLimit").defaultTo(0).notNullable();
+      t.jsonb("accessTokenTrustedIps").notNullable();
+      t.timestamps(true, true, true);
+      t.uuid("identityId").notNullable().unique();
+      t.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE");
+      t.string("type").notNullable();
+      t.string("allowedServiceAccounts").notNullable();
+      t.string("allowedProjects").notNullable();
+      t.string("allowedZones").notNullable(); // GCE only (fully qualified zone names)
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityGcpAuth);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityGcpAuth);
+  await dropOnUpdateTrigger(knex, TableName.IdentityGcpAuth);
+}

backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts

@@ -0,0 +1,24 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.SecretReference))) {
+    await knex.schema.createTable(TableName.SecretReference, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("environment").notNullable();
+      t.string("secretPath").notNullable();
+      t.uuid("secretId").notNullable();
+      t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+
+    await createOnUpdateTrigger(knex, TableName.SecretReference);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.SecretReference);
+  await dropOnUpdateTrigger(knex, TableName.SecretReference);
+}

backend/src/db/migrations/20240518142614_kubernetes-auth.ts

@@ -0,0 +1,36 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.IdentityKubernetesAuth))) {
+    await knex.schema.createTable(TableName.IdentityKubernetesAuth, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.bigInteger("accessTokenTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenMaxTTL").defaultTo(7200).notNullable();
+      t.bigInteger("accessTokenNumUsesLimit").defaultTo(0).notNullable();
+      t.jsonb("accessTokenTrustedIps").notNullable();
+      t.timestamps(true, true, true);
+      t.uuid("identityId").notNullable().unique();
+      t.foreign("identityId").references("id").inTable(TableName.Identity).onDelete("CASCADE");
+      t.string("kubernetesHost").notNullable();
+      t.text("encryptedCaCert").notNullable();
+      t.string("caCertIV").notNullable();
+      t.string("caCertTag").notNullable();
+      t.text("encryptedTokenReviewerJwt").notNullable();
+      t.string("tokenReviewerJwtIV").notNullable();
+      t.string("tokenReviewerJwtTag").notNullable();
+      t.string("allowedNamespaces").notNullable();
+      t.string("allowedNames").notNullable();
+      t.string("allowedAudience").notNullable();
+    });
+  }
+
+  await createOnUpdateTrigger(knex, TableName.IdentityKubernetesAuth);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.IdentityKubernetesAuth);
+  await dropOnUpdateTrigger(knex, TableName.IdentityKubernetesAuth);
+}

backend/src/db/migrations/20240520064127_add-integration-sync-status.ts

@@ -0,0 +1,43 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasIsSyncedColumn = await knex.schema.hasColumn(TableName.Integration, "isSynced");
+  const hasSyncMessageColumn = await knex.schema.hasColumn(TableName.Integration, "syncMessage");
+  const hasLastSyncJobId = await knex.schema.hasColumn(TableName.Integration, "lastSyncJobId");
+
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    if (!hasIsSyncedColumn) {
+      t.boolean("isSynced").nullable();
+    }
+
+    if (!hasSyncMessageColumn) {
+      t.text("syncMessage").nullable();
+    }
+
+    if (!hasLastSyncJobId) {
+      t.string("lastSyncJobId").nullable();
+    }
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasIsSyncedColumn = await knex.schema.hasColumn(TableName.Integration, "isSynced");
+  const hasSyncMessageColumn = await knex.schema.hasColumn(TableName.Integration, "syncMessage");
+  const hasLastSyncJobId = await knex.schema.hasColumn(TableName.Integration, "lastSyncJobId");
+
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    if (hasIsSyncedColumn) {
+      t.dropColumn("isSynced");
+    }
+
+    if (hasSyncMessageColumn) {
+      t.dropColumn("syncMessage");
+    }
+
+    if (hasLastSyncJobId) {
+      t.dropColumn("lastSyncJobId");
+    }
+  });
+}

backend/src/db/migrations/20240522193447_index-audit-logs-project-id-org-id.ts

@@ -0,0 +1,26 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesProjectIdExist) t.index("projectId");
+      if (doesOrgIdExist) t.index("orgId");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesProjectIdExist) t.dropIndex("projectId");
+      if (doesOrgIdExist) t.dropIndex("orgId");
+    });
+  }
+}

backend/src/db/migrations/20240522203425_index-secret-snapshot-secrets-envid.ts

@@ -0,0 +1,22 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesEnvIdExist = await knex.schema.hasColumn(TableName.SnapshotSecret, "envId");
+  if (await knex.schema.hasTable(TableName.SnapshotSecret)) {
+    await knex.schema.alterTable(TableName.SnapshotSecret, (t) => {
+      if (doesEnvIdExist) t.index("envId");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesEnvIdExist = await knex.schema.hasColumn(TableName.SnapshotSecret, "envId");
+
+  if (await knex.schema.hasTable(TableName.SnapshotSecret)) {
+    await knex.schema.alterTable(TableName.SnapshotSecret, (t) => {
+      if (doesEnvIdExist) t.dropIndex("envId");
+    });
+  }
+}

backend/src/db/migrations/20240522204414_index-secret-version-envId.ts

@@ -0,0 +1,22 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesEnvIdExist = await knex.schema.hasColumn(TableName.SecretVersion, "envId");
+  if (await knex.schema.hasTable(TableName.SecretVersion)) {
+    await knex.schema.alterTable(TableName.SecretVersion, (t) => {
+      if (doesEnvIdExist) t.index("envId");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesEnvIdExist = await knex.schema.hasColumn(TableName.SecretVersion, "envId");
+
+  if (await knex.schema.hasTable(TableName.SecretVersion)) {
+    await knex.schema.alterTable(TableName.SecretVersion, (t) => {
+      if (doesEnvIdExist) t.dropIndex("envId");
+    });
+  }
+}

backend/src/db/migrations/20240522212706_secret-snapshot-secrets-index-on-snapshotId.ts

@@ -0,0 +1,21 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesSnapshotIdExist = await knex.schema.hasColumn(TableName.SnapshotSecret, "snapshotId");
+  if (await knex.schema.hasTable(TableName.SnapshotSecret)) {
+    await knex.schema.alterTable(TableName.SnapshotSecret, (t) => {
+      if (doesSnapshotIdExist) t.index("snapshotId");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesSnapshotIdExist = await knex.schema.hasColumn(TableName.SnapshotSecret, "snapshotId");
+  if (await knex.schema.hasTable(TableName.SnapshotSecret)) {
+    await knex.schema.alterTable(TableName.SnapshotSecret, (t) => {
+      if (doesSnapshotIdExist) t.dropIndex("snapshotId");
+    });
+  }
+}

backend/src/db/migrations/20240522221147_secret-snapshot-folder-index-on-snapshotId.ts

@@ -0,0 +1,21 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesSnapshotIdExist = await knex.schema.hasColumn(TableName.SnapshotFolder, "snapshotId");
+  if (await knex.schema.hasTable(TableName.SnapshotFolder)) {
+    await knex.schema.alterTable(TableName.SnapshotFolder, (t) => {
+      if (doesSnapshotIdExist) t.index("snapshotId");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesSnapshotIdExist = await knex.schema.hasColumn(TableName.SnapshotFolder, "snapshotId");
+  if (await knex.schema.hasTable(TableName.SnapshotFolder)) {
+    await knex.schema.alterTable(TableName.SnapshotFolder, (t) => {
+      if (doesSnapshotIdExist) t.dropIndex("snapshotId");
+    });
+  }
+}

backend/src/db/migrations/20240522225402_secrets-index-on-folder-id-user-id.ts

@@ -0,0 +1,24 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesFolderIdExist = await knex.schema.hasColumn(TableName.Secret, "folderId");
+  const doesUserIdExist = await knex.schema.hasColumn(TableName.Secret, "userId");
+  if (await knex.schema.hasTable(TableName.Secret)) {
+    await knex.schema.alterTable(TableName.Secret, (t) => {
+      if (doesFolderIdExist && doesUserIdExist) t.index(["folderId", "userId"]);
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesFolderIdExist = await knex.schema.hasColumn(TableName.Secret, "folderId");
+  const doesUserIdExist = await knex.schema.hasColumn(TableName.Secret, "userId");
+
+  if (await knex.schema.hasTable(TableName.Secret)) {
+    await knex.schema.alterTable(TableName.Secret, (t) => {
+      if (doesUserIdExist && doesFolderIdExist) t.dropIndex(["folderId", "userId"]);
+    });
+  }
+}

backend/src/db/migrations/20240523003158_audit-log-add-expireAt-index.ts

@@ -0,0 +1,22 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesExpireAtExist = await knex.schema.hasColumn(TableName.AuditLog, "expiresAt");
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesExpireAtExist) t.index("expiresAt");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesExpireAtExist = await knex.schema.hasColumn(TableName.AuditLog, "expiresAt");
+
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      if (doesExpireAtExist) t.dropIndex("expiresAt");
+    });
+  }
+}

backend/src/db/schemas/access-approval-policies-approvers.ts

@@ -0,0 +1,25 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const AccessApprovalPoliciesApproversSchema = z.object({
+  id: z.string().uuid(),
+  approverId: z.string().uuid(),
+  policyId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TAccessApprovalPoliciesApprovers = z.infer<typeof AccessApprovalPoliciesApproversSchema>;
+export type TAccessApprovalPoliciesApproversInsert = Omit<
+  z.input<typeof AccessApprovalPoliciesApproversSchema>,
+  TImmutableDBKeys
+>;
+export type TAccessApprovalPoliciesApproversUpdate = Partial<
+  Omit<z.input<typeof AccessApprovalPoliciesApproversSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/access-approval-policies.ts

@@ -7,16 +7,18 @@ import { z } from "zod";
 
 import { TImmutableDBKeys } from "./models";
 
-export const SecretApprovalPoliciesSchema = z.object({
+export const AccessApprovalPoliciesSchema = z.object({
   id: z.string().uuid(),
   name: z.string(),
-  secretPath: z.string().nullable().optional(),
   approvals: z.number().default(1),
+  secretPath: z.string().nullable().optional(),
   envId: z.string().uuid(),
   createdAt: z.date(),
-  updatedAt: z.date(),
+  updatedAt: z.date()
 });
 
-export type TSecretApprovalPolicies = z.infer<typeof SecretApprovalPoliciesSchema>;
-export type TSecretApprovalPoliciesInsert = Omit<TSecretApprovalPolicies, TImmutableDBKeys>;
-export type TSecretApprovalPoliciesUpdate = Partial<Omit<TSecretApprovalPolicies, TImmutableDBKeys>>;
+export type TAccessApprovalPolicies = z.infer<typeof AccessApprovalPoliciesSchema>;
+export type TAccessApprovalPoliciesInsert = Omit<z.input<typeof AccessApprovalPoliciesSchema>, TImmutableDBKeys>;
+export type TAccessApprovalPoliciesUpdate = Partial<
+  Omit<z.input<typeof AccessApprovalPoliciesSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/access-approval-requests-reviewers.ts

@@ -0,0 +1,26 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const AccessApprovalRequestsReviewersSchema = z.object({
+  id: z.string().uuid(),
+  member: z.string().uuid(),
+  status: z.string(),
+  requestId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TAccessApprovalRequestsReviewers = z.infer<typeof AccessApprovalRequestsReviewersSchema>;
+export type TAccessApprovalRequestsReviewersInsert = Omit<
+  z.input<typeof AccessApprovalRequestsReviewersSchema>,
+  TImmutableDBKeys
+>;
+export type TAccessApprovalRequestsReviewersUpdate = Partial<
+  Omit<z.input<typeof AccessApprovalRequestsReviewersSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/access-approval-requests.ts

@@ -0,0 +1,26 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const AccessApprovalRequestsSchema = z.object({
+  id: z.string().uuid(),
+  policyId: z.string().uuid(),
+  privilegeId: z.string().uuid().nullable().optional(),
+  requestedBy: z.string().uuid(),
+  isTemporary: z.boolean(),
+  temporaryRange: z.string().nullable().optional(),
+  permissions: z.unknown(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TAccessApprovalRequests = z.infer<typeof AccessApprovalRequestsSchema>;
+export type TAccessApprovalRequestsInsert = Omit<z.input<typeof AccessApprovalRequestsSchema>, TImmutableDBKeys>;
+export type TAccessApprovalRequestsUpdate = Partial<
+  Omit<z.input<typeof AccessApprovalRequestsSchema>, TImmutableDBKeys>
+>;

backend/src/db/schemas/api-keys.ts

@@ -19,5 +19,5 @@ export const ApiKeysSchema = z.object({
 });
 
 export type TApiKeys = z.infer<typeof ApiKeysSchema>;
-export type TApiKeysInsert = Omit<TApiKeys, TImmutableDBKeys>;
-export type TApiKeysUpdate = Partial<Omit<TApiKeys, TImmutableDBKeys>>;
+export type TApiKeysInsert = Omit<z.input<typeof ApiKeysSchema>, TImmutableDBKeys>;
+export type TApiKeysUpdate = Partial<Omit<z.input<typeof ApiKeysSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/audit-log-streams.ts

@@ -0,0 +1,25 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const AuditLogStreamsSchema = z.object({
+  id: z.string().uuid(),
+  url: z.string(),
+  encryptedHeadersCiphertext: z.string().nullable().optional(),
+  encryptedHeadersIV: z.string().nullable().optional(),
+  encryptedHeadersTag: z.string().nullable().optional(),
+  encryptedHeadersAlgorithm: z.string().nullable().optional(),
+  encryptedHeadersKeyEncoding: z.string().nullable().optional(),
+  orgId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TAuditLogStreams = z.infer<typeof AuditLogStreamsSchema>;
+export type TAuditLogStreamsInsert = Omit<z.input<typeof AuditLogStreamsSchema>, TImmutableDBKeys>;
+export type TAuditLogStreamsUpdate = Partial<Omit<z.input<typeof AuditLogStreamsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/audit-logs.ts

@@ -24,5 +24,5 @@ export const AuditLogsSchema = z.object({
 });
 
 export type TAuditLogs = z.infer<typeof AuditLogsSchema>;
-export type TAuditLogsInsert = Omit<TAuditLogs, TImmutableDBKeys>;
-export type TAuditLogsUpdate = Partial<Omit<TAuditLogs, TImmutableDBKeys>>;
+export type TAuditLogsInsert = Omit<z.input<typeof AuditLogsSchema>, TImmutableDBKeys>;
+export type TAuditLogsUpdate = Partial<Omit<z.input<typeof AuditLogsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/auth-token-sessions.ts

@@ -20,5 +20,5 @@ export const AuthTokenSessionsSchema = z.object({
 });
 
 export type TAuthTokenSessions = z.infer<typeof AuthTokenSessionsSchema>;
-export type TAuthTokenSessionsInsert = Omit<TAuthTokenSessions, TImmutableDBKeys>;
-export type TAuthTokenSessionsUpdate = Partial<Omit<TAuthTokenSessions, TImmutableDBKeys>>;
+export type TAuthTokenSessionsInsert = Omit<z.input<typeof AuthTokenSessionsSchema>, TImmutableDBKeys>;
+export type TAuthTokenSessionsUpdate = Partial<Omit<z.input<typeof AuthTokenSessionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/auth-tokens.ts

@@ -21,5 +21,5 @@ export const AuthTokensSchema = z.object({
 });
 
 export type TAuthTokens = z.infer<typeof AuthTokensSchema>;
-export type TAuthTokensInsert = Omit<TAuthTokens, TImmutableDBKeys>;
-export type TAuthTokensUpdate = Partial<Omit<TAuthTokens, TImmutableDBKeys>>;
+export type TAuthTokensInsert = Omit<z.input<typeof AuthTokensSchema>, TImmutableDBKeys>;
+export type TAuthTokensUpdate = Partial<Omit<z.input<typeof AuthTokensSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/backup-private-key.ts

@@ -22,5 +22,5 @@ export const BackupPrivateKeySchema = z.object({
 });
 
 export type TBackupPrivateKey = z.infer<typeof BackupPrivateKeySchema>;
-export type TBackupPrivateKeyInsert = Omit<TBackupPrivateKey, TImmutableDBKeys>;
-export type TBackupPrivateKeyUpdate = Partial<Omit<TBackupPrivateKey, TImmutableDBKeys>>;
+export type TBackupPrivateKeyInsert = Omit<z.input<typeof BackupPrivateKeySchema>, TImmutableDBKeys>;
+export type TBackupPrivateKeyUpdate = Partial<Omit<z.input<typeof BackupPrivateKeySchema>, TImmutableDBKeys>>;

backend/src/db/schemas/dynamic-secret-leases.ts

@@ -0,0 +1,24 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const DynamicSecretLeasesSchema = z.object({
+  id: z.string().uuid(),
+  version: z.number(),
+  externalEntityId: z.string(),
+  expireAt: z.date(),
+  status: z.string().nullable().optional(),
+  statusDetails: z.string().nullable().optional(),
+  dynamicSecretId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TDynamicSecretLeases = z.infer<typeof DynamicSecretLeasesSchema>;
+export type TDynamicSecretLeasesInsert = Omit<z.input<typeof DynamicSecretLeasesSchema>, TImmutableDBKeys>;
+export type TDynamicSecretLeasesUpdate = Partial<Omit<z.input<typeof DynamicSecretLeasesSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/dynamic-secrets.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const DynamicSecretsSchema = z.object({
+  id: z.string().uuid(),
+  name: z.string(),
+  version: z.number(),
+  type: z.string(),
+  defaultTTL: z.string(),
+  maxTTL: z.string().nullable().optional(),
+  inputIV: z.string(),
+  inputCiphertext: z.string(),
+  inputTag: z.string(),
+  algorithm: z.string().default("aes-256-gcm"),
+  keyEncoding: z.string().default("utf8"),
+  folderId: z.string().uuid(),
+  status: z.string().nullable().optional(),
+  statusDetails: z.string().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TDynamicSecrets = z.infer<typeof DynamicSecretsSchema>;
+export type TDynamicSecretsInsert = Omit<z.input<typeof DynamicSecretsSchema>, TImmutableDBKeys>;
+export type TDynamicSecretsUpdate = Partial<Omit<z.input<typeof DynamicSecretsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/git-app-install-sessions.ts

@@ -17,5 +17,5 @@ export const GitAppInstallSessionsSchema = z.object({
 });
 
 export type TGitAppInstallSessions = z.infer<typeof GitAppInstallSessionsSchema>;
-export type TGitAppInstallSessionsInsert = Omit<TGitAppInstallSessions, TImmutableDBKeys>;
-export type TGitAppInstallSessionsUpdate = Partial<Omit<TGitAppInstallSessions, TImmutableDBKeys>>;
+export type TGitAppInstallSessionsInsert = Omit<z.input<typeof GitAppInstallSessionsSchema>, TImmutableDBKeys>;
+export type TGitAppInstallSessionsUpdate = Partial<Omit<z.input<typeof GitAppInstallSessionsSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/git-app-org.ts

@@ -17,5 +17,5 @@ export const GitAppOrgSchema = z.object({
 });
 
 export type TGitAppOrg = z.infer<typeof GitAppOrgSchema>;
-export type TGitAppOrgInsert = Omit<TGitAppOrg, TImmutableDBKeys>;
-export type TGitAppOrgUpdate = Partial<Omit<TGitAppOrg, TImmutableDBKeys>>;
+export type TGitAppOrgInsert = Omit<z.input<typeof GitAppOrgSchema>, TImmutableDBKeys>;
+export type TGitAppOrgUpdate = Partial<Omit<z.input<typeof GitAppOrgSchema>, TImmutableDBKeys>>;

backend/src/db/schemas/group-project-membership-roles.ts

@@ -0,0 +1,31 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const GroupProjectMembershipRolesSchema = z.object({
+  id: z.string().uuid(),
+  role: z.string(),
+  projectMembershipId: z.string().uuid(),
+  customRoleId: z.string().uuid().nullable().optional(),
+  isTemporary: z.boolean().default(false),
+  temporaryMode: z.string().nullable().optional(),
+  temporaryRange: z.string().nullable().optional(),
+  temporaryAccessStartTime: z.date().nullable().optional(),
+  temporaryAccessEndTime: z.date().nullable().optional(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TGroupProjectMembershipRoles = z.infer<typeof GroupProjectMembershipRolesSchema>;
+export type TGroupProjectMembershipRolesInsert = Omit<
+  z.input<typeof GroupProjectMembershipRolesSchema>,
+  TImmutableDBKeys
+>;
+export type TGroupProjectMembershipRolesUpdate = Partial<
+  Omit<z.input<typeof GroupProjectMembershipRolesSchema>, TImmutableDBKeys>
+>;