update rate limits

feat: changed backend-pg to backend

.github/workflows/build-docker-image-to-prod.yml

@@ -3,6 +3,7 @@ on:
   push:
     tags:
       - "infisical/v*.*.*"
+      - "!infisical/v*.*.*-postgres"
 
 jobs:
   backend-image:

.github/workflows/release-standalone-docker-img-postgres-offical.yml

@@ -0,0 +1,57 @@
+name: Release standalone docker image
+on:
+  push:
+    tags:
+      - "infisical/v*.*.*-postgres"
+
+jobs:
+  infisical-standalone:
+    name: Build infisical standalone image postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract version from tag
+        id: extract_version
+        run: echo "::set-output name=version::${GITHUB_REF_NAME#infisical/}"
+      - name: ☁️ Checkout source
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: 📦 Install dependencies to test all dependencies
+        run: npm ci --only-production
+        working-directory: backend
+      - name: version output
+        run: |
+          echo "Output Value: ${{ steps.version.outputs.major }}"
+          echo "Output Value: ${{ steps.version.outputs.minor }}"
+          echo "Output Value: ${{ steps.version.outputs.patch }}"
+          echo "Output Value: ${{ steps.version.outputs.version }}"
+          echo "Output Value: ${{ steps.version.outputs.version_type }}"
+          echo "Output Value: ${{ steps.version.outputs.increment }}"
+      - name: Save commit hashes for tag
+        id: commit
+        uses: pr-mpt/actions-commit-hash@v2
+      - name: 🔧 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: 🐋 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
+      - name: 📦 Build backend and export to Docker
+        uses: depot/build-push-action@v1
+        with:
+          project: 64mmf0n610
+          token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
+          push: true
+          context: .
+          tags: |
+            infisical/infisical:latest-postgres
+            infisical/infisical:${{ steps.commit.outputs.short }}
+            infisical/infisical:${{ steps.extract_version.outputs.version }}
+          platforms: linux/amd64,linux/arm64
+          file: Dockerfile.standalone-infisical
+          build-args: |
+            POSTHOG_API_KEY=${{ secrets.PUBLIC_POSTHOG_API_KEY }}
+            INFISICAL_PLATFORM_VERSION=${{ steps.extract_version.outputs.version }}

.github/workflows/release-standalone-docker-img.yml

@@ -3,6 +3,7 @@ on:
   push:
     tags:
       - "infisical/v*.*.*"
+      - "!infisical/v*.*.*-postgres"
 
 jobs:
   infisical-standalone:

.gitignore

@@ -1,6 +1,7 @@
 # backend
 node_modules
 .env
+.env.test
 .env.dev
 .env.gamma
 .env.prod
@@ -61,4 +62,4 @@ yarn-error.log*
 # Editor specific
 .vscode/*
 
-frontend-build
+frontend-build

Dockerfile.standalone-infisical

@@ -2,7 +2,7 @@ ARG POSTHOG_HOST=https://app.posthog.com
 ARG POSTHOG_API_KEY=posthog-api-key
 ARG INTERCOM_ID=intercom-id
 
-FROM  node:16-alpine AS base
+FROM  node:20-alpine AS base
 
 FROM base AS frontend-dependencies
 
@@ -73,6 +73,7 @@ RUN npm ci --only-production
 
 COPY /backend .
 COPY --chown=non-root-user:nodejs standalone-entrypoint.sh standalone-entrypoint.sh
+RUN npm i -D tsconfig-paths
 RUN npm run build
 
 # Production stage
@@ -103,14 +104,17 @@ ENV NEXT_PUBLIC_INTERCOM_ID=$INTERCOM_ID \
 WORKDIR / 
 
 COPY --from=backend-runner /app /backend
+COPY --from=backend-runner /app/dist/services/smtp/templates /backend/dist/templates
 
 COPY --from=frontend-runner /app ./backend/frontend-build
 
+
 ENV PORT 8080
+ENV HOST=0.0.0.0
 ENV HTTPS_ENABLED false 
 ENV NODE_ENV production
 ENV STANDALONE_BUILD true 
-
+ENV STANDALONE_MODE true
 WORKDIR /backend
 
 ENV TELEMETRY_ENABLED true
@@ -119,10 +123,8 @@ HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \
   CMD node healthcheck.js
 
 EXPOSE 8080
+EXPOSE 443
 
 USER non-root-user
 
 CMD ["./standalone-entrypoint.sh"]
-
-
-

Makefile

@@ -7,6 +7,9 @@ push:
 up-dev:
 	docker-compose -f docker-compose.dev.yml up --build
 
+up-pg-dev:
+	docker compose -f docker-compose.pg.yml up --build
+
 i-dev:
 	infisical run -- docker-compose -f docker-compose.dev.yml up --build
 

backend-mongo/.dockerignore

@@ -0,0 +1,11 @@
+node_modules
+.env
+.env.*
+.git
+.gitignore
+Dockerfile
+.dockerignore
+docker-compose.*
+.DS_Store
+*.swp
+*~

backend-mongo/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+built

backend-mongo/Dockerfile

@@ -0,0 +1,33 @@
+# Build stage
+FROM node:16-alpine AS build
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci --only-production
+
+COPY . .
+RUN npm run build
+
+# Production stage
+FROM node:16-alpine
+
+WORKDIR /app
+
+ENV npm_config_cache /home/node/.npm
+
+COPY package*.json ./
+RUN npm ci --only-production && npm cache clean --force
+
+COPY --from=build /app .
+
+RUN apk add --no-cache bash curl && curl -1sLf \
+  'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
+  && apk add infisical=0.8.1 && apk add --no-cache git
+
+HEALTHCHECK --interval=10s --timeout=3s --start-period=10s \  
+  CMD node healthcheck.js
+
+EXPOSE 4000
+
+CMD ["node", "build/index.js"]

backend-mongo/nodemon.json

@@ -0,0 +1,6 @@
+{
+    "watch": ["src"],
+    "ext": ".ts,.js",
+    "ignore": [],
+    "exec": "ts-node ./src/index.ts"
+}

backend-mongo/package.json

@@ -0,0 +1,148 @@
+{
+  "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.319.0",
+    "@casl/ability": "^6.5.0",
+    "@casl/mongoose": "^7.2.1",
+    "@godaddy/terminus": "^4.12.0",
+    "@node-saml/passport-saml": "^4.0.4",
+    "@octokit/rest": "^19.0.5",
+    "@sentry/node": "^7.77.0",
+    "@sentry/tracing": "^7.48.0",
+    "@serdnam/pino-cloudwatch-transport": "^1.0.4",
+    "@types/crypto-js": "^4.1.1",
+    "@types/libsodium-wrappers": "^0.7.10",
+    "@ucast/mongo2js": "^1.3.4",
+    "ajv": "^8.12.0",
+    "argon2": "^0.30.3",
+    "aws-sdk": "^2.1364.0",
+    "axios": "^1.6.0",
+    "axios-retry": "^3.4.0",
+    "bcrypt": "^5.1.0",
+    "bigint-conversion": "^2.4.0",
+    "cookie-parser": "^1.4.6",
+    "cors": "^2.8.5",
+    "crypto-js": "^4.2.0",
+    "dotenv": "^16.0.1",
+    "express": "^4.18.1",
+    "express-async-errors": "^3.1.1",
+    "express-rate-limit": "^6.7.0",
+    "express-validator": "^6.14.2",
+    "handlebars": "^4.7.7",
+    "helmet": "^5.1.1",
+    "infisical-node": "^1.2.1",
+    "ioredis": "^5.3.2",
+    "jmespath": "^0.16.0",
+    "js-yaml": "^4.1.0",
+    "jsonwebtoken": "^9.0.0",
+    "jsrp": "^0.2.4",
+    "libsodium-wrappers": "^0.7.10",
+    "lodash": "^4.17.21",
+    "mongoose": "^7.4.1",
+    "mysql2": "^3.6.2",
+    "nanoid": "^3.3.6",
+    "node-cache": "^5.1.2",
+    "nodemailer": "^6.8.0",
+    "ora": "^5.4.1",
+    "passport": "^0.6.0",
+    "passport-github": "^1.1.0",
+    "passport-gitlab2": "^5.0.0",
+    "passport-google-oauth20": "^2.0.0",
+    "pg": "^8.11.3",
+    "pino": "^8.16.1",
+    "pino-http": "^8.5.1",
+    "posthog-node": "^2.6.0",
+    "probot": "^12.3.3",
+    "query-string": "^7.1.3",
+    "rate-limit-mongo": "^2.3.2",
+    "rimraf": "^3.0.2",
+    "swagger-ui-express": "^4.6.2",
+    "tweetnacl": "^1.0.3",
+    "tweetnacl-util": "^0.15.1",
+    "typescript": "^4.9.3",
+    "utility-types": "^3.10.0",
+    "zod": "^3.22.3"
+  },
+  "overrides": {
+    "rate-limit-mongo": {
+      "mongodb": "5.8.0"
+    }
+  },
+  "name": "infisical-api",
+  "version": "1.0.0",
+  "main": "src/index.js",
+  "scripts": {
+    "start": "node build/index.js",
+    "dev": "nodemon index.js",
+    "swagger-autogen": "node ./swagger/index.ts",
+    "build": "rimraf ./build && tsc && cp -R ./src/templates ./build && cp -R ./src/data ./build",
+    "lint": "eslint . --ext .ts",
+    "lint-and-fix": "eslint . --ext .ts --fix",
+    "lint-staged": "lint-staged",
+    "pretest": "docker compose -f test-resources/docker-compose.test.yml up -d",
+    "test": "cross-env NODE_ENV=test jest --verbose --testTimeout=10000 --detectOpenHandles; npm run posttest",
+    "test:ci": "npm test -- --watchAll=false --ci --reporters=default --reporters=jest-junit --reporters=github-actions --coverage --testLocationInResults --json --outputFile=coverage/report.json",
+    "posttest": "docker compose -f test-resources/docker-compose.test.yml down"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Infisical/infisical-api.git"
+  },
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/Infisical/infisical-api/issues"
+  },
+  "homepage": "https://github.com/Infisical/infisical-api#readme",
+  "description": "",
+  "devDependencies": {
+    "@jest/globals": "^29.3.1",
+    "@posthog/plugin-scaffold": "^1.3.4",
+    "@swc/core": "^1.3.99",
+    "@swc/helpers": "^0.5.3",
+    "@types/bcrypt": "^5.0.0",
+    "@types/bcryptjs": "^2.4.2",
+    "@types/bull": "^4.10.0",
+    "@types/cookie-parser": "^1.4.3",
+    "@types/cors": "^2.8.12",
+    "@types/express": "^4.17.14",
+    "@types/jest": "^29.5.0",
+    "@types/jmespath": "^0.15.1",
+    "@types/jsonwebtoken": "^8.5.9",
+    "@types/lodash": "^4.14.191",
+    "@types/node": "^18.11.3",
+    "@types/nodemailer": "^6.4.6",
+    "@types/passport": "^1.0.12",
+    "@types/pg": "^8.10.7",
+    "@types/picomatch": "^2.3.0",
+    "@types/pino": "^7.0.5",
+    "@types/supertest": "^2.0.12",
+    "@types/swagger-jsdoc": "^6.0.1",
+    "@types/swagger-ui-express": "^4.1.3",
+    "@typescript-eslint/eslint-plugin": "^5.54.0",
+    "@typescript-eslint/parser": "^5.40.1",
+    "cross-env": "^7.0.3",
+    "eslint": "^8.26.0",
+    "eslint-plugin-unused-imports": "^2.0.0",
+    "install": "^0.13.0",
+    "jest": "^29.3.1",
+    "jest-junit": "^15.0.0",
+    "nodemon": "^2.0.19",
+    "npm": "^8.19.3",
+    "pino-pretty": "^10.2.3",
+    "regenerator-runtime": "^0.14.0",
+    "smee-client": "^1.2.3",
+    "supertest": "^6.3.3",
+    "swagger-autogen": "^2.23.5",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1"
+  },
+  "jest-junit": {
+    "outputDirectory": "reports",
+    "outputName": "jest-junit.xml",
+    "ancestorSeparator": " › ",
+    "uniqueOutputName": "false",
+    "suiteNameTemplate": "{filepath}",
+    "classNameTemplate": "{classname}",
+    "titleTemplate": "{title}"
+  }
+}

backend-mongo/src/config/index.ts

@@ -6,6 +6,9 @@ export const client = new InfisicalClient({
   token: process.env.INFISICAL_TOKEN!
 });
 
+export const getIsMigrationMode = async () =>
+  (await client.getSecret("MIGRATION_MODE")).secretValue === "true";
+
 export const getPort = async () => (await client.getSecret("PORT")).secretValue || 4000;
 export const getEncryptionKey = async () => {
   const secretValue = (await client.getSecret("ENCRYPTION_KEY")).secretValue;

backend-mongo/src/config/serverConfig.ts

@@ -3,11 +3,11 @@ import { IServerConfig, ServerConfig } from "../models/serverConfig";
 let serverConfig: IServerConfig;
 
 export const serverConfigInit = async () => {
-  const cfg = await ServerConfig.findOne({});
+  const cfg = await ServerConfig.findOne({}).lean();
   if (!cfg) {
     const cfg = new ServerConfig();
     await cfg.save();
-    serverConfig = cfg;
+    serverConfig = cfg.toObject();
   } else {
     serverConfig = cfg;
   }
@@ -19,6 +19,6 @@ export const getServerConfig = () => serverConfig;
 export const updateServerConfig = async (data: Partial<IServerConfig>) => {
   const cfg = await ServerConfig.findByIdAndUpdate(serverConfig._id, data, { new: true });
   if (!cfg) throw new Error("Failed to update server config");
-  serverConfig = cfg;
+  serverConfig = cfg.toObject();
   return serverConfig;
 };

backend-mongo/src/controllers/v1/adminController.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from "express";
-import { getHttpsEnabled } from "../../config";
+import { getHttpsEnabled, getIsMigrationMode } from "../../config";
 import { getServerConfig, updateServerConfig as setServerConfig } from "../../config/serverConfig";
 import { initializeDefaultOrg, issueAuthTokens } from "../../helpers";
 import { validateRequest } from "../../helpers/validation";
@@ -8,9 +8,10 @@ import { TelemetryService } from "../../services";
 import { BadRequestError, UnauthorizedRequestError } from "../../utils/errors";
 import * as reqValidator from "../../validation/admin";
 
-export const getServerConfigInfo = (_req: Request, res: Response) => {
+export const getServerConfigInfo = async (_req: Request, res: Response) => {
   const config = getServerConfig();
-  return res.send({ config });
+  const isMigrationModeOn = await getIsMigrationMode();
+  return res.send({ config: { ...config, isMigrationModeOn } });
 };
 
 export const updateServerConfig = async (req: Request, res: Response) => {

backend-mongo/src/controllers/v1/index.ts

@@ -1,6 +1,5 @@
 import * as authController from "./authController";
 import * as universalAuthController from "./universalAuthController";
-import * as ldapController from "./ldapController";
 import * as botController from "./botController";
 import * as integrationAuthController from "./integrationAuthController";
 import * as integrationController from "./integrationController";
@@ -23,7 +22,6 @@ import * as adminController from "./adminController";
 export {
   authController,
   universalAuthController,
-  ldapController,
   botController,
   integrationAuthController,
   integrationController,

backend-mongo/src/controllers/v3/signupController.ts

@@ -101,15 +101,14 @@ export const completeAccountSignup = async (req: Request, res: Response) => {
       salt,
       verifier
     });
-    
+
     if (!user) throw new Error("Failed to complete account for non-existent user"); // ensure user is non-null
 
-    // this might need to consider LDAP
-    const hasOrgAuthMethodEnabled = user.authMethods.some((authMethod: AuthMethod) =>
-      [AuthMethod.OKTA_SAML, AuthMethod.AZURE_SAML, AuthMethod.JUMPCLOUD_SAML, AuthMethod.LDAP].includes(authMethod)
+    const hasSamlEnabled = user.authMethods.some((authMethod: AuthMethod) =>
+      [AuthMethod.OKTA_SAML, AuthMethod.AZURE_SAML, AuthMethod.JUMPCLOUD_SAML].includes(authMethod)
     );
 
-    if (!hasOrgAuthMethodEnabled) {
+    if (!hasSamlEnabled) {
       // TODO: modify this part
       // initialize default organization and workspace
       await initializeDefaultOrg({

backend-mongo/src/ee/helpers/organizations.ts

@@ -1,6 +1,5 @@
 import { Types } from "mongoose";
 import {
-    LDAPConfig,
     SSOConfig
 } from "../models";
 import {
@@ -62,53 +61,4 @@ export const getSSOConfigHelper = async ({
         issuer,
         cert
     });
-}
-
-export const getLdapConfigHelper = async ({
-    organizationId
-}: {
-    organizationId: Types.ObjectId;
-}) => {
-    
-    const ldapConfig = await LDAPConfig.findOne({
-        organization: organizationId
-    });
-    
-    if (!ldapConfig) throw new Error("Failed to find organization LDAP data");
-    
-    const key = await BotOrgService.getSymmetricKey(
-        ldapConfig.organization
-    );
-    
-    const bindDN = client.decryptSymmetric(
-        ldapConfig.encryptedBindDN,
-        key,
-        ldapConfig.bindDNIV,
-        ldapConfig.bindDNTag
-    );
-
-    const bindPass = client.decryptSymmetric(
-        ldapConfig.encryptedBindPass,
-        key,
-        ldapConfig.bindPassIV,
-        ldapConfig.bindPassTag
-    );
-
-    const caCert = client.decryptSymmetric(
-        ldapConfig.encryptedCACert,
-        key,
-        ldapConfig.caCertIV,
-        ldapConfig.caCertTag
-    );
-    
-    return ({
-        _id: ldapConfig._id,
-        organization: ldapConfig.organization,
-        isActive: ldapConfig.isActive,
-        url: ldapConfig.url,
-        bindDN,
-        bindPass,
-        searchBase: ldapConfig.searchBase,
-        caCert
-    });
 }

backend-mongo/src/ee/models/index.ts

@@ -3,11 +3,10 @@ export * from "./secretVersion";
 export * from "./folderVersion";
 export * from "./role";
 export * from "./ssoConfig";
-export * from "./ldapConfig";
 export * from "./trustedIp";
 export * from "./auditLog";
 export * from  "./gitRisks";
 export * from  "./gitAppOrganizationInstallation";
 export * from  "./gitAppInstallationSession";
 export * from "./secretApprovalPolicy";
-export * from "./secretApprovalRequest";
+export * from "./secretApprovalRequest";