Merge pull request #1362 from akhilmhdh/fix/tsup-cp-template

feat: enabled tsup code splitting and esm directory import, removed manual copy of files

.github/values.yaml

@@ -23,10 +23,10 @@ infisical:
   image:
     repository: infisical/staging_infisical
     tag: "latest"
-    pullPolicy: IfNotPresent
+    pullPolicy: Always
 
   deploymentAnnotations:
-    secrets.infisical.com/auto-reload: "false"
+    secrets.infisical.com/auto-reload: "true"
 
   kubeSecretRef: "infisical-gamma-secrets"
 

Dockerfile.standalone-infisical

@@ -104,7 +104,6 @@ ENV NEXT_PUBLIC_INTERCOM_ID=$INTERCOM_ID \
 WORKDIR / 
 
 COPY --from=backend-runner /app /backend
-COPY --from=backend-runner /app/dist/services/smtp/templates /backend/dist/templates
 
 COPY --from=frontend-runner /app ./backend/frontend-build
 

README.md

@@ -33,7 +33,7 @@
     <img src="https://img.shields.io/github/commit-activity/m/infisical/infisical" alt="git commit activity" />
   </a>
   <a href="https://cloudsmith.io/~infisical/repos/">
-    <img src="https://img.shields.io/badge/Downloads-2.58M-orange" alt="Cloudsmith downloads" />
+    <img src="https://img.shields.io/badge/Downloads-6.95M-orange" alt="Cloudsmith downloads" />
   </a>
   <a href="https://infisical.com/slack">
     <img src="https://img.shields.io/badge/chat-on%20Slack-blueviolet" alt="Slack community channel" />
@@ -53,17 +53,19 @@ We're on a mission to make secret management more accessible to everyone, not ju
 
 ## Features
 
-- **[User-friendly dashboard](https://infisical.com/docs/documentation/platform/project)** to manage secrets across projects and environments (e.g. development, production, etc.)
-- **[Client SDKs](https://infisical.com/docs/sdks/overview)** to fetch secrets for your apps and infrastructure on demand
-- **[Infisical CLI](https://infisical.com/docs/cli/overview)** to fetch and inject secrets into any framework in local development
-- **[Native integrations](https://infisical.com/docs/integrations/overview)** with platforms like GitHub, Vercel, Netlify, and more
-- [**Automatic Kubernetes deployment secret reloads**](https://infisical.com/docs/documentation/getting-started/kubernetes)
-- **[Complete control over your data](https://infisical.com/docs/self-hosting/overview)** - host it yourself on any infrastructure
-- **[Secret versioning](https://infisical.com/docs/documentation/platform/secret-versioning)** and **[Point-in-Time Recovery]()** to version every secret and project state
-- **[Audit logs](https://infisical.com/docs/documentation/platform/audit-logs)** to record every action taken in a project
-- **Role-based Access Controls** per environment
-- [**Simple on-premise deployments** to AWS, Digital Ocean, and more](https://infisical.com/docs/self-hosting/overview)
-- [**Secret Scanning and Leak Prevention**](https://infisical.com/docs/cli/scanning-overview)
+- **[User-friendly dashboard](https://infisical.com/docs/documentation/platform/project)** to manage secrets across projects and environments (e.g. development, production, etc.). 
+- **[Client SDKs](https://infisical.com/docs/sdks/overview)** to fetch secrets for your apps and infrastructure on demand. 
+- **[Infisical CLI](https://infisical.com/docs/cli/overview)** to fetch and inject secrets into any framework in local development and CI/CD. 
+- **[Infisical API](https://infisical.com/docs/api-reference/overview/introduction)** to perform CRUD operation on secrets, users, projects, and any other resource in Infisical. 
+- **[Native integrations](https://infisical.com/docs/integrations/overview)** with platforms like [GitHub](https://infisical.com/docs/integrations/cicd/githubactions), [Vercel](https://infisical.com/docs/integrations/cloud/vercel), [AWS](https://infisical.com/docs/integrations/cloud/aws-secret-manager), and tools like [Terraform](https://infisical.com/docs/integrations/frameworks/terraform), [Ansible](https://infisical.com/docs/integrations/platforms/ansible), and more. 
+- **[Infisical Kubernetes operator](https://infisical.com/docs/documentation/getting-started/kubernetes)** to managed secrets in k8s, automatically reload deployments, and more.
+- **[Infisical Agent](https://infisical.com/docs/infisical-agent/overview)** to inject secrets into your applications without modifying any code logic. 
+- **[Self-hosting and on-prem](https://infisical.com/docs/self-hosting/overview)** to get complete control over your data.
+- **[Secret versioning](https://infisical.com/docs/documentation/platform/secret-versioning)** and **[Point-in-Time Recovery](https://infisical.com/docs/documentation/platform/pit-recovery)** to version every secret and project state. 
+- **[Audit logs](https://infisical.com/docs/documentation/platform/audit-logs)** to record every action taken in a project. 
+- **[Role-based Access Controls](https://infisical.com/docs/documentation/platform/role-based-access-controls)** to create permission sets on any resource in Infisica and assign those to user or machine identities. 
+- **[Simple on-premise deployments](https://infisical.com/docs/self-hosting/overview)** to AWS, Digital Ocean, and more.
+- **[Secret Scanning and Leak Prevention](https://infisical.com/docs/cli/scanning-overview)** to prevent secrets from leaking to git. 
 
 And much more.
 
@@ -115,9 +117,9 @@ Lean about Infisical's code scanning feature [here](https://infisical.com/docs/c
 
 This repo available under the [MIT expat license](https://github.com/Infisical/infisical/blob/main/LICENSE), with the exception of the `ee` directory which will contain premium enterprise features requiring a Infisical license.
 
-If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://cal.com/vmatsiiako/infisical-demo):
+If you are interested in managed Infisical Cloud of self-hosted Enterprise Offering, take a look at [our website](https://infisical.com/) or [book a meeting with us](https://infisical.cal.com/vlad/infisical-demo):
 
-<a href="https://cal.com/vmatsiiako/infisical-demo"><img alt="Schedule a meeting" src="https://cal.com/book-with-cal-dark.svg" /></a>
+<a href="[https://infisical.cal.com/vlad/infisical-demo](https://infisical.cal.com/vlad/infisical-demo)"><img alt="Schedule a meeting" src="https://cal.com/book-with-cal-dark.svg" /></a>
 
 ## Security
 

backend/package.json

@@ -2,12 +2,12 @@
   "name": "backend",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "main": "./dist/main.mjs",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "dev": "tsx watch --clear-screen=false ./src/main.ts  | pino-pretty --colorize --colorizeObjects --singleLine",
     "dev:docker": "nodemon",
-    "build": "rimraf dist && tsup && cp -R ./src/lib/validator/disposable_emails.txt ./dist && cp -R ./src/services/smtp/templates ./dist",
+    "build": "tsup",
     "start": "node dist/main.mjs",
     "type:check": "tsc --noEmit",
     "lint:fix": "eslint --fix --ext js,ts ./src",
@@ -44,7 +44,13 @@
     "@types/pg": "^8.10.9",
     "@types/picomatch": "^2.3.3",
     "@types/prompt-sync": "^4.2.3",
+    "@types/resolve": "^1.20.6",
     "@types/uuid": "^9.0.7",
+    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/parser": "^6.20.0",
+    "eslint": "^8.56.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-airbnb-typescript": "^17.1.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-import": "^2.29.1",
@@ -55,6 +61,7 @@
     "prompt-sync": "^4.2.0",
     "rimraf": "^5.0.5",
     "ts-node": "^10.9.1",
+    "tsc-alias": "^1.8.8",
     "tsconfig-paths": "^4.2.0",
     "tsup": "^8.0.1",
     "tsx": "^4.4.0",
@@ -80,8 +87,6 @@
     "@octokit/webhooks-types": "^7.3.1",
     "@serdnam/pino-cloudwatch-transport": "^1.0.4",
     "@sindresorhus/slugify": "^2.2.1",
-    "@typescript-eslint/eslint-plugin": "^6.20.0",
-    "@typescript-eslint/parser": "^6.20.0",
     "@ucast/mongo2js": "^1.3.4",
     "ajv": "^8.12.0",
     "argon2": "^0.31.2",
@@ -91,9 +96,6 @@
     "bcrypt": "^5.1.1",
     "bullmq": "^5.1.1",
     "dotenv": "^16.3.1",
-    "eslint": "^8.56.0",
-    "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-config-airbnb-typescript": "^17.1.0",
     "fastify": "^4.24.3",
     "fastify-plugin": "^4.5.1",
     "handlebars": "^4.7.8",
@@ -108,7 +110,6 @@
     "nanoid": "^5.0.4",
     "node-cache": "^5.1.2",
     "nodemailer": "^6.9.7",
-    "ora": "^7.0.1",
     "passport-github": "^1.1.0",
     "passport-gitlab2": "^5.0.0",
     "passport-google-oauth20": "^2.0.0",

backend/src/db/instance.ts

@@ -1,11 +1,23 @@
 import knex from "knex";
 
 export type TDbClient = ReturnType<typeof initDbConnection>;
-export const initDbConnection = (dbConnectionUri: string) => {
+export const initDbConnection = ({
+  dbConnectionUri,
+  dbRootCert
+}: {
+  dbConnectionUri: string;
+  dbRootCert?: string;
+}) => {
   const db = knex({
     client: "pg",
-    connection: dbConnectionUri
+    connection: {
+      connectionString: dbConnectionUri,
+      ssl: dbRootCert ? {
+        rejectUnauthorized: true,
+        ca: Buffer.from(dbRootCert, 'base64').toString('ascii')
+      } : false
+    }
   });
 
   return db;
-};
+};

backend/src/lib/config/env.ts

@@ -15,7 +15,8 @@ const envSchema = z
     PORT: z.coerce.number().default(4000),
     REDIS_URL: zpStr(z.string()),
     HOST: zpStr(z.string().default("localhost")),
-    DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database conntection string")),
+    DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")),
+    DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()),
     NODE_ENV: z.enum(["development", "test", "production"]).default("production"),
     SALT_ROUNDS: z.coerce.number().default(10),
     // TODO(akhilmhdh): will be changed to one

backend/src/main.ts

@@ -12,7 +12,11 @@ dotenv.config();
 const run = async () => {
   const logger = await initLogger();
   const appCfg = initEnvConfig(logger);
-  const db = initDbConnection(appCfg.DB_CONNECTION_URI);
+  const db = initDbConnection({
+    dbConnectionUri: appCfg.DB_CONNECTION_URI,
+    dbRootCert: appCfg.DB_ROOT_CERT
+  });
+  
   const smtp = smtpServiceFactory(formatSmtpConfig());
   const queue = queueServiceFactory(appCfg.REDIS_URL);
 

backend/src/server/app.ts

@@ -14,11 +14,10 @@ import fasitfy from "fastify";
 import { Knex } from "knex";
 import { Logger } from "pino";
 
+import { getConfig } from "@app/lib/config/env";
 import { TQueueServiceFactory } from "@app/queue";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 
-import { getConfig } from "@lib/config/env";
-
 import { globalRateLimiterCfg } from "./config/rateLimiter";
 import { fastifyErrHandler } from "./plugins/error-handler";
 import { registerExternalNextjs } from "./plugins/external-nextjs";
@@ -40,6 +39,7 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => {
   const server = fasitfy({
     logger,
     trustProxy: true,
+    connectionTimeout: 30 * 1000,
     ignoreTrailingSlash: true
   }).withTypeProvider<ZodTypeProvider>();
 
@@ -75,7 +75,7 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => {
     if (appCfg.isProductionMode) {
       await server.register(registerExternalNextjs, {
         standaloneMode: appCfg.STANDALONE_MODE,
-        dir: path.join(__dirname, "../"),
+        dir: path.join(__dirname, "../../"),
         port: appCfg.PORT
       });
     }

backend/src/server/plugins/external-nextjs.ts

@@ -45,6 +45,9 @@ export const registerExternalNextjs = async (
     server.route({
       method: ["GET", "PUT", "PATCH", "POST", "DELETE"],
       url: "/*",
+      schema: {
+        hide: true
+      },
       handler: (req, res) =>
         nextApp
           .getRequestHandler()(req.raw, res.raw)

backend/src/server/plugins/swagger.ts

@@ -43,6 +43,7 @@ export const fastifySwagger = fp(async (fastify) => {
   });
 
   await fastify.register(swaggerUI, {
-    routePrefix: "/docs"
+    routePrefix: "/api/docs",
+    prefix: "/api/docs"
   });
 });

backend/src/server/routes/v3/secret-router.ts

@@ -479,7 +479,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         }
       }
 
-      const shouldCapture = req.headers["user-agent"] !== "k8-operator" || shouldRecordK8Event;
+      const shouldCapture =
+        req.query.workspaceId !== "650e71fbae3e6c8572f436d4" &&
+        (req.headers["user-agent"] !== "k8-operator" || shouldRecordK8Event);
       const approximateNumberTotalSecrets = secrets.length * 20;
       if (shouldCapture) {
         server.services.telemetry.sendPostHogEvents({

backend/src/services/project/project-dal.ts

@@ -21,7 +21,11 @@ export const projectDALFactory = (db: TDbClient) => {
           db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
           db.ref("name").withSchema(TableName.Environment).as("envName")
         )
-        .orderBy("createdAt", "asc", "last");
+        .orderBy([
+          { column: `${TableName.Project}.name`, order: "asc" },
+          { column: `${TableName.Environment}.position`, order: "asc" }
+        ]);
+
       const nestedWorkspaces = sqlNestRelationships({
         data: workspaces,
         key: "id",
@@ -102,7 +106,11 @@ export const projectDALFactory = (db: TDbClient) => {
           db.ref("id").withSchema(TableName.Environment).as("envId"),
           db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
           db.ref("name").withSchema(TableName.Environment).as("envName")
-        );
+        )
+        .orderBy([
+          { column: `${TableName.Project}.name`, order: "asc" },
+          { column: `${TableName.Environment}.position`, order: "asc" }
+        ]);
       return sqlNestRelationships({
         data: workspaces,
         key: "id",

backend/tsconfig.json

@@ -22,11 +22,9 @@
     "skipLibCheck": true,
     "baseUrl": ".",
     "paths": {
-      "@app/*": ["./src/*"],
-      "@lib/*": ["./src/lib/*"],
-      "@server/*": ["./src/server/*"]
+      "@app/*": ["./src/*"]
     }
   },
-  "include": ["src/**/*", "scripts/**/*", "e2e-test/**/*","./.eslintrc.js"],
+  "include": ["src/**/*", "scripts/**/*", "e2e-test/**/*", "./.eslintrc.js", "./tsup.config.js"],
   "exclude": ["node_modules"]
 }

backend/tsup.config.js

@@ -1,14 +1,73 @@
+/* eslint-disable */
+import path from "node:path";
+
+import fs from "fs/promises";
+import { replaceTscAliasPaths } from "tsc-alias";
 import { defineConfig } from "tsup";
 
+// Instead of using tsx or tsc for building, consider using tsup.
+// TSX serves as an alternative to Node.js, allowing you to build directly on the Node.js runtime.
+// Its functionality mirrors Node.js, with the only difference being the absence of a final build step. Production should ideally be launched with TSX.
+// TSC is effective for creating a final build, but it requires manual copying of all static files such as handlebars, emails, etc.
+// A significant challenge is the shift towards ESM, as more packages are adopting ESM. If the output is in CommonJS, it may lead to errors.
+// The suggested configuration offers a balance, accommodating both ESM and CommonJS requirements.
+
 export default defineConfig({
   shims: true,
+  clean: true,
+  minify: false,
+  keepNames: true,
+  splitting: false,
   format: "esm",
+  // copy the files to output
   loader: {
     ".handlebars": "copy",
-    ".md": "copy"
+    ".md": "copy",
+    ".txt": "copy"
   },
   external: ["../../../frontend/node_modules/next/dist/server/next-server.js"],
   outDir: "dist",
+  tsconfig: "./tsconfig.json",
   entry: ["./src"],
-  sourceMap: "inline"
+  sourceMap: true,
+  skipNodeModulesBundle: true,
+  esbuildPlugins: [
+    {
+      // esm directory import are not allowed
+      // /folder1 should be explicitly imported as /folder1/index.ts
+      // this plugin will append it automatically on build time to all imports
+      name: "commonjs-esm-directory-import",
+      setup(build) {
+        build.onResolve({ filter: /.*/ }, async (args) => {
+          if (args.importer) {
+            if (args.kind === "import-statement") {
+              const isRelativePath = args.path.startsWith(".");
+              const absPath = isRelativePath
+                ? path.join(args.resolveDir, args.path)
+                : path.join(args.path.replace("@app", "./src"));
+
+              const isFile = await fs
+                .stat(`${absPath}.ts`)
+                .then((el) => el.isFile)
+                .catch((err) => err.code === "ENOTDIR");
+
+              return {
+                path: isFile ? `${args.path}.mjs` : `${args.path}/index.mjs`,
+                external: true
+              };
+            }
+          }
+          return undefined;
+        });
+      }
+    }
+  ],
+  async onSuccess() {
+    // this will replace all tsconfig paths
+    await replaceTscAliasPaths({
+      configFile: "tsconfig.json",
+      watch: false,
+      outDir: "dist"
+    });
+  }
 });