Merge pull request #1369 from akhilmhdh/chore/feature-x-guide

feat: added guides for new backend development

.env.migration.example

@@ -0,0 +1 @@
+DB_CONNECTION_URI=

.gitignore

@@ -6,7 +6,7 @@ node_modules
 .env.gamma
 .env.prod
 .env.infisical
-
+.env.migration
 *~
 *.swp
 *.swo

backend/scripts/generate-schema-types.ts

@@ -3,13 +3,9 @@ import dotenv from "dotenv";
 import path from "path";
 import knex from "knex";
 import { writeFileSync } from "fs";
-import promptSync from "prompt-sync";
-
-const prompt = promptSync({ sigint: true });
 
 dotenv.config({
-  path: path.join(__dirname, "../.env"),
-  debug: true
+  path: path.join(__dirname, "../../.env.migration")
 });
 
 const db = knex({
@@ -94,17 +90,7 @@ const main = async () => {
       .orderBy("table_name")
   ).filter((el) => !el.tableName.includes("_migrations"));
 
-  console.log("Select a table to generate schema");
-  console.table(tables);
-  console.log("all: all tables");
-  const selectedTables = prompt("Type table numbers comma seperated: ");
-  const tableNumbers =
-    selectedTables !== "all" ? selectedTables.split(",").map((el) => Number(el)) : [];
-
   for (let i = 0; i < tables.length; i += 1) {
-    // skip if not desired table
-    if (selectedTables !== "all" && !tableNumbers.includes(i)) continue;
-
     const { tableName } = tables[i];
     const columns = await db(tableName).columnInfo();
     const columnNames = Object.keys(columns);
@@ -124,16 +110,16 @@ const main = async () => {
       if (colInfo.nullable) {
         ztype = ztype.concat(".nullable().optional()");
       }
-      schema = schema.concat(`${!schema ? "\n" : ""}  ${columnName}: ${ztype},\n`);
+      schema = schema.concat(
+        `${!schema ? "\n" : ""}  ${columnName}: ${ztype}${colNum === columnNames.length - 1 ? "" : ","}\n`
+      );
     }
 
     const dashcase = tableName.split("_").join("-");
     const pascalCase = tableName
       .split("_")
-      .reduce(
-        (prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`,
-        ""
-      );
+      .reduce((prev, curr) => prev + `${curr.at(0)?.toUpperCase()}${curr.slice(1).toLowerCase()}`, "");
+
     writeFileSync(
       path.join(__dirname, "../src/db/schemas", `${dashcase}.ts`),
       `// Code generated by automation script, DO NOT EDIT.
@@ -152,15 +138,6 @@ export type T${pascalCase}Insert = Omit<T${pascalCase}, TImmutableDBKeys>;
 export type T${pascalCase}Update = Partial<Omit<T${pascalCase}, TImmutableDBKeys>>;
 `
     );
-
-    // const file = readFileSync(path.join(__dirname, "../src/db/schemas/index.ts"), "utf8");
-    // if (!file.includes(`export * from "./${dashcase};"`)) {
-    //   appendFileSync(
-    //     path.join(__dirname, "../src/db/schemas/index.ts"),
-    //     `\nexport * from "./${dashcase}";`,
-    //     "utf8"
-    //   );
-    // }
   }
 
   process.exit(0);

backend/src/db/knexfile.ts

@@ -7,7 +7,7 @@ import path from "path";
 
 // Update with your config settings.
 dotenv.config({
-  path: path.join(__dirname, "../../.env"),
+  path: path.join(__dirname, "../../../.env.migration"),
   debug: true
 });
 export default {

docs/contributing/platform/backend/folder-structure.mdx

@@ -0,0 +1,82 @@
+---
+title: 'Backend folder structure'
+---
+
+```
+├── scripts
+├── e2e-test
+└── src/
+    ├── @types/
+    │   ├── knex.d.ts
+    │   └── fastify.d.ts
+    ├── db/
+    │   ├── migrations
+    │   ├── schemas
+    │   └── seed
+    ├── lib/
+    │   ├── fn
+    │   ├── date
+    │   └── config
+    ├── queue
+    ├── server/
+    │   ├── routes/
+    │   │   ├── v1
+    │   │   └── v2
+    │   ├── plugins
+    │   └── config
+    ├── services/
+    │   ├── auth
+    │   ├── org
+    │   └── project/
+    │       ├── project-service.ts
+    │       ├── project-types.ts
+    │       └── project-dal.ts
+    └── ee/
+        ├── routes
+        └── services
+```
+
+### `backend/scripts` 
+Contains reusable scripts for backend automation, like running migrations and generating SQL schemas.
+
+### `backend/e2e-test` 
+Integration tests for the APIs.
+
+### `backend/src`
+The source code of the backend.
+
+- `@types`: Type definitions for libraries like Fastify and Knex.
+- `db`: Knex.js configuration for the database, including migration, seed files, and SQL type schemas.
+- `lib`: Stateless, reusable functions used across the codebase.
+- `queue`: Infisical's queue system based on BullMQ.
+
+### `src/server`
+
+- Scope anything related to Fastify/service here.
+- Includes routes, Fastify plugins, and server configurations.
+- The routes folder contains various versions of routes separated into v1, v2, etc.
+
+### `src/services`
+
+- Handles the core business logic for all operations.
+- Follows the co-location principle: related components should be kept together.
+- Each service component typically contains:
+
+  1. **dal**: Database Access Layer functions for database operations
+  2. **service**: The service layer containing business logic.
+  3. **type**: Type definitions used within the service component.
+  4. **fns**: An optional component for sharing reusable functions related to the service.
+  5. **queue**: An optional component for queue-specific logic, like `secret-queue.ts`.
+
+### `src/ee`
+
+Follows the same pattern as above, with the exception of a license change from MIT to Infisical Proprietary License.
+
+### Guidelines and Best Practices
+
+- All services are interconnected at `/src/server/routes/index.ts`, following the principle of simple dependency injection.
+- Files should be named in dash-case.
+- Avoid using classes in the codebase; opt for simple functions instead.
+- All committed code must be properly linted using `npm run lint:fix` and type-checked with `npm run type:check`.
+- Minimize shared logic between services as much as possible.
+- Controllers within a router component should ideally call only one service layer, with exceptions for services like `audit-log` that require access to request object data.

docs/contributing/platform/backend/how-to-create-a-feature.mdx

@@ -0,0 +1,56 @@
+---
+title: "Backend development guide"
+---
+
+Suppose you're interested in implementing a new feature, let's call it "feature-x." Here are the steps you should follow:
+
+## Database schema migration
+In order to run [schema migrations](https://en.wikipedia.org/wiki/Schema_migration#:~:text=A%20schema%20migration%20is%20performed,some%20newer%20or%20older%20version) you need to expose your database connection string. Create a `.env.migration` file to set the database connection URI for migration scripts, or alternatively, export the `DB_CONNECTION_URI` environment variable.
+
+## Creating new database model 
+If your feature involves a change in the database, you need to first address this by generating the necessary database schemas.
+
+1. If you're adding a new table, update the `TableName` enum in `/src/db/schemas/models.ts` to include the new table name.
+2. Create a new migration file by running `npm run migration:new` and give it a relevant name, such as `feature-x`.
+3. Navigate to `/src/db/migrations/<timestamp>_<feature-x>.ts`.
+4. Modify both the `up` and `down` functions to create or alter Postgres fields on migration up and to revert these changes on migration down, ensuring idempotency as outlined [here](https://github.com/graphile/migrate/blob/main/docs/idempotent-examples.md).
+
+### Generating TS Schemas
+
+While typically you would need to manually write TS types for Knex type-sense, we have automated this process:
+
+1. Start the server.
+2. Run `npm run migration:latest` to apply all database changes.
+3. Execute `npm run generate:schema` to automatically generate types and schemas using [zod](https://github.com/colinhacks/zod) in the `/src/db/schemas` folder.
+4. Update the barrel export in `schema/index` and include the new tables in `/src/@types/knex.d.ts` to enable type-sensing in Knex.js.
+
+## Business Logic
+
+Once the database changes are in place, it's time to create the APIs for `feature-x`:
+
+1. Execute `npm run generate:component`.
+2. Choose option 1 for the service component.
+3. Name the service in dash-case, like `feature-x`. This will create a `feature-x` folder in `/src/services` containing three files.
+    1. `feature-x-dal`: The Database Access Layer functions.
+    2. `feature-x-service`: The service layer where all the business logic is handled.
+    3. `feature-x-type`: The types used by `feature-x`.
+
+For reusable shared functions, set up a file named `feature-x-fns`.
+
+Use the custom Infisical function `ormify` in `src/lib/knex` for simple database operations within the DAL.
+
+## Connecting the Service Layer to the Server Layer
+
+Server-related logic is handled in `/src/server`. To connect the service layer to the server layer, we use Fastify plugins for dependency injection:
+
+1. Add the service type in the `fastify.d.ts` file under the `service` namespace of a FastifyServerInstance type.
+2. In `/src/server/routes/index.ts`, instantiate the required dependencies for `feature-x`, such as the DAL and service layers, and then pass them to `fastify.register("service,{...dependencies})`.
+3. This makes the service layer accessible within all routes under the Fastify service instance, accessed via `server.services.<registered service name>.<function>`.
+
+## Writing API Routes
+
+1. To create a route component, run `npm generate:component`.
+2. Select option 3, type the router name in dash-case, and provide the version number. This will generate a router file in `src/server/routes/v<version-number>/<router component name>`
+    1. Implement your logic to connect with the service layer as needed.
+    2. Import the router component in the version folder's index.ts. For instance, if it's in v1, import it in `v1/index.ts`.
+    3. Finally, register it under the appropriate prefix for access.

docs/mint.json

@@ -463,7 +463,9 @@
         {
           "group": "Contributing to platform",
           "pages": [
-            "contributing/platform/developing"
+            "contributing/platform/developing",
+            "contributing/platform/backend/how-to-create-a-feature",
+            "contributing/platform/backend/folder-structure"
           ]
         },
         {