reduce ttl

feat(server): moved back audit log to queue now with keystore license

Makefile

@@ -11,4 +11,4 @@ up-prod:
 	docker-compose -f docker-compose.prod.yml up --build
 
 down:
-	docker-compose down
+	docker compose -f docker-compose.dev.yml down

backend/package-lock.json

@@ -47,7 +47,6 @@
         "lodash.isequal": "^4.5.0",
         "mysql2": "^3.9.1",
         "nanoid": "^5.0.4",
-        "node-cache": "^5.1.2",
         "nodemailer": "^6.9.9",
         "ora": "^7.0.1",
         "passport-github": "^1.1.0",
@@ -56,7 +55,7 @@
         "pg": "^8.11.3",
         "picomatch": "^3.0.1",
         "pino": "^8.16.2",
-        "posthog-node": "^3.6.0",
+        "posthog-node": "^3.6.2",
         "probot": "^13.0.0",
         "smee-client": "^2.0.0",
         "tweetnacl": "^1.0.3",
@@ -5706,14 +5705,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/clone": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-      "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
     "node_modules/cluster-key-slot": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
@@ -9258,17 +9249,6 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
       "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
     },
-    "node_modules/node-cache": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz",
-      "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==",
-      "dependencies": {
-        "clone": "2.x"
-      },
-      "engines": {
-        "node": ">= 8.0.0"
-      }
-    },
     "node_modules/node-fetch": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -10318,9 +10298,9 @@
       "dev": true
     },
     "node_modules/posthog-node": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.0.tgz",
-      "integrity": "sha512-N/4//SIQR4fhwbHnDdJ2rQCYdu9wo0EVPK4lVgZswp5R/E42RKlpuO6ZfPsBl+Bcg06OYiOd/WR/jLV90FCoSw==",
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-3.6.2.tgz",
+      "integrity": "sha512-tVIaShR3SxBx17AlAUS86jQTweKuJIFRedBB504fCz7YPnXJTYSrVcUHn5IINE2wu4jUQimQK6ihQr90Djrdrg==",
       "dependencies": {
         "axios": "^1.6.2",
         "rusha": "^0.8.14"

backend/package.json

@@ -108,7 +108,6 @@
     "lodash.isequal": "^4.5.0",
     "mysql2": "^3.9.1",
     "nanoid": "^5.0.4",
-    "node-cache": "^5.1.2",
     "nodemailer": "^6.9.9",
     "ora": "^7.0.1",
     "passport-github": "^1.1.0",
@@ -117,7 +116,7 @@
     "pg": "^8.11.3",
     "picomatch": "^3.0.1",
     "pino": "^8.16.2",
-    "posthog-node": "^3.6.0",
+    "posthog-node": "^3.6.2",
     "probot": "^13.0.0",
     "smee-client": "^2.0.0",
     "tweetnacl": "^1.0.3",

backend/src/db/knexfile.ts

@@ -17,7 +17,15 @@ dotenv.config({
 export default {
   development: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10
@@ -31,7 +39,15 @@ export default {
   },
   production: {
     client: "postgres",
-    connection: process.env.DB_CONNECTION_URI,
+    connection: {
+      connectionString: process.env.DB_CONNECTION_URI,
+      ssl: process.env.DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
     pool: {
       min: 2,
       max: 10

backend/src/db/migrations/20240226094411_instance-id.ts

@@ -1,3 +1,5 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
 import { Knex } from "knex";
 
 import { TableName } from "../schemas";
@@ -8,10 +10,12 @@ export async function up(knex: Knex): Promise<void> {
   await knex.schema.alterTable(TableName.SuperAdmin, (t) => {
     t.uuid("instanceId").notNullable().defaultTo(knex.fn.uuid());
   });
-  // this is updated to avoid race condition on replication
-  // eslint-disable-next-line
-  // @ts-ignore
-  await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1);
+
+  const superUserConfigExists = await knex(TableName.SuperAdmin).where("id", ADMIN_CONFIG_UUID).first();
+  if (!superUserConfigExists) {
+    // eslint-disable-next-line
+    await knex(TableName.SuperAdmin).update({ id: ADMIN_CONFIG_UUID }).whereNotNull("id").limit(1);
+  }
 }
 
 export async function down(knex: Knex): Promise<void> {

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -24,7 +24,7 @@ export const auditLogQueueServiceFactory = ({
   const pushToLog = async (data: TCreateAuditLogDTO) => {
     await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, {
       removeOnFail: {
-        count: 5
+        count: 3
       },
       removeOnComplete: true
     });
@@ -46,6 +46,7 @@ export const auditLogQueueServiceFactory = ({
     const ttl = plan.auditLogsRetentionDays * MS_IN_DAY;
     // skip inserting if audit log retention is 0 meaning its not supported
     if (ttl === 0) return;
+
     await auditLogDAL.create({
       actor: actor.type,
       actorMetadata: actor.metadata,

backend/src/ee/services/license/license-service.ts

@@ -5,8 +5,8 @@
 // TODO(akhilmhdh): With tony find out the api structure and fill it here
 
 import { ForbiddenError } from "@casl/ability";
-import NodeCache from "node-cache";
 
+import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { BadRequestError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
@@ -39,6 +39,7 @@ type TLicenseServiceFactoryDep = {
   orgDAL: Pick<TOrgDALFactory, "findOrgById">;
   permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
   licenseDAL: TLicenseDALFactory;
+  keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "deleteItem">;
 };
 
 export type TLicenseServiceFactory = ReturnType<typeof licenseServiceFactory>;
@@ -46,12 +47,18 @@ export type TLicenseServiceFactory = ReturnType<typeof licenseServiceFactory>;
 const LICENSE_SERVER_CLOUD_LOGIN = "/api/auth/v1/license-server-login";
 const LICENSE_SERVER_ON_PREM_LOGIN = "/api/auth/v1/license-login";
 
-const FEATURE_CACHE_KEY = (orgId: string, projectId?: string) => `${orgId}-${projectId || ""}`;
-export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: TLicenseServiceFactoryDep) => {
+const LICENSE_SERVER_CLOUD_PLAN_TTL = 30; // 30 second
+const FEATURE_CACHE_KEY = (orgId: string) => `infisical-cloud-plan-${orgId}`;
+
+export const licenseServiceFactory = ({
+  orgDAL,
+  permissionService,
+  licenseDAL,
+  keyStore
+}: TLicenseServiceFactoryDep) => {
   let isValidLicense = false;
   let instanceType = InstanceType.OnPrem;
   let onPremFeatures: TFeatureSet = getDefaultOnPremFeatures();
-  const featureStore = new NodeCache({ stdTTL: 60 });
 
   const appCfg = getConfig();
   const licenseServerCloudApi = setupLicenceRequestWithStore(
@@ -75,6 +82,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
         isValidLicense = true;
         return;
       }
+
       if (appCfg.LICENSE_KEY) {
         const token = await licenseServerOnPremApi.refreshLicence();
         if (token) {
@@ -100,22 +108,21 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
     logger.info(`getPlan: attempting to fetch plan for [orgId=${orgId}] [projectId=${projectId}]`);
     try {
       if (instanceType === InstanceType.Cloud) {
-        const cachedPlan = featureStore.get<TFeatureSet>(FEATURE_CACHE_KEY(orgId, projectId));
-        if (cachedPlan) return cachedPlan;
+        const cachedPlan = await keyStore.getItem(FEATURE_CACHE_KEY(orgId));
+        if (cachedPlan) return JSON.parse(cachedPlan) as TFeatureSet;
 
         const org = await orgDAL.findOrgById(orgId);
         if (!org) throw new BadRequestError({ message: "Org not found" });
         const {
           data: { currentPlan }
         } = await licenseServerCloudApi.request.get<{ currentPlan: TFeatureSet }>(
-          `/api/license-server/v1/customers/${org.customerId}/cloud-plan`,
-          {
-            params: {
-              workspaceId: projectId
-            }
-          }
+          `/api/license-server/v1/customers/${org.customerId}/cloud-plan`
+        );
+        await keyStore.setItemWithExpiry(
+          FEATURE_CACHE_KEY(org.id),
+          LICENSE_SERVER_CLOUD_PLAN_TTL,
+          JSON.stringify(currentPlan)
         );
-        featureStore.set(FEATURE_CACHE_KEY(org.id, projectId), currentPlan);
         return currentPlan;
       }
     } catch (error) {
@@ -123,15 +130,20 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
         `getPlan: encountered an error when fetching pan [orgId=${orgId}] [projectId=${projectId}] [error]`,
         error
       );
+      await keyStore.setItemWithExpiry(
+        FEATURE_CACHE_KEY(orgId),
+        LICENSE_SERVER_CLOUD_PLAN_TTL,
+        JSON.stringify(onPremFeatures)
+      );
       return onPremFeatures;
     }
     return onPremFeatures;
   };
 
-  const refreshPlan = async (orgId: string, projectId?: string) => {
+  const refreshPlan = async (orgId: string) => {
     if (instanceType === InstanceType.Cloud) {
-      featureStore.del(FEATURE_CACHE_KEY(orgId, projectId));
-      await getPlan(orgId, projectId);
+      await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
+      await getPlan(orgId);
     }
   };
 
@@ -166,7 +178,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
           quantity: count
         });
       }
-      featureStore.del(orgId);
+      await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
     } else if (instanceType === InstanceType.EnterpriseOnPrem) {
       const usedSeats = await licenseDAL.countOfOrgMembers(null);
       await licenseServerOnPremApi.request.patch(`/api/license/v1/license`, { usedSeats });
@@ -215,7 +227,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
       `/api/license-server/v1/customers/${organization.customerId}/session/trial`,
       { success_url }
     );
-    featureStore.del(FEATURE_CACHE_KEY(orgId));
+    await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
     return { url };
   };
 

backend/src/server/routes/index.ts

@@ -194,7 +194,7 @@ export const registerRoutes = async (
     projectRoleDAL,
     serviceTokenDAL
   });
-  const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL });
+  const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore });
   const trustedIpService = trustedIpServiceFactory({
     licenseService,
     projectDAL,

backend/src/server/routes/v1/identity-ua.ts

@@ -39,11 +39,12 @@ export const registerIdentityUaRouter = async (server: FastifyZodProvider) => {
       }
     },
     handler: async (req) => {
-      const { identityUa, accessToken, identityAccessToken, validClientSecretInfo } =
+      const { identityUa, accessToken, identityAccessToken, validClientSecretInfo, identityMembershipOrg } =
         await server.services.identityUa.login(req.body.clientId, req.body.clientSecret, req.realIp);
 
       await server.services.auditLog.createAuditLog({
         ...req.auditLogInfo,
+        orgId: identityMembershipOrg?.orgId,
         event: {
           type: EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH,
           metadata: {

backend/src/services/identity-ua/identity-ua-service.ts

@@ -54,6 +54,8 @@ export const identityUaServiceFactory = ({
     const identityUa = await identityUaDAL.findOne({ clientId });
     if (!identityUa) throw new UnauthorizedError();
 
+    const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId });
+
     checkIPAgainstBlocklist({
       ipAddress: ip,
       trustedIps: identityUa.clientSecretTrustedIps as TIp[]
@@ -131,7 +133,7 @@ export const identityUaServiceFactory = ({
       }
     );
 
-    return { accessToken, identityUa, validClientSecretInfo, identityAccessToken };
+    return { accessToken, identityUa, validClientSecretInfo, identityAccessToken, identityMembershipOrg };
   };
 
   const attachUa = async ({

backend/src/services/secret/secret-service.ts

@@ -7,7 +7,7 @@ import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/
 import { getConfig } from "@app/lib/config/env";
 import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
-import { groupBy, pick } from "@app/lib/fn";
+import { groupBy, pick, unique } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 
 import { ActorType } from "../auth/auth-type";
@@ -202,12 +202,13 @@ export const secretServiceFactory = ({
     return deletedSecrets;
   };
 
-  // this is a utility function for secret modification
-  // this will check given secret name blind index exist or not
-  // if its a created secret set isNew to true
-  // thus if these blindindex exist it will throw an error
-  // vice versa when u need to check for updated secret
-  // this will also return the blind index grouped by secretName
+  /**
+   * Checks and handles secrets using a blind index method.
+   * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
+   * For new secrets (isNew = true), it ensures they don't already exist in the database.
+   * For existing secrets, it verifies their presence in the database.
+   * If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
+   */
   const fnSecretBlindIndexCheck = async ({
     inputSecrets,
     folderId,
@@ -242,10 +243,18 @@ export const secretServiceFactory = ({
 
     if (isNew) {
       if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
-    } else if (secrets.length !== inputSecrets.length)
-      throw new BadRequestError({
-        message: `Secret not found: blind index ${JSON.stringify(keyName2BlindIndex)}`
-      });
+    } else {
+      const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
+        (el) => blindIndex2KeyName[el.secretBlindIndex as string]
+      );
+      const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
+      if (hasUnknownSecretsProvided) {
+        const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
+        throw new BadRequestError({
+          message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
+        });
+      }
+    }
 
     return { blindIndex2KeyName, keyName2BlindIndex, secrets };
   };

cli/agent-config.yaml

@@ -1,5 +1,5 @@
 infisical:
-  address: "http://localhost:8080"
+  address: "https://app.infisical.com/"
 auth:
   type: "universal-auth"
   config:
@@ -13,3 +13,12 @@ sinks:
 templates:
   - source-path: my-dot-ev-secret-template
     destination-path: my-dot-env.env
+    config:
+      polling-interval: 60s
+      execute:
+        command: docker-compose -f docker-compose.prod.yml down && docker-compose -f docker-compose.prod.yml up -d
+  - source-path: my-dot-ev-secret-template1
+    destination-path: my-dot-env-1.env
+    config:
+      exec:
+        command: mkdir hello-world1

cli/packages/api/api.go

@@ -490,5 +490,7 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques
 		return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unsuccessful response [%v %v] [status-code=%v] [response=%v]", response.Request.Method, response.Request.URL, response.StatusCode(), response.String())
 	}
 
+	getRawSecretsV3Response.ETag = response.Header().Get(("etag"))
+
 	return getRawSecretsV3Response, nil
 }

cli/packages/api/model.go

@@ -505,4 +505,5 @@ type GetRawSecretsV3Response struct {
 		SecretComment string `json:"secretComment"`
 	} `json:"secrets"`
 	Imports []any `json:"imports"`
+	ETag		string
 }

cli/packages/cmd/agent.go

@@ -5,12 +5,15 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path"
+	"runtime"
 	"strings"
 	"sync"
 	"syscall"
@@ -71,12 +74,56 @@ type Template struct {
 	SourcePath            string `yaml:"source-path"`
 	Base64TemplateContent string `yaml:"base64-template-content"`
 	DestinationPath       string `yaml:"destination-path"`
+
+	Config struct { // Configurations for the template
+		PollingInterval string `yaml:"polling-interval"` // How often to poll for changes in the secret
+		Execute         struct {
+			Command string `yaml:"command"` // Command to execute once the template has been rendered
+			Timeout int64  `yaml:"timeout"` // Timeout for the command
+		} `yaml:"execute"` // Command to execute once the template has been rendered
+	} `yaml:"config"`
 }
 
 func ReadFile(filePath string) ([]byte, error) {
 	return ioutil.ReadFile(filePath)
 }
 
+func ExecuteCommandWithTimeout(command string, timeout int64) error {
+
+	shell := [2]string{"sh", "-c"}
+	if runtime.GOOS == "windows" {
+		shell = [2]string{"cmd", "/C"}
+	} else {
+		currentShell := os.Getenv("SHELL")
+		if currentShell != "" {
+			shell[0] = currentShell
+		}
+	}
+
+	ctx := context.Background()
+	if timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+		defer cancel()
+	}
+
+	cmd := exec.CommandContext(ctx, shell[0], shell[1], command)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok { // type assertion
+			if exitError.ProcessState.ExitCode() == -1 {
+				return fmt.Errorf("command timed out")
+			}
+		}
+		return err
+	} else {
+		return nil
+	}
+}
+
 func FileExists(filepath string) bool {
 	info, err := os.Stat(filepath)
 	if os.IsNotExist(err) {
@@ -170,20 +217,24 @@ func ParseAgentConfig(configFile []byte) (*Config, error) {
 	return config, nil
 }
 
-func secretTemplateFunction(accessToken string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) {
+func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) {
 	return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) {
-		secrets, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false)
+		res, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false)
 		if err != nil {
 			return nil, err
 		}
 
-		return secrets, nil
+		if existingEtag != res.Etag {
+			*currentEtag = res.Etag
+		}
+
+		return res.Secrets, nil
 	}
 }
 
-func ProcessTemplate(templatePath string, data interface{}, accessToken string) (*bytes.Buffer, error) {
+func ProcessTemplate(templatePath string, data interface{}, accessToken string, existingEtag string, currentEtag *string) (*bytes.Buffer, error) {
 	// custom template function to fetch secrets from Infisical
-	secretFunction := secretTemplateFunction(accessToken)
+	secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag)
 	funcs := template.FuncMap{
 		"secret": secretFunction,
 	}
@@ -203,7 +254,7 @@ func ProcessTemplate(templatePath string, data interface{}, accessToken string)
 	return &buf, nil
 }
 
-func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string) (*bytes.Buffer, error) {
+func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string, existingEtag string, currentEtag *string) (*bytes.Buffer, error) {
 	// custom template function to fetch secrets from Infisical
 	decoded, err := base64.StdEncoding.DecodeString(encodedTemplate)
 	if err != nil {
@@ -212,7 +263,7 @@ func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken
 
 	templateString := string(decoded)
 
-	secretFunction := secretTemplateFunction(accessToken)
+	secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) // TODO: Fix this
 	funcs := template.FuncMap{
 		"secret": secretFunction,
 	}
@@ -250,7 +301,16 @@ type TokenManager struct {
 }
 
 func NewTokenManager(fileDeposits []Sink, templates []Template, clientIdPath string, clientSecretPath string, newAccessTokenNotificationChan chan bool, removeClientSecretOnRead bool, exitAfterAuth bool) *TokenManager {
-	return &TokenManager{filePaths: fileDeposits, templates: templates, clientIdPath: clientIdPath, clientSecretPath: clientSecretPath, newAccessTokenNotificationChan: newAccessTokenNotificationChan, removeClientSecretOnRead: removeClientSecretOnRead, exitAfterAuth: exitAfterAuth}
+	return &TokenManager{
+		filePaths:                      fileDeposits,
+		templates:                      templates,
+		clientIdPath:                   clientIdPath,
+		clientSecretPath:               clientSecretPath,
+		newAccessTokenNotificationChan: newAccessTokenNotificationChan,
+		removeClientSecretOnRead:       removeClientSecretOnRead,
+		exitAfterAuth:                  exitAfterAuth,
+	}
+
 }
 
 func (tm *TokenManager) SetToken(token string, accessTokenTTL time.Duration, accessTokenMaxTTL time.Duration) {
@@ -428,38 +488,80 @@ func (tm *TokenManager) WriteTokenToFiles() {
 	}
 }
 
-func (tm *TokenManager) FetchSecrets() {
-	log.Info().Msgf("template engine started...")
+func (tm *TokenManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template) {
+	if err := WriteBytesToFile(bytes, template.DestinationPath); err != nil {
+		log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err)
+		return
+	}
+	log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", template.SourcePath, template.DestinationPath)
+}
+
+func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan chan os.Signal) {
+
+	pollingInterval := time.Duration(5 * time.Minute)
+
+	if secretTemplate.Config.PollingInterval != "" {
+		interval, err := util.ConvertPollingIntervalToTime(secretTemplate.Config.PollingInterval)
+
+		if err != nil {
+			log.Error().Msgf("unable to convert polling interval to time because %v", err)
+			sigChan <- syscall.SIGINT
+			return
+
+		} else {
+			pollingInterval = interval
+		}
+	}
+
+	var existingEtag string
+	var currentEtag string
+	var firstRun = true
+
+	execTimeout := secretTemplate.Config.Execute.Timeout
+	execCommand := secretTemplate.Config.Execute.Command
+
 	for {
 		token := tm.GetToken()
+
 		if token != "" {
-			for _, secretTemplate := range tm.templates {
-				var processedTemplate *bytes.Buffer
-				var err error
-				if secretTemplate.SourcePath != "" {
-					processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token)
-				} else {
-					processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token)
-				}
 
-				if err != nil {
-					log.Error().Msgf("template engine: unable to render secrets because %s. Will try again on next cycle", err)
+			var processedTemplate *bytes.Buffer
+			var err error
 
-					continue
-				}
-
-				if err := WriteBytesToFile(processedTemplate, secretTemplate.DestinationPath); err != nil {
-					log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err)
-
-					continue
-				}
-
-				log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", secretTemplate.SourcePath, secretTemplate.DestinationPath)
+			if secretTemplate.SourcePath != "" {
+				processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token, existingEtag, &currentEtag)
+			} else {
+				processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token, existingEtag, &currentEtag)
 			}
 
-			// fetch new secrets every 5 minutes (TODO: add PubSub in the future )
-			time.Sleep(5 * time.Minute)
+			if err != nil {
+				log.Error().Msgf("unable to process template because %v", err)
+			} else {
+				if (existingEtag != currentEtag) || firstRun {
+
+					tm.WriteTemplateToFile(processedTemplate, &secretTemplate)
+					existingEtag = currentEtag
+
+					if !firstRun && execCommand != "" {
+						log.Info().Msgf("executing command: %s", execCommand)
+						err := ExecuteCommandWithTimeout(execCommand, execTimeout)
+
+						if err != nil {
+							log.Error().Msgf("unable to execute command because %v", err)
+						}
+
+					}
+					if firstRun {
+						firstRun = false
+					}
+				}
+			}
+			time.Sleep(pollingInterval)
+		} else {
+			// It fails to get the access token. So we will re-try in 3 seconds. We do this because if we don't, the user will have to wait for the next polling interval to get the first secret render.
+			time.Sleep(3 * time.Second)
 		}
+
 	}
 }
 
@@ -544,7 +646,11 @@ var agentCmd = &cobra.Command{
 		tm := NewTokenManager(filePaths, agentConfig.Templates, configUniversalAuthType.ClientIDPath, configUniversalAuthType.ClientSecretPath, tokenRefreshNotifier, configUniversalAuthType.RemoveClientSecretOnRead, agentConfig.Infisical.ExitAfterAuth)
 
 		go tm.ManageTokenLifecycle()
-		go tm.FetchSecrets()
+
+		for i, template := range agentConfig.Templates {
+			log.Info().Msgf("template engine started for template %v...", i+1)
+			go tm.MonitorSecretChanges(template, sigChan)
+		}
 
 		for {
 			select {

cli/packages/models/cli.go

@@ -34,6 +34,11 @@ type SingleEnvironmentVariable struct {
 	Comment string `json:"comment"`
 }
 
+type PlaintextSecretResult struct {
+	Secrets []SingleEnvironmentVariable
+	Etag    string
+}
+
 type SingleFolder struct {
 	ID   string `json:"_id"`
 	Name string `json:"name"`

cli/packages/util/agent.go

@@ -0,0 +1,41 @@
+package util
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// ConvertPollingIntervalToTime converts a string representation of a polling interval to a time.Duration
+func ConvertPollingIntervalToTime(pollingInterval string) (time.Duration, error) {
+	length := len(pollingInterval)
+	if length < 2 {
+		return 0, fmt.Errorf("invalid format")
+	}
+
+	unit := pollingInterval[length-1:]
+	numberPart := pollingInterval[:length-1]
+
+	number, err := strconv.Atoi(numberPart)
+	if err != nil {
+		return 0, err
+	}
+
+	switch unit {
+	case "s":
+		if number < 60 {
+			return 0, fmt.Errorf("polling interval should be at least 60 seconds")
+		}
+		return time.Duration(number) * time.Second, nil
+	case "m":
+		return time.Duration(number) * time.Minute, nil
+	case "h":
+		return time.Duration(number) * time.Hour, nil
+	case "d":
+		return time.Duration(number) * 24 * time.Hour, nil
+	case "w":
+		return time.Duration(number) * 7 * 24 * time.Hour, nil
+	default:
+		return 0, fmt.Errorf("invalid time unit")
+	}
+}

cli/packages/util/secrets.go

@@ -152,7 +152,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work
 	return plainTextSecrets, nil
 }
 
-func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) ([]models.SingleEnvironmentVariable, error) {
+func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) (models.PlaintextSecretResult, error) {
 	httpClient := resty.New()
 	httpClient.SetAuthToken(accessToken).
 		SetHeader("Accept", "application/json")
@@ -170,12 +170,12 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin
 
 	rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{WorkspaceId: workspaceId, SecretPath: secretsPath, Environment: environmentName})
 	if err != nil {
-		return nil, err
+		return models.PlaintextSecretResult{}, err
 	}
 
 	plainTextSecrets := []models.SingleEnvironmentVariable{}
 	if err != nil {
-		return nil, fmt.Errorf("unable to decrypt your secrets [err=%v]", err)
+		return models.PlaintextSecretResult{}, fmt.Errorf("unable to decrypt your secrets [err=%v]", err)
 	}
 
 	for _, secret := range rawSecrets.Secrets {
@@ -189,7 +189,10 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin
 	// 	}
 	// }
 
-	return plainTextSecrets, nil
+	return models.PlaintextSecretResult{
+		Secrets: plainTextSecrets,
+		Etag:    rawSecrets.ETag,
+	}, nil
 }
 
 func InjectImportedSecret(plainTextWorkspaceKey []byte, secrets []models.SingleEnvironmentVariable, importedSecrets []api.ImportedSecretV3) ([]models.SingleEnvironmentVariable, error) {

docs/changelog/overview.mdx

@@ -4,7 +4,22 @@ title: "Changelog"
 
 The changelog below reflects new product developments and updates on a monthly basis.
 
-## January 2024
+## Feb 2024
+- Added org-scoped authentication enforcement for SAML
+- Added support for [SCIM](https://infisical.com/docs/documentation/platform/scim/overview) along with instructions for setting it up with [Okta](https://infisical.com/docs/documentation/platform/scim/okta), [Azure](https://infisical.com/docs/documentation/platform/scim/azure), and [JumpCloud](https://infisical.com/docs/documentation/platform/scim/jumpcloud).
+- Pushed out project update for non-E2EE w/ new endpoints like for project creation and member invitation.
+- Added API Integration testing for new backend.
+- Added capability to create projects in Terraform.
+- Added slug-based capabilities to both organizations and projects to gradually make the API more developer-friendly moving forward.
+- Fixed + improved various analytics/telemetry-related items.
+- Fixed various issues associated with the Python SDK: build during installation on Mac OS, Rust dependency.
+- Updated self-hosting documentation to reflect [new backend](https://infisical.com/docs/self-hosting/overview).
+- Released [Postgres-based Infisical helm chart](https://cloudsmith.io/~infisical/repos/helm-charts/packages/detail/helm/infisical-standalone/).
+- Added checks to ensure that breaking API changes don't get released.
+- Automated API reference documentation to be inline with latest releases of Infisical.
+
+## Jan 2024
+- Completed Postgres migration initiative with restructed Fastify-based backend.
 - Reduced size of Infisical Node.js SDK by ≈90%.
 - Added secret fallback support to all SDK's.
 - Added Machine Identity support to [Terraform Provider](https://github.com/Infisical/terraform-provider-infisical).
@@ -12,21 +27,21 @@ The changelog below reflects new product developments and updates on a monthly b
 - Added symmetric encryption support to all SDK's.
 - Fixed secret reminders bug, where reminders were not being updated correctly.
 
-## December 2023
+## Dec 2023
 
 - Released [(machine) identities](https://infisical.com/docs/documentation/platform/identities/overview) and [universal auth](https://infisical.com/docs/documentation/platform/identities/universal-auth) features.
 - Created new cross-language SDKs for [Python](https://infisical.com/docs/sdks/languages/python), [Node](https://infisical.com/docs/sdks/languages/node), and [Java](https://infisical.com/docs/sdks/languages/java). 
 - Released first version of the [Infisical Agent](https://infisical.com/docs/infisical-agent/overview)
 - Added ability to [manage folders via CLI](https://infisical.com/docs/cli/commands/secrets).  
 
-## November 2023
+## Nov 2023
 
 - Replaced internal [Winston](https://github.com/winstonjs/winston) with [Pino](https://github.com/pinojs/pino) logging library with external logging to AWS CloudWatch
 - Added admin panel to self-hosting experience.
 - Released [secret rotation](https://infisical.com/docs/documentation/platform/secret-rotation/overview) feature with preliminary support for rotating [SendGrid](https://infisical.com/docs/documentation/platform/secret-rotation/sendgrid), [PostgreSQL/CockroachDB](https://infisical.com/docs/documentation/platform/secret-rotation/postgres), and [MySQL/MariaDB](https://infisical.com/docs/documentation/platform/secret-rotation/mysql) credentials.
 - Released secret reminders feature.
 
-## October 2023
+## Oct 2023
 
 - Added support for [GitLab SSO](https://infisical.com/docs/documentation/platform/sso/gitlab).
 - Became SOC 2 (Type II) certified.
@@ -35,7 +50,7 @@ The changelog below reflects new product developments and updates on a monthly b
 - Added native [Hasura Cloud integration](https://infisical.com/docs/integrations/cloud/hasura-cloud).
 - Updated resource deletion logic for user, organization, and project deletion.
 
-## September 2023
+## Sep 2023
 
 - Released [secret approvals](https://infisical.com/docs/documentation/platform/pr-workflows) feature.
 - Released an update to access controls; every user role now clearly defines and enforces a certain set of conditions across Infisical.
@@ -43,7 +58,7 @@ The changelog below reflects new product developments and updates on a monthly b
 - Added a native integration with [Qovery](https://infisical.com/docs/integrations/cloud/qovery).
 - Added service token generation capability for the CLI.
 
-## August 2023
+## Aug 2023
 
 - Release Audit Logs V2.
 - Add support for [GitHub SSO](https://infisical.com/docs/documentation/platform/sso/github).
@@ -171,7 +186,7 @@ The changelog below reflects new product developments and updates on a monthly b
 - Added sorting capability to sort keys by name alphabetically in dashboard.
 - Added downloading secrets back as `.env` file capability.
 
-## August 2022
+## Aug 2022
 
 - Released first version of the Infisical platform with push/pull capability and end-to-end encryption.
 - Improved security handling of authentication tokens by storing refresh tokens in HttpOnly cookies.

docs/self-hosting/configuration/requirements.mdx

@@ -58,7 +58,7 @@ Redis requirements:
 
 - Use Redis versions 6.x or 7.x. We advise upgrading to at least Redis 6.2.
 - Redis Cluster mode is currently not supported; use Redis Standalone, with or without High Availability (HA).
-- Redis storage needs are minimal: a setup with 1 vCPU, 1 GB RAM, and 1GB SSD will be sufficient for most deployments.
+- Redis storage needs are minimal: a setup with 1 vCPU, 1 GB RAM, and 1GB SSD will be sufficient for small deployments.
 
 ## Supported Web Browsers
 
@@ -68,4 +68,4 @@ Infisical supports a range of web browsers. However, features such as browser-ba
 - [Google Chrome](https://www.google.com/chrome/) 
 - [Chromium](https://www.chromium.org/getting-involved/dev-channel/)
 - [Apple Safari](https://www.apple.com/safari/)
-- [Microsoft Edge](https://www.microsoft.com/en-us/edge?form=MA13FJ)
+- [Microsoft Edge](https://www.microsoft.com/en-us/edge?form=MA13FJ)

docs/self-hosting/deployment-options/kubernetes-helm.mdx

@@ -46,27 +46,34 @@ description: "Use Helm chart to install Infisical on your Kubernetes cluster"
 
     <Tabs>
       <Tab title="Proof of concept deployment">
-          For test or proof-of-concept purposes, you may omit `DB_CONNECTION_URI` and `REDIS_URL` from `infisical-secrets`. This is because the Helm chart will automatically provision and connect to the in-cluster instances of Postgres and Redis by default.
-
+        For test or proof-of-concept purposes, you may omit `DB_CONNECTION_URI` and `REDIS_URL` from `infisical-secrets`. This is because the Helm chart will automatically provision and connect to the in-cluster instances of Postgres and Redis by default.
+        ```yaml simple-values-example.yaml 
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: infisical-secrets
+        type: Opaque
+        stringData:
+          AUTH_SECRET: <>
+          ENCRYPTION_KEY: <>
+        ```
       </Tab>
       <Tab title="Production deployment">
         For production environments, we recommend using Cloud-based Platform as a Service (PaaS) solutions for PostgreSQL and Redis to ensure high availability. In on-premise setups, it's recommended to configure Redis and Postgres for high availability, either by using Bitnami charts or a custom configuration.
-
+        ```yaml simple-values-example.yaml 
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: infisical-secrets
+        type: Opaque
+        stringData:
+          AUTH_SECRET: <>
+          ENCRYPTION_KEY: <>
+          REDIS_URL: <>
+          DB_CONNECTION_URI: <>
+        ```
       </Tab>
     </Tabs>
-
-    ```yaml simple-values-example.yaml 
-      apiVersion: v1
-      kind: Secret
-      metadata:
-        name: infisical-secrets
-      type: Opaque
-      stringData:
-        AUTH_SECRET: <>
-        ENCRYPTION_KEY: <>
-        REDIS_URL: <>
-        DB_CONNECTION_URI: <>
-    ```
   </Step>
 
   <Step title="Database schema migration ">

docs/self-hosting/guides/mongo-to-postgres.mdx

@@ -14,6 +14,7 @@ The newly released Postgres version of Infisical is the only version of Infisica
 
 Before starting the migration, ensure you have the following command line tools installed:
 
+- [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
 - [pg_dump](https://www.postgresql.org/docs/current/app-pgrestore.html) 
 - [pg_restore](https://www.postgresql.org/docs/current/app-pgdump.html)
 - [mongodump](https://www.mongodb.com/docs/database-tools/mongodump/)
@@ -103,7 +104,7 @@ Once started, the migration script will transform MongoDB data into an equivalen
   <Step title="Clone Infisical Repository">
     Clone the Infisical MongoDB repository.
     ```
-    git clone https://github.com/Infisical/infisical.git
+    git clone -b infisical/v0.46.11-postgres https://github.com/Infisical/infisical.git
     ```
   </Step>
   <Step title="Install dependencies for backend">
@@ -186,9 +187,14 @@ Rather than transferring the data row-by-row from your local machine to the prod
 
 ## Post-Migration Steps
 
-After successfully migrating the data to PostgreSQL, you can proceed to deploy Infisical using your preferred deployment method. 
-Refer to [Infisical's self-hosting documentation](https://infisical.com/docs/self-hosting/overview) for deployment options. 
-Remember to use your production PostgreSQL connection string for the new deployment and transfer all [environment variables](/self-hosting/configuration/envars) from the MongoDB version of Infisical to the new version (they are all compatible).
+Once the data migration to PostgreSQL is complete, you're ready to deploy Infisical using the deployment method of your choice.
+For guidance on deployment options, please visit the [self-hosting documentation](/self-hosting/overview). 
+Remember to transfer the necessary [environment variables](/self-hosting/configuration/envars) from the MongoDB version of Infisical to the new Postgres based Infisical; rest assured, they are fully compatible.
+
+<Warning>
+The first deployment of Postgres based Infisical must be deployed with Docker image tag `v0.46.11-postgres`. 
+After deploying this version, you can proceed to update to any subsequent versions.
+</Warning>
 
 ## Additional discussion 
 - When you visit Infisical's [docker hub](https://hub.docker.com/r/infisical/infisical) page, you will notice that image tags end with `-postgres`. 

frontend/src/pages/integrations/cloudflare-workers/create.tsx

@@ -1,10 +1,12 @@
 import { useEffect, useState } from "react";
 import { useRouter } from "next/router";
+import axios from "axios";
 import queryString from "query-string";
 
+import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider";
 import { useCreateIntegration, useGetWorkspaceById } from "@app/hooks/api";
 
-import { Button, Card, CardTitle, FormControl, Select, SelectItem } from "../../../components/v2";
+import { Button, Card, CardTitle, FormControl, Input, Select, SelectItem } from "../../../components/v2";
 import {
   useGetIntegrationAuthApps,
   useGetIntegrationAuthById
@@ -13,6 +15,7 @@ import {
 export default function CloudflareWorkersIntegrationPage() {
   const router = useRouter();
   const { mutateAsync } = useCreateIntegration();
+  const { createNotification } = useNotificationContext();
 
   const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]);
   const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? "");
@@ -22,6 +25,8 @@ export default function CloudflareWorkersIntegrationPage() {
   });
 
   const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState("");
+  const [secretPath, setSecretPath] = useState("/");
+
   const [targetApp, setTargetApp] = useState("");
   const [targetAppId, setTargetAppId] = useState("");
 
@@ -56,7 +61,7 @@ export default function CloudflareWorkersIntegrationPage() {
         app: targetApp,
         appId: targetAppId,
         sourceEnvironment: selectedSourceEnvironment,
-        secretPath: "/"
+        secretPath
       });
 
       setIsLoading(false);
@@ -64,6 +69,18 @@ export default function CloudflareWorkersIntegrationPage() {
       router.push(`/integrations/${localStorage.getItem("projectData.id")}`);
     } catch (err) {
       console.error(err);
+
+      let errorMessage: string = "Something went wrong!";
+      if (axios.isAxiosError(err)) {
+        const { message } = err?.response?.data as { message: string };
+        errorMessage = message;
+      }
+
+      createNotification({
+        text: errorMessage,
+        type: "error"
+      });
+      setIsLoading(false);
     }
   };
 
@@ -96,6 +113,13 @@ export default function CloudflareWorkersIntegrationPage() {
             ))}
           </Select>
         </FormControl>
+        <FormControl label="Infisical Secret Path" className="mt-2 px-6">
+          <Input
+            value={secretPath}
+            onChange={(evt) => setSecretPath(evt.target.value)}
+            placeholder="Provide a path, default is /"
+          />
+        </FormControl>
         <FormControl label="Cloudflare Workers Project" className="mt-4 px-6">
           <Select
             value={targetApp}