Merge pull request #1513 from akhilmhdh/fix/delay-audit-log

feat(server): moved back audit log to queue now with keystore license

backend/package-lock.json

@@ -47,7 +47,6 @@
         "lodash.isequal": "^4.5.0",
         "mysql2": "^3.9.1",
         "nanoid": "^5.0.4",
-        "node-cache": "^5.1.2",
         "nodemailer": "^6.9.9",
         "ora": "^7.0.1",
         "passport-github": "^1.1.0",
@@ -5706,14 +5705,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/clone": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-      "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
     "node_modules/cluster-key-slot": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
@@ -9258,17 +9249,6 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
       "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
     },
-    "node_modules/node-cache": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz",
-      "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==",
-      "dependencies": {
-        "clone": "2.x"
-      },
-      "engines": {
-        "node": ">= 8.0.0"
-      }
-    },
     "node_modules/node-fetch": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",

backend/package.json

@@ -108,7 +108,6 @@
     "lodash.isequal": "^4.5.0",
     "mysql2": "^3.9.1",
     "nanoid": "^5.0.4",
-    "node-cache": "^5.1.2",
     "nodemailer": "^6.9.9",
     "ora": "^7.0.1",
     "passport-github": "^1.1.0",

backend/src/ee/services/audit-log/audit-log-queue.ts

@@ -24,7 +24,7 @@ export const auditLogQueueServiceFactory = ({
   const pushToLog = async (data: TCreateAuditLogDTO) => {
     await queueService.queue(QueueName.AuditLog, QueueJobs.AuditLog, data, {
       removeOnFail: {
-        count: 5
+        count: 3
       },
       removeOnComplete: true
     });
@@ -46,6 +46,7 @@ export const auditLogQueueServiceFactory = ({
     const ttl = plan.auditLogsRetentionDays * MS_IN_DAY;
     // skip inserting if audit log retention is 0 meaning its not supported
     if (ttl === 0) return;
+
     await auditLogDAL.create({
       actor: actor.type,
       actorMetadata: actor.metadata,

backend/src/ee/services/license/license-service.ts

@@ -5,8 +5,8 @@
 // TODO(akhilmhdh): With tony find out the api structure and fill it here
 
 import { ForbiddenError } from "@casl/ability";
-import NodeCache from "node-cache";
 
+import { TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { BadRequestError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
@@ -39,6 +39,7 @@ type TLicenseServiceFactoryDep = {
   orgDAL: Pick<TOrgDALFactory, "findOrgById">;
   permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
   licenseDAL: TLicenseDALFactory;
+  keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "deleteItem">;
 };
 
 export type TLicenseServiceFactory = ReturnType<typeof licenseServiceFactory>;
@@ -46,12 +47,18 @@ export type TLicenseServiceFactory = ReturnType<typeof licenseServiceFactory>;
 const LICENSE_SERVER_CLOUD_LOGIN = "/api/auth/v1/license-server-login";
 const LICENSE_SERVER_ON_PREM_LOGIN = "/api/auth/v1/license-login";
 
-const FEATURE_CACHE_KEY = (orgId: string, projectId?: string) => `${orgId}-${projectId || ""}`;
-export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }: TLicenseServiceFactoryDep) => {
+const LICENSE_SERVER_CLOUD_PLAN_TTL = 60; // 60s
+const FEATURE_CACHE_KEY = (orgId: string) => `infisical-cloud-plan-${orgId}`;
+
+export const licenseServiceFactory = ({
+  orgDAL,
+  permissionService,
+  licenseDAL,
+  keyStore
+}: TLicenseServiceFactoryDep) => {
   let isValidLicense = false;
   let instanceType = InstanceType.OnPrem;
   let onPremFeatures: TFeatureSet = getDefaultOnPremFeatures();
-  const featureStore = new NodeCache({ stdTTL: 60 });
 
   const appCfg = getConfig();
   const licenseServerCloudApi = setupLicenceRequestWithStore(
@@ -75,6 +82,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
         isValidLicense = true;
         return;
       }
+
       if (appCfg.LICENSE_KEY) {
         const token = await licenseServerOnPremApi.refreshLicence();
         if (token) {
@@ -100,22 +108,21 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
     logger.info(`getPlan: attempting to fetch plan for [orgId=${orgId}] [projectId=${projectId}]`);
     try {
       if (instanceType === InstanceType.Cloud) {
-        const cachedPlan = featureStore.get<TFeatureSet>(FEATURE_CACHE_KEY(orgId, projectId));
-        if (cachedPlan) return cachedPlan;
+        const cachedPlan = await keyStore.getItem(FEATURE_CACHE_KEY(orgId));
+        if (cachedPlan) return JSON.parse(cachedPlan) as TFeatureSet;
 
         const org = await orgDAL.findOrgById(orgId);
         if (!org) throw new BadRequestError({ message: "Org not found" });
         const {
           data: { currentPlan }
         } = await licenseServerCloudApi.request.get<{ currentPlan: TFeatureSet }>(
-          `/api/license-server/v1/customers/${org.customerId}/cloud-plan`,
-          {
-            params: {
-              workspaceId: projectId
-            }
-          }
+          `/api/license-server/v1/customers/${org.customerId}/cloud-plan`
+        );
+        await keyStore.setItemWithExpiry(
+          FEATURE_CACHE_KEY(org.id),
+          LICENSE_SERVER_CLOUD_PLAN_TTL,
+          JSON.stringify(currentPlan)
         );
-        featureStore.set(FEATURE_CACHE_KEY(org.id, projectId), currentPlan);
         return currentPlan;
       }
     } catch (error) {
@@ -123,15 +130,20 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
         `getPlan: encountered an error when fetching pan [orgId=${orgId}] [projectId=${projectId}] [error]`,
         error
       );
+      await keyStore.setItemWithExpiry(
+        FEATURE_CACHE_KEY(orgId),
+        LICENSE_SERVER_CLOUD_PLAN_TTL,
+        JSON.stringify(onPremFeatures)
+      );
       return onPremFeatures;
     }
     return onPremFeatures;
   };
 
-  const refreshPlan = async (orgId: string, projectId?: string) => {
+  const refreshPlan = async (orgId: string) => {
     if (instanceType === InstanceType.Cloud) {
-      featureStore.del(FEATURE_CACHE_KEY(orgId, projectId));
-      await getPlan(orgId, projectId);
+      await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
+      await getPlan(orgId);
     }
   };
 
@@ -166,7 +178,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
           quantity: count
         });
       }
-      featureStore.del(orgId);
+      await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
     } else if (instanceType === InstanceType.EnterpriseOnPrem) {
       const usedSeats = await licenseDAL.countOfOrgMembers(null);
       await licenseServerOnPremApi.request.patch(`/api/license/v1/license`, { usedSeats });
@@ -215,7 +227,7 @@ export const licenseServiceFactory = ({ orgDAL, permissionService, licenseDAL }:
       `/api/license-server/v1/customers/${organization.customerId}/session/trial`,
       { success_url }
     );
-    featureStore.del(FEATURE_CACHE_KEY(orgId));
+    await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
     return { url };
   };
 

backend/src/server/routes/index.ts

@@ -194,7 +194,7 @@ export const registerRoutes = async (
     projectRoleDAL,
     serviceTokenDAL
   });
-  const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL });
+  const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore });
   const trustedIpService = trustedIpServiceFactory({
     licenseService,
     projectDAL,

backend/src/server/routes/v1/identity-ua.ts

@@ -39,11 +39,12 @@ export const registerIdentityUaRouter = async (server: FastifyZodProvider) => {
       }
     },
     handler: async (req) => {
-      const { identityUa, accessToken, identityAccessToken, validClientSecretInfo } =
+      const { identityUa, accessToken, identityAccessToken, validClientSecretInfo, identityMembershipOrg } =
         await server.services.identityUa.login(req.body.clientId, req.body.clientSecret, req.realIp);
 
       await server.services.auditLog.createAuditLog({
         ...req.auditLogInfo,
+        orgId: identityMembershipOrg?.orgId,
         event: {
           type: EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH,
           metadata: {

backend/src/services/identity-ua/identity-ua-service.ts

@@ -54,6 +54,8 @@ export const identityUaServiceFactory = ({
     const identityUa = await identityUaDAL.findOne({ clientId });
     if (!identityUa) throw new UnauthorizedError();
 
+    const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId });
+
     checkIPAgainstBlocklist({
       ipAddress: ip,
       trustedIps: identityUa.clientSecretTrustedIps as TIp[]
@@ -131,7 +133,7 @@ export const identityUaServiceFactory = ({
       }
     );
 
-    return { accessToken, identityUa, validClientSecretInfo, identityAccessToken };
+    return { accessToken, identityUa, validClientSecretInfo, identityAccessToken, identityMembershipOrg };
   };
 
   const attachUa = async ({

cli/packages/cmd/agent.go

@@ -543,6 +543,7 @@ func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan ch
 					existingEtag = currentEtag
 
 					if !firstRun && execCommand != "" {
+						log.Info().Msgf("executing command: %s", execCommand)
 						err := ExecuteCommandWithTimeout(execCommand, execTimeout)
 
 						if err != nil {

cli/packages/util/secrets.go

@@ -191,7 +191,7 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin
 
 	return models.PlaintextSecretResult{
 		Secrets: plainTextSecrets,
-		Hash:    rawSecrets.ETag,
+		Etag:    rawSecrets.ETag,
 	}, nil
 }
 

docs/self-hosting/configuration/requirements.mdx

@@ -58,7 +58,7 @@ Redis requirements:
 
 - Use Redis versions 6.x or 7.x. We advise upgrading to at least Redis 6.2.
 - Redis Cluster mode is currently not supported; use Redis Standalone, with or without High Availability (HA).
-- Redis storage needs are minimal: a setup with 1 vCPU, 1 GB RAM, and 1GB SSD will be sufficient for most deployments.
+- Redis storage needs are minimal: a setup with 1 vCPU, 1 GB RAM, and 1GB SSD will be sufficient for small deployments.
 
 ## Supported Web Browsers
 
@@ -68,4 +68,4 @@ Infisical supports a range of web browsers. However, features such as browser-ba
 - [Google Chrome](https://www.google.com/chrome/) 
 - [Chromium](https://www.chromium.org/getting-involved/dev-channel/)
 - [Apple Safari](https://www.apple.com/safari/)
-- [Microsoft Edge](https://www.microsoft.com/en-us/edge?form=MA13FJ)
+- [Microsoft Edge](https://www.microsoft.com/en-us/edge?form=MA13FJ)