Feat: Silent integration errors

docs(sdks): Updated Go SDK docs

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -18,7 +18,7 @@ import {
   UpdateSecretCommand
 } from "@aws-sdk/client-secrets-manager";
 import { Octokit } from "@octokit/rest";
-import AWS from "aws-sdk";
+import AWS, { AWSError } from "aws-sdk";
 import { AxiosError } from "axios";
 import sodium from "libsodium-wrappers";
 import isEqual from "lodash.isequal";
@@ -452,7 +452,11 @@ const syncSecretsAWSParameterStore = async ({
   accessId: string | null;
   accessToken: string;
 }) => {
-  if (!accessId) return;
+  let response: { isSynced: boolean; syncMessage: string } | null = null;
+
+  if (!accessId) {
+    throw new Error("AWS access ID is required");
+  }
 
   const config = new AWS.Config({
     region: integration.region as string,
@@ -557,6 +561,11 @@ const syncSecretsAWSParameterStore = async ({
                 `AWS Parameter Store Error [integration=${integration.id}]: double check AWS account permissions (refer to the Infisical docs)`
               );
             }
+
+            response = {
+              isSynced: false,
+              syncMessage: (err as AWSError)?.message || "Error syncing with AWS Parameter Store"
+            };
           }
         }
       }
@@ -585,6 +594,8 @@ const syncSecretsAWSParameterStore = async ({
       }
     }
   }
+
+  return response;
 };
 
 /**
@@ -603,7 +614,9 @@ const syncSecretsAWSSecretManager = async ({
 }) => {
   const metadata = z.record(z.any()).parse(integration.metadata || {});
 
-  if (!accessId) return;
+  if (!accessId) {
+    throw new Error("AWS access ID is required");
+  }
 
   const secretsManager = new SecretsManagerClient({
     region: integration.region as string,
@@ -722,7 +735,7 @@ const syncSecretsAWSSecretManager = async ({
         }
       }
     } catch (err) {
-      // case when AWS manager can't find the specified secret
+      // case 1: when AWS manager can't find the specified secret
       if (err instanceof ResourceNotFoundException && secretsManager) {
         await secretsManager.send(
           new CreateSecretCommand({
@@ -734,6 +747,9 @@ const syncSecretsAWSSecretManager = async ({
               : []
           })
         );
+        // case 2: something unexpected went wrong, so we'll throw the error to reflect the error in the integration sync status
+      } else {
+        throw err;
       }
     }
   };
@@ -753,14 +769,12 @@ const syncSecretsAWSSecretManager = async ({
 const syncSecretsHeroku = async ({
   createManySecretsRawFn,
   updateManySecretsRawFn,
-  integrationDAL,
   integration,
   secrets,
   accessToken
 }: {
   createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
   updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise<Array<TSecrets & { _id: string }>>;
-  integrationDAL: Pick<TIntegrationDALFactory, "updateById">;
   integration: TIntegrations & {
     projectId: string;
     environment: {
@@ -862,10 +876,6 @@ const syncSecretsHeroku = async ({
       }
     }
   );
-
-  await integrationDAL.updateById(integration.id, {
-    lastUsed: new Date()
-  });
 };
 
 /**
@@ -2656,7 +2666,9 @@ const syncSecretsHashiCorpVault = async ({
   accessId: string | null;
   accessToken: string;
 }) => {
-  if (!accessId) return;
+  if (!accessId) {
+    throw new Error("Access ID is required");
+  }
 
   interface LoginAppRoleRes {
     auth: {
@@ -3486,6 +3498,8 @@ export const syncIntegrationSecrets = async ({
   accessToken: string;
   appendices?: { prefix: string; suffix: string };
 }) => {
+  let response: { isSynced: boolean; syncMessage: string } | null = null;
+
   switch (integration.integration) {
     case Integrations.GCP_SECRET_MANAGER:
       await syncSecretsGCPSecretManager({
@@ -3502,7 +3516,7 @@ export const syncIntegrationSecrets = async ({
       });
       break;
     case Integrations.AWS_PARAMETER_STORE:
-      await syncSecretsAWSParameterStore({
+      response = await syncSecretsAWSParameterStore({
         integration,
         secrets,
         accessId,
@@ -3521,7 +3535,6 @@ export const syncIntegrationSecrets = async ({
       await syncSecretsHeroku({
         createManySecretsRawFn,
         updateManySecretsRawFn,
-        integrationDAL,
         integration,
         secrets,
         accessToken
@@ -3727,4 +3740,6 @@ export const syncIntegrationSecrets = async ({
     default:
       throw new BadRequestError({ message: "Invalid integration" });
   }
+
+  return response;
 };

backend/src/services/secret/secret-queue.ts

@@ -421,94 +421,88 @@ export const secretQueueFactory = ({
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
-      logger.error(new Error("Secret path not found"));
-      return;
+      throw new Error("Secret path not found");
     }
 
-    // start syncing all linked imports also
-    if (depth < MAX_SYNC_SECRET_DEPTH) {
-      // find all imports made with the given environment and secret path
-      const linkSourceDto = {
-        projectId,
-        importEnv: folder.environment.id,
-        importPath: secretPath,
-        isReplication: false
-      };
-      const imports = await secretImportDAL.find(linkSourceDto);
+    // find all imports made with the given environment and secret path
+    const linkSourceDto = {
+      projectId,
+      importEnv: folder.environment.id,
+      importPath: secretPath,
+      isReplication: false
+    };
+    const imports = await secretImportDAL.find(linkSourceDto);
 
-      if (imports.length) {
-        // keep calling sync secret for all the imports made
-        const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
-        const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
-        const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
-        logger.info(
-          `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
-        );
-        await Promise.all(
-          imports
-            .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string))
-            // filter out already synced ones
-            .filter(
-              ({ folderId }) =>
-                !deDupeQueue[
-                  uniqueSecretQueueKey(
-                    foldersGroupedById[folderId][0]?.environmentSlug as string,
-                    foldersGroupedById[folderId][0]?.path as string
-                  )
-                ]
-            )
-            .map(({ folderId }) =>
-              syncSecrets({
-                projectId,
-                secretPath: foldersGroupedById[folderId][0]?.path as string,
-                environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
-                _deDupeQueue: deDupeQueue,
-                _depth: depth + 1,
-                excludeReplication: true
-              })
-            )
-        );
-      }
-
-      const secretReferences = await secretDAL.findReferencedSecretReferences(
-        projectId,
-        folder.environment.slug,
-        secretPath
+    if (imports.length) {
+      // keep calling sync secret for all the imports made
+      const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
+      const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
+      const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
+      logger.info(
+        `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+      );
+      await Promise.all(
+        imports
+          .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string))
+          // filter out already synced ones
+          .filter(
+            ({ folderId }) =>
+              !deDupeQueue[
+                uniqueSecretQueueKey(
+                  foldersGroupedById[folderId][0]?.environmentSlug as string,
+                  foldersGroupedById[folderId][0]?.path as string
+                )
+              ]
+          )
+          .map(({ folderId }) =>
+            syncSecrets({
+              projectId,
+              secretPath: foldersGroupedById[folderId][0]?.path as string,
+              environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string,
+              _deDupeQueue: deDupeQueue,
+              _depth: depth + 1,
+              excludeReplication: true
+            })
+          )
+      );
+    }
+
+    const secretReferences = await secretDAL.findReferencedSecretReferences(
+      projectId,
+      folder.environment.slug,
+      secretPath
+    );
+    if (secretReferences.length) {
+      const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
+      const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
+      const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string);
+      logger.info(
+        `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+      );
+      await Promise.all(
+        secretReferences
+          .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path))
+          // filter out already synced ones
+          .filter(
+            ({ folderId }) =>
+              !deDupeQueue[
+                uniqueSecretQueueKey(
+                  referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
+                  referencedFoldersGroupedById[folderId][0]?.path as string
+                )
+              ]
+          )
+          .map(({ folderId }) =>
+            syncSecrets({
+              projectId,
+              secretPath: referencedFoldersGroupedById[folderId][0]?.path as string,
+              environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
+              _deDupeQueue: deDupeQueue,
+              _depth: depth + 1,
+              excludeReplication: true
+            })
+          )
       );
-      if (secretReferences.length) {
-        const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
-        const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
-        const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string);
-        logger.info(
-          `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
-        );
-        await Promise.all(
-          secretReferences
-            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path))
-            // filter out already synced ones
-            .filter(
-              ({ folderId }) =>
-                !deDupeQueue[
-                  uniqueSecretQueueKey(
-                    referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
-                    referencedFoldersGroupedById[folderId][0]?.path as string
-                  )
-                ]
-            )
-            .map(({ folderId }) =>
-              syncSecrets({
-                projectId,
-                secretPath: referencedFoldersGroupedById[folderId][0]?.path as string,
-                environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string,
-                _deDupeQueue: deDupeQueue,
-                _depth: depth + 1,
-                excludeReplication: true
-              })
-            )
-        );
-      }
-    } else {
-      logger.info(`getIntegrationSecrets: Secret depth exceeded for [projectId=${projectId}] [folderId=${folder.id}]`);
     }
 
     const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment
@@ -550,7 +544,7 @@ export const secretQueueFactory = ({
       }
 
       try {
-        await syncIntegrationSecrets({
+        const response = await syncIntegrationSecrets({
           createManySecretsRawFn,
           updateManySecretsRawFn,
           integrationDAL,
@@ -568,13 +562,15 @@ export const secretQueueFactory = ({
         await integrationDAL.updateById(integration.id, {
           lastSyncJobId: job.id,
           lastUsed: new Date(),
-          syncMessage: "",
-          isSynced: true
+          syncMessage: response?.syncMessage ?? "",
+          isSynced: response?.isSynced ?? true
         });
-      } catch (err: unknown) {
+      } catch (err) {
         logger.info("Secret integration sync error: %o", err);
+
         const message =
-          err instanceof AxiosError ? JSON.stringify((err as AxiosError)?.response?.data) : (err as Error)?.message;
+          (err instanceof AxiosError ? JSON.stringify(err?.response?.data) : (err as Error)?.message) ||
+          "Unknown error occurred.";
 
         await integrationDAL.updateById(integration.id, {
           lastSyncJobId: job.id,

docs/changelog/overview.mdx

@@ -4,6 +4,25 @@ title: "Changelog"
 
 The changelog below reflects new product developments and updates on a monthly basis.
 
+## May 2024
+- Released [AWS](https://infisical.com/docs/documentation/platform/identities/aws-auth), [GCP](https://infisical.com/docs/documentation/platform/identities/gcp-auth), [Azure](https://infisical.com/docs/documentation/platform/identities/azure-auth), and [Kubernetes](https://infisical.com/docs/documentation/platform/identities/kubernetes-auth) Native Auth Methods.
+- Added [Secret Sharing](https://infisical.com/docs/documentation/platform/secret-sharing) functionality for sharing sensitive data through encrypted links – within and outside of an organization.
+- Updated [Secret Referencing](https://infisical.com/docs/documentation/platform/secret-reference) to be supported in all Infisical clients. Infisical UI is now able to provide automatic reference suggestions when typing. 
+- Released new [Infisical Jenkins Plugin](https://infisical.com/docs/integrations/cicd/jenkins). 
+- Added statuses and manual sync option to integrations in the Dashboard UI. 
+- Released universal [Audit Log Streaming](https://infisical.com/docs/documentation/platform/audit-log-streams). 
+- Added [Dynamic Secret template for AWS IAM](https://infisical.com/docs/documentation/platform/dynamic-secrets/aws-iam). 
+- Added support for syncing tags and custom KMS keys to [AWS Secrets Manager](https://infisical.com/docs/integrations/cloud/aws-secret-manager) and [Parameter Store](https://infisical.com/docs/integrations/cloud/aws-parameter-store) Integrations.
+- Officially released Infisical on [AWS Marketplace](https://infisical.com/blog/infisical-launches-on-aws-marketplace).
+
+## April 2024
+- Added [Access Requests](https://infisical.com/docs/documentation/platform/access-controls/access-requests) as part of self-serve secrets management workflows. 
+- Added [Temporary Access Provisioning](https://infisical.com/docs/documentation/platform/access-controls/temporary-access) for roles and additional privileges. 
+
+## May 2024
+- Released support for [Dynamic Secrets](https://infisical.com/docs/documentation/platform/dynamic-secrets/overview).
+- Released the concept of [Additional Privileges](https://infisical.com/docs/documentation/platform/access-controls/additional-privileges) on top of user/machine roles. 
+
 ## Feb 2024
 - Added org-scoped authentication enforcement for SAML
 - Added support for [SCIM](https://infisical.com/docs/documentation/platform/scim/overview) along with instructions for setting it up with [Okta](https://infisical.com/docs/documentation/platform/scim/okta), [Azure](https://infisical.com/docs/documentation/platform/scim/azure), and [JumpCloud](https://infisical.com/docs/documentation/platform/scim/jumpcloud).