Merge pull request #2636 from Infisical/daniel/envkey-refactor

feat: envkey import refactor
This commit is contained in:
Maidul Islam
2024-10-26 14:01:32 -04:00
committed by GitHub
3 changed files with 554 additions and 77 deletions

View File

@ -4,7 +4,7 @@ import sjcl from "sjcl";
import tweetnacl from "tweetnacl";
import tweetnaclUtil from "tweetnacl-util";
import { SecretType } from "@app/db/schemas";
import { SecretType, TSecretFolders } from "@app/db/schemas";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { chunkArray } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
@ -35,7 +35,7 @@ export type TImportDataIntoInfisicalDTO = {
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create">;
secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath">;
folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath" | "findById">;
projectService: Pick<TProjectServiceFactory, "createProject">;
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;
@ -67,6 +67,7 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
const infisicalImportData: InfisicalImportData = {
projects: [],
environments: [],
folders: [],
secrets: []
};
@ -80,25 +81,387 @@ export const parseEnvKeyDataFn = async (decryptedJson: string): Promise<Infisica
envTemplates.set(env.id, env.defaultName);
}
// environments
for (const env of parsedJson.baseEnvironments) {
infisicalImportData.environments.push({
id: env.id,
name: envTemplates.get(env.environmentRoleId)!,
projectId: env.envParentId
});
// custom base environments
for (const env of parsedJson.nonDefaultEnvironmentRoles) {
envTemplates.set(env.id, env.name);
}
// secrets
// environments
for (const env of parsedJson.baseEnvironments) {
const appId = parsedJson.apps.find((a) => a.id === env.envParentId)?.id;
// If we find the app from the envParentId, we know this is a root-level environment.
if (appId) {
infisicalImportData.environments.push({
id: env.id,
name: envTemplates.get(env.environmentRoleId)!,
projectId: appId
});
}
}
const findRootInheritedSecret = (
secret: { val?: string; inheritsEnvironmentId?: string },
secretName: string,
envs: typeof parsedJson.envs
): { val?: string } => {
if (!secret) {
return {
val: ""
};
}
// If we have a direct value, return it
if (secret.val !== undefined) {
return secret;
}
// If there's no inheritance, return the secret as is
if (!secret.inheritsEnvironmentId) {
return secret;
}
const inheritedEnv = envs[secret.inheritsEnvironmentId];
if (!inheritedEnv) return secret;
return findRootInheritedSecret(inheritedEnv.variables[secretName], secretName, envs);
};
const processBranches = () => {
for (const subEnv of parsedJson.subEnvironments) {
const app = parsedJson.apps.find((a) => a.id === subEnv.envParentId);
const block = parsedJson.blocks.find((b) => b.id === subEnv.envParentId);
if (app) {
// Handle regular app branches
const branchEnvironment = infisicalImportData.environments.find((e) => e.id === subEnv.parentEnvironmentId);
infisicalImportData.folders.push({
name: subEnv.subName,
parentFolderId: subEnv.parentEnvironmentId,
environmentId: branchEnvironment!.id,
id: subEnv.id
});
}
if (block) {
// Handle block branches
// 1. Find all apps that use this block
const appsUsingBlock = parsedJson.appBlocks.filter((ab) => ab.blockId === block.id);
for (const { appId, orderIndex } of appsUsingBlock) {
// 2. Find the matching environment in the app based on the environment role
const blockBaseEnv = parsedJson.baseEnvironments.find((be) => be.id === subEnv.parentEnvironmentId);
// eslint-disable-next-line no-continue
if (!blockBaseEnv) continue;
const matchingAppEnv = parsedJson.baseEnvironments.find(
(be) => be.envParentId === appId && be.environmentRoleId === blockBaseEnv.environmentRoleId
);
// eslint-disable-next-line no-continue
if (!matchingAppEnv) continue;
// 3. Create a folder in the matching app environment
infisicalImportData.folders.push({
name: subEnv.subName,
parentFolderId: matchingAppEnv.id,
environmentId: matchingAppEnv.id,
id: `${subEnv.id}-${appId}` // Create unique ID for each app's copy of the branch
});
// 4. Process secrets in the block branch for this app
const branchSecrets = parsedJson.envs[subEnv.id]?.variables || {};
for (const [secretName, secretData] of Object.entries(branchSecrets)) {
if (secretData.inheritsEnvironmentId) {
const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs);
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
(s) => s.name === secretName && s.environmentId === matchingAppEnv.id
);
if (preExistingSecretIndex !== -1) {
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
if (
preExistingSecret.appBlockOrderIndex !== undefined &&
orderIndex > preExistingSecret.appBlockOrderIndex
) {
// if the existing secret has a lower orderIndex, we should replace it
infisicalImportData.secrets[preExistingSecretIndex] = {
...preExistingSecret,
value: resolvedSecret.val || "",
appBlockOrderIndex: orderIndex
};
}
// eslint-disable-next-line no-continue
continue;
}
infisicalImportData.secrets.push({
id: randomUUID(),
name: secretName,
environmentId: matchingAppEnv.id,
value: resolvedSecret.val || "",
folderId: `${subEnv.id}-${appId}`,
appBlockOrderIndex: orderIndex
});
} else {
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
(s) => s.name === secretName && s.environmentId === matchingAppEnv.id
);
if (preExistingSecretIndex !== -1) {
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
if (
preExistingSecret.appBlockOrderIndex !== undefined &&
orderIndex > preExistingSecret.appBlockOrderIndex
) {
// if the existing secret has a lower orderIndex, we should replace it
infisicalImportData.secrets[preExistingSecretIndex] = {
...preExistingSecret,
value: secretData.val || "",
appBlockOrderIndex: orderIndex
};
}
// eslint-disable-next-line no-continue
continue;
}
infisicalImportData.secrets.push({
id: randomUUID(),
name: secretName,
environmentId: matchingAppEnv.id,
value: secretData.val || "",
folderId: `${subEnv.id}-${appId}`,
appBlockOrderIndex: orderIndex
});
}
}
}
}
}
};
const processBlocksForApp = (appIds: string[]) => {
for (const appId of appIds) {
const blocksInApp = parsedJson.appBlocks.filter((ab) => ab.appId === appId);
logger.info(
{
blocksInApp
},
"[processBlocksForApp]: Processing blocks for app"
);
for (const appBlock of blocksInApp) {
// 1. find all base environments for this block
const blockBaseEnvironments = parsedJson.baseEnvironments.filter((env) => env.envParentId === appBlock.blockId);
logger.info(
{
blockBaseEnvironments
},
"[processBlocksForApp]: Processing block base environments"
);
for (const blockBaseEnvironment of blockBaseEnvironments) {
// 2. find the corresponding environment that is not from the block
const matchingEnv = parsedJson.baseEnvironments.find(
(be) =>
be.environmentRoleId === blockBaseEnvironment.environmentRoleId && be.envParentId !== appBlock.blockId
);
if (!matchingEnv) {
throw new Error(`Could not find environment for block ${appBlock.blockId}`);
}
// 3. find all the secrets for this environment block
const blockSecrets = parsedJson.envs[blockBaseEnvironment.id].variables;
logger.info(
{
blockSecretsLength: Object.keys(blockSecrets).length
},
"[processBlocksForApp]: Processing block secrets"
);
// 4. process each secret
for (const secret of Object.keys(blockSecrets)) {
const selectedSecret = blockSecrets[secret];
if (selectedSecret.inheritsEnvironmentId) {
const resolvedSecret = findRootInheritedSecret(selectedSecret, secret, parsedJson.envs);
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
(s) => s.name === secret && s.environmentId === matchingEnv.id
);
if (preExistingSecretIndex !== -1) {
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
if (
preExistingSecret.appBlockOrderIndex !== undefined &&
appBlock.orderIndex > preExistingSecret.appBlockOrderIndex
) {
// if the existing secret has a lower orderIndex, we should replace it
infisicalImportData.secrets[preExistingSecretIndex] = {
...preExistingSecret,
value: selectedSecret.val || "",
appBlockOrderIndex: appBlock.orderIndex
};
}
// eslint-disable-next-line no-continue
continue;
}
infisicalImportData.secrets.push({
id: randomUUID(),
name: secret,
environmentId: matchingEnv.id,
value: resolvedSecret.val || "",
appBlockOrderIndex: appBlock.orderIndex
});
} else {
// If the secret already exists in the environment, we need to check the orderIndex of the appBlock. The appBlock with the highest orderIndex should take precedence.
const preExistingSecretIndex = infisicalImportData.secrets.findIndex(
(s) => s.name === secret && s.environmentId === matchingEnv.id
);
if (preExistingSecretIndex !== -1) {
const preExistingSecret = infisicalImportData.secrets[preExistingSecretIndex];
if (
preExistingSecret.appBlockOrderIndex !== undefined &&
appBlock.orderIndex > preExistingSecret.appBlockOrderIndex
) {
// if the existing secret has a lower orderIndex, we should replace it
infisicalImportData.secrets[preExistingSecretIndex] = {
...preExistingSecret,
value: selectedSecret.val || "",
appBlockOrderIndex: appBlock.orderIndex
};
}
// eslint-disable-next-line no-continue
continue;
}
infisicalImportData.secrets.push({
id: randomUUID(),
name: secret,
environmentId: matchingEnv.id,
value: selectedSecret.val || "",
appBlockOrderIndex: appBlock.orderIndex
});
}
}
}
}
}
};
processBranches();
processBlocksForApp(infisicalImportData.projects.map((app) => app.id));
for (const env of Object.keys(parsedJson.envs)) {
if (!env.includes("|")) {
const envData = parsedJson.envs[env];
for (const secret of Object.keys(envData.variables)) {
// Skip user-specific environments
// eslint-disable-next-line no-continue
if (env.includes("|")) continue;
const envData = parsedJson.envs[env];
const baseEnv = parsedJson.baseEnvironments.find((be) => be.id === env);
const subEnv = parsedJson.subEnvironments.find((se) => se.id === env);
// Skip if we can't find either a base environment or sub-environment
if (!baseEnv && !subEnv) {
logger.info(
{
envId: env
},
"[parseEnvKeyDataFn]: Could not find base or sub environment for env, skipping"
);
// eslint-disable-next-line no-continue
continue;
}
// If this is a base environment of a block, skip it (handled by processBlocksForApp)
if (baseEnv) {
const isBlock = parsedJson.appBlocks.some((block) => block.blockId === baseEnv.envParentId);
if (isBlock) {
logger.info(
{
envId: env,
baseEnv
},
"[parseEnvKeyDataFn]: Skipping block environment (handled separately)"
);
// eslint-disable-next-line no-continue
continue;
}
}
// Process each secret in this environment or branch
for (const [secretName, secretData] of Object.entries(envData.variables)) {
const environmentId = subEnv ? subEnv.parentEnvironmentId : env;
const indexOfExistingSecret = infisicalImportData.secrets.findIndex(
(s) => s.name === secretName && s.environmentId === environmentId
);
if (secretData.inheritsEnvironmentId) {
const resolvedSecret = findRootInheritedSecret(secretData, secretName, parsedJson.envs);
// Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block.
// Variables from the normal environment should take precedence over variables from the block.
if (indexOfExistingSecret !== -1) {
// if a existing secret is found, we should replace it directly
const newSecret: (typeof infisicalImportData.secrets)[number] = {
...infisicalImportData.secrets[indexOfExistingSecret],
value: resolvedSecret.val || ""
};
infisicalImportData.secrets[indexOfExistingSecret] = newSecret;
// eslint-disable-next-line no-continue
continue;
}
infisicalImportData.secrets.push({
id: randomUUID(),
name: secret,
environmentId: env,
value: envData.variables[secret].val
name: secretName,
environmentId: subEnv ? subEnv.parentEnvironmentId : env,
value: resolvedSecret.val || "",
...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret
});
} else {
// Check if there's already a secret with this name in the environment, if there is, we should override it. Because if there's already one, we know its coming from a block.
// Variables from the normal environment should take precedence over variables from the block.
if (indexOfExistingSecret !== -1) {
// if a existing secret is found, we should replace it directly
const newSecret: (typeof infisicalImportData.secrets)[number] = {
...infisicalImportData.secrets[indexOfExistingSecret],
value: secretData.val || ""
};
infisicalImportData.secrets[indexOfExistingSecret] = newSecret;
// eslint-disable-next-line no-continue
continue;
}
infisicalImportData.secrets.push({
id: randomUUID(),
name: secretName,
environmentId: subEnv ? subEnv.parentEnvironmentId : env,
value: secretData.val || "",
...(subEnv && { folderId: subEnv.id }) // Add folderId if this is a branch secret
});
}
}
@ -125,7 +488,17 @@ export const importDataIntoInfisicalFn = async ({
}
const originalToNewProjectId = new Map<string, string>();
const originalToNewEnvironmentId = new Map<string, string>();
const originalToNewEnvironmentId = new Map<
string,
{ envId: string; envSlug: string; rootFolderId: string; projectId: string }
>();
const originalToNewFolderId = new Map<
string,
{
folderId: string;
projectId: string;
}
>();
const projectsNotImported: string[] = [];
await projectDAL.transaction(async (tx) => {
@ -170,65 +543,161 @@ export const importDataIntoInfisicalFn = async ({
const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx);
const doc = await projectEnvDAL.create({ slug, name: environment.name, projectId, position: lastPos + 1 }, tx);
await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
const folder = await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx);
originalToNewEnvironmentId.set(environment.id, doc.slug);
originalToNewEnvironmentId.set(environment.id, {
envSlug: doc.slug,
envId: doc.id,
rootFolderId: folder.id,
projectId
});
}
}
if (data.folders) {
for await (const folder of data.folders) {
const parentEnv = originalToNewEnvironmentId.get(folder.parentFolderId as string);
if (!parentEnv) {
// eslint-disable-next-line no-continue
continue;
}
const newFolder = await folderDAL.create(
{
name: folder.name,
envId: parentEnv.envId,
parentId: parentEnv.rootFolderId
},
tx
);
originalToNewFolderId.set(folder.id, {
folderId: newFolder.id,
projectId: parentEnv.projectId
});
}
}
// Useful for debugging:
// console.log("data.secrets", data.secrets);
// console.log("data.folders", data.folders);
// console.log("data.environment", data.environments);
if (data.secrets && data.secrets.length > 0) {
const mappedToEnvironmentId = new Map<
string,
{
secretKey: string;
secretValue: string;
folderId?: string;
}[]
>();
for (const secret of data.secrets) {
if (!originalToNewEnvironmentId.get(secret.environmentId)) {
const targetId = secret.folderId || secret.environmentId;
// Skip if we can't find either an environment or folder mapping for this secret
if (!originalToNewEnvironmentId.get(secret.environmentId) && !originalToNewFolderId.get(targetId)) {
// eslint-disable-next-line no-continue
continue;
}
if (!mappedToEnvironmentId.has(secret.environmentId)) {
mappedToEnvironmentId.set(secret.environmentId, []);
if (!mappedToEnvironmentId.has(targetId)) {
mappedToEnvironmentId.set(targetId, []);
}
mappedToEnvironmentId.get(secret.environmentId)!.push({
mappedToEnvironmentId.get(targetId)!.push({
secretKey: secret.name,
secretValue: secret.value || ""
secretValue: secret.value || "",
folderId: secret.folderId
});
}
// for each of the mappedEnvironmentId
for await (const [envId, secrets] of mappedToEnvironmentId) {
const environment = data.environments.find((env) => env.id === envId);
const projectId = originalToNewProjectId.get(environment?.projectId as string)!;
for await (const [targetId, secrets] of mappedToEnvironmentId) {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for targetId", targetId);
if (!projectId) {
throw new BadRequestError({ message: `Failed to import secret, project not found` });
let selectedFolder: TSecretFolders | undefined;
let selectedProjectId: string | undefined;
// Case 1: Secret belongs to a folder / branch / branch of a block
const foundFolder = originalToNewFolderId.get(targetId);
if (foundFolder) {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for folder");
selectedFolder = await folderDAL.findById(foundFolder.folderId, tx);
selectedProjectId = foundFolder.projectId;
} else {
logger.info("[importDataIntoInfisicalFn]: Processing secrets for normal environment");
const environment = data.environments.find((env) => env.id === targetId);
if (!environment) {
logger.info(
{
targetId
},
"[importDataIntoInfisicalFn]: Could not find environment for secret"
);
// eslint-disable-next-line no-continue
continue;
}
const projectId = originalToNewProjectId.get(environment.projectId)!;
if (!projectId) {
throw new BadRequestError({ message: `Failed to import secret, project not found` });
}
const env = originalToNewEnvironmentId.get(targetId);
if (!env) {
logger.info(
{
targetId
},
"[importDataIntoInfisicalFn]: Could not find environment for secret"
);
// eslint-disable-next-line no-continue
continue;
}
const folder = await folderDAL.findBySecretPath(projectId, env.envSlug, "/", tx);
if (!folder) {
throw new NotFoundError({
message: `Folder not found for the given environment slug (${env.envSlug}) & secret path (/)`,
name: "Create secret"
});
}
selectedFolder = folder;
selectedProjectId = projectId;
}
if (!selectedFolder) {
throw new NotFoundError({
message: `Folder not found for the given environment slug & secret path`,
name: "CreateSecret"
});
}
if (!selectedProjectId) {
throw new NotFoundError({
message: `Project not found for the given environment slug & secret path`,
name: "CreateSecret"
});
}
const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(
{
type: KmsDataKey.SecretManager,
projectId
projectId: selectedProjectId
},
tx
);
const envSlug = originalToNewEnvironmentId.get(envId)!;
const folder = await folderDAL.findBySecretPath(projectId, envSlug, "/", tx);
if (!folder)
throw new NotFoundError({
message: `Folder not found for the given environment slug (${envSlug}) & secret path (/)`,
name: "Create secret"
});
const secretBatches = chunkArray(secrets, 2500);
for await (const secretBatch of secretBatches) {
const secretsByKeys = await secretDAL.findBySecretKeys(
folder.id,
selectedFolder.id,
secretBatch.map((el) => ({
key: el.secretKey,
type: SecretType.Shared
@ -254,7 +723,7 @@ export const importDataIntoInfisicalFn = async ({
type: SecretType.Shared
};
}),
folderId: folder.id,
folderId: selectedFolder.id,
secretDAL,
secretVersionDAL,
secretTagDAL,

View File

@ -31,7 +31,7 @@ export type TExternalMigrationQueueFactoryDep = {
secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "create">;
secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany" | "create">;
folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath">;
folderDAL: Pick<TSecretFolderDALFactory, "create" | "findBySecretPath" | "findOne" | "findById">;
projectService: Pick<TProjectServiceFactory, "createProject">;
projectEnvService: Pick<TProjectEnvServiceFactory, "createEnvironment">;
secretV2BridgeService: Pick<TSecretV2BridgeServiceFactory, "createManySecret">;

View File

@ -2,8 +2,16 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type";
export type InfisicalImportData = {
projects: Array<{ name: string; id: string }>;
environments: Array<{ name: string; id: string; projectId: string }>;
secrets: Array<{ name: string; id: string; environmentId: string; value: string }>;
environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }>;
folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }>;
secrets: Array<{
id: string;
name: string;
environmentId: string;
value: string;
folderId?: string;
appBlockOrderIndex?: number; // Not used for infisical import, only used for building the import structure to determine which block(s) take precedence.
}>;
};
export type TImportEnvKeyDataCreate = {
@ -28,62 +36,62 @@ export type TEnvKeyExportJSON = {
org: {
id: string;
name: string;
settings: {
auth: {
inviteExpirationMs: number;
deviceGrantExpirationMs: number;
tokenExpirationMs: number;
};
crypto: {
requiresPassphrase: boolean;
requiresLockout: boolean;
};
envs: {
autoCaps: boolean;
autoCommitLocals: boolean;
};
};
};
// Apps are projects
apps: {
id: string;
name: string;
settings: Record<string, unknown>;
}[];
defaultOrgRoles: {
// Blocks are basically global projects that can be imported in other projects
blocks: {
id: string;
defaultName: string;
name: string;
}[];
defaultAppRoles: {
id: string;
defaultName: string;
appBlocks: {
appId: string;
blockId: string;
orderIndex: number;
}[];
defaultEnvironmentRoles: {
id: string;
defaultName: string;
settings: {
autoCommit: boolean;
};
}[];
nonDefaultEnvironmentRoles: {
id: string;
name: string;
}[];
baseEnvironments: {
id: string;
envParentId: string;
environmentRoleId: string;
settings: Record<string, unknown>;
}[];
orgUsers: {
// Branches for both blocks and apps
subEnvironments: {
id: string;
firstName: string;
lastName: string;
email: string;
provider: string;
orgRoleId: string;
uid: string;
envParentId: string;
environmentRoleId: string;
parentEnvironmentId: string;
subName: string;
}[];
envs: Record<
string,
{
variables: Record<string, { val: string }>;
inherits: Record<string, unknown>;
variables: Record<
string,
{
val?: string;
inheritsEnvironmentId?: string;
}
>;
inherits: Record<string, string[]>;
}
>;
};