Compare commits

..

3 Commits

Author SHA1 Message Date
Maidul Islam
e6f42e1231 Merge pull request #1933 from Infisical/add-folder-sorting
added sorting for folders in overview
2024-06-08 22:31:07 -04:00
Vladyslav Matsiiako
06e7a90a44 added sorting for folders in overview 2024-06-08 22:26:49 -04:00
Maidul Islam
f075ff23a9 patch encoding type for kms 2024-06-08 18:38:25 -04:00
18 changed files with 15 additions and 437 deletions

View File

@@ -1,21 +0,0 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit");
await knex.schema.alterTable(TableName.Project, (tb) => {
if (!hasPitVersionLimitColumn) {
tb.integer("pitVersionLimit").notNullable().defaultTo(10);
}
});
}
export async function down(knex: Knex): Promise<void> {
const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit");
await knex.schema.alterTable(TableName.Project, (tb) => {
if (hasPitVersionLimitColumn) {
tb.dropColumn("pitVersionLimit");
}
});
}

View File

@@ -16,8 +16,7 @@ export const ProjectsSchema = z.object({
createdAt: z.date(),
updatedAt: z.date(),
version: z.number().default(1),
upgradeStatus: z.string().nullable().optional(),
pitVersionLimit: z.number().default(10)
upgradeStatus: z.string().nullable().optional()
});
export type TProjects = z.infer<typeof ProjectsSchema>;

View File

@@ -81,7 +81,8 @@ export const secretSnapshotServiceFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, path);
if (!folder) throw new BadRequestError({ message: "Folder not found" });
return snapshotDAL.countOfSnapshotsByFolderId(folder.id);
const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id);
return count;
};
const listSnapshots = async ({

View File

@@ -1,4 +1,3 @@
/* eslint-disable no-await-in-loop */
import { Knex } from "knex";
import { TDbClient } from "@app/db";
@@ -12,7 +11,6 @@ import {
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
export type TSnapshotDALFactory = ReturnType<typeof snapshotDALFactory>;
@@ -327,152 +325,12 @@ export const snapshotDALFactory = (db: TDbClient) => {
}
};
/**
* Prunes excess snapshots from the database to ensure only a specified number of recent snapshots are retained for each folder.
*
* This function operates in three main steps:
* 1. Pruning snapshots from root/non-versioned folders.
* 2. Pruning snapshots from versioned folders.
* 3. Removing orphaned snapshots that do not belong to any existing folder or folder version.
*
* The function processes snapshots in batches, determined by the `PRUNE_FOLDER_BATCH_SIZE` constant,
* to manage the large datasets without overwhelming the DB.
*
* Steps:
* - Fetch a batch of folder IDs.
* - For each batch, use a Common Table Expression (CTE) to rank snapshots within each folder by their creation date.
* - Identify and delete snapshots that exceed the project's point-in-time version limit (`pitVersionLimit`).
* - Repeat the process for versioned folders.
* - Finally, delete orphaned snapshots that do not have an associated folder.
*/
const pruneExcessSnapshots = async () => {
const PRUNE_FOLDER_BATCH_SIZE = 10000;
try {
let uuidOffset = "00000000-0000-0000-0000-000000000000";
// cleanup snapshots from root/non-versioned folders
// eslint-disable-next-line no-constant-condition, no-unreachable-loop
while (true) {
const folderBatch = await db(TableName.SecretFolder)
.where("id", ">", uuidOffset)
.where("isReserved", false)
.orderBy("id", "asc")
.limit(PRUNE_FOLDER_BATCH_SIZE)
.select("id");
const batchEntries = folderBatch.map((folder) => folder.id);
if (folderBatch.length) {
try {
logger.info(`Pruning snapshots in [range=${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}]`);
await db(TableName.Snapshot)
.with("snapshot_cte", (qb) => {
void qb
.from(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, batchEntries)
.select(
"folderId",
`${TableName.Snapshot}.id as id`,
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereNull(`${TableName.SecretFolder}.parentId`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (err) {
logger.error(
`Failed to prune snapshots from root/non-versioned folders in range ${batchEntries[0]}:${
batchEntries[batchEntries.length - 1]
}`
);
} finally {
uuidOffset = batchEntries[batchEntries.length - 1];
}
} else {
break;
}
}
// cleanup snapshots from versioned folders
uuidOffset = "00000000-0000-0000-0000-000000000000";
// eslint-disable-next-line no-constant-condition
while (true) {
const folderBatch = await db(TableName.SecretFolderVersion)
.select("folderId")
.distinct("folderId")
.where("folderId", ">", uuidOffset)
.orderBy("folderId", "asc")
.limit(PRUNE_FOLDER_BATCH_SIZE);
const batchEntries = folderBatch.map((folder) => folder.folderId);
if (folderBatch.length) {
try {
logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`);
await db(TableName.Snapshot)
.with("snapshot_cte", (qb) => {
void qb
.from(TableName.Snapshot)
.whereIn(`${TableName.Snapshot}.folderId`, batchEntries)
.select(
"folderId",
`${TableName.Snapshot}.id as id`,
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num`
)
);
})
.join(
TableName.SecretFolderVersion,
`${TableName.SecretFolderVersion}.folderId`,
`${TableName.Snapshot}.folderId`
)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`)
.whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (err) {
logger.error(
`Failed to prune snapshots from versioned folders in range ${batchEntries[0]}:${
batchEntries[batchEntries.length - 1]
}`
);
} finally {
uuidOffset = batchEntries[batchEntries.length - 1];
}
} else {
break;
}
}
// cleanup orphaned snapshots (those that don't belong to an existing folder and folder version)
await db(TableName.Snapshot)
.whereNotIn("folderId", (qb) => {
void qb
.select("folderId")
.from(TableName.SecretFolderVersion)
.union((qb1) => void qb1.select("id").from(TableName.SecretFolder));
})
.delete();
} catch (error) {
throw new DatabaseError({ error, name: "SnapshotPrune" });
}
};
return {
...secretSnapshotOrm,
findById,
findLatestSnapshotByFolderId,
findRecursivelySnapshots,
countOfSnapshotsByFolderId,
findSecretSnapshotDataById,
pruneExcessSnapshots
findSecretSnapshotDataById
};
};

View File

@@ -824,9 +824,6 @@ export const registerRoutes = async (
const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({
auditLogDAL,
queueService,
secretVersionDAL,
secretFolderVersionDAL: folderVersionDAL,
snapshotDAL,
identityAccessTokenDAL,
secretSharingDAL
});

View File

@@ -334,44 +334,6 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
}
});
server.route({
method: "PUT",
url: "/:workspaceSlug/version-limit",
config: {
rateLimit: writeLimit
},
schema: {
params: z.object({
workspaceSlug: z.string().trim()
}),
body: z.object({
pitVersionLimit: z.number().min(1).max(100)
}),
response: {
200: z.object({
message: z.string(),
workspace: ProjectsSchema
})
}
},
onRequest: verifyAuth([AuthMode.JWT]),
handler: async (req) => {
const workspace = await server.services.project.updateVersionLimit({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
pitVersionLimit: req.body.pitVersionLimit,
workspaceSlug: req.params.workspaceSlug
});
return {
message: "Successfully changed workspace version limit",
workspace
};
}
});
server.route({
method: "GET",
url: "/:workspaceId/integrations",

View File

@@ -77,7 +77,7 @@ export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsSe
// This will switch to a seal process and HMS flow in future
const encryptionKey = appCfg.ENCRYPTION_KEY || appCfg.ROOT_ENCRYPTION_KEY;
// if root key its base64 encoded
const isBase64 = Boolean(appCfg.ROOT_ENCRYPTION_KEY);
const isBase64 = !appCfg.ENCRYPTION_KEY;
if (!encryptionKey) throw new Error("Root encryption key not found for KMS service.");
const encryptionKeyBuffer = Buffer.from(encryptionKey, isBase64 ? "base64" : "utf8");

View File

@@ -39,7 +39,6 @@ import {
TToggleProjectAutoCapitalizationDTO,
TUpdateProjectDTO,
TUpdateProjectNameDTO,
TUpdateProjectVersionLimitDTO,
TUpgradeProjectDTO
} from "./project-types";
@@ -134,8 +133,7 @@ export const projectServiceFactory = ({
name: workspaceName,
orgId: organization.id,
slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`),
version: ProjectVersion.V2,
pitVersionLimit: 10
version: ProjectVersion.V2
},
tx
);
@@ -408,35 +406,6 @@ export const projectServiceFactory = ({
return updatedProject;
};
const updateVersionLimit = async ({
actor,
actorId,
actorOrgId,
actorAuthMethod,
pitVersionLimit,
workspaceSlug
}: TUpdateProjectVersionLimitDTO) => {
const project = await projectDAL.findProjectBySlug(workspaceSlug, actorOrgId);
if (!project) {
throw new BadRequestError({
message: "Project not found"
});
}
const { hasRole } = await permissionService.getProjectPermission(
actor,
actorId,
project.id,
actorAuthMethod,
actorOrgId
);
if (!hasRole(ProjectMembershipRole.Admin))
throw new BadRequestError({ message: "Only admins are allowed to take this action" });
return projectDAL.updateById(project.id, { pitVersionLimit });
};
const updateName = async ({
projectId,
actor,
@@ -532,7 +501,6 @@ export const projectServiceFactory = ({
getAProject,
toggleAutoCapitalization,
updateName,
upgradeProject,
updateVersionLimit
upgradeProject
};
};

View File

@@ -43,11 +43,6 @@ export type TToggleProjectAutoCapitalizationDTO = {
autoCapitalization: boolean;
} & TProjectPermission;
export type TUpdateProjectVersionLimitDTO = {
pitVersionLimit: number;
workspaceSlug: string;
} & Omit<TProjectPermission, "projectId">;
export type TUpdateProjectNameDTO = {
name: string;
} & TProjectPermission;

View File

@@ -1,19 +1,13 @@
import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal";
import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
type TDailyResourceCleanUpQueueServiceFactoryDep = {
auditLogDAL: Pick<TAuditLogDALFactory, "pruneAuditLog">;
identityAccessTokenDAL: Pick<TIdentityAccessTokenDALFactory, "removeExpiredTokens">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
queueService: TQueueServiceFactory;
};
@@ -23,9 +17,6 @@ export type TDailyResourceCleanUpQueueServiceFactory = ReturnType<typeof dailyRe
export const dailyResourceCleanUpQueueServiceFactory = ({
auditLogDAL,
queueService,
snapshotDAL,
secretVersionDAL,
secretFolderVersionDAL,
identityAccessTokenDAL,
secretSharingDAL
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
@@ -34,9 +25,6 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await secretSharingDAL.pruneExpiredSharedSecrets();
await snapshotDAL.pruneExcessSnapshots();
await secretVersionDAL.pruneExcessVersions();
await secretFolderVersionDAL.pruneExcessVersions();
logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`);
});

View File

@@ -62,32 +62,5 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => {
}
};
const pruneExcessVersions = async () => {
try {
await db(TableName.SecretFolderVersion)
.with("folder_cte", (qb) => {
void qb
.from(TableName.SecretFolderVersion)
.select(
"id",
"folderId",
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`)
.whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (error) {
throw new DatabaseError({
error,
name: "Secret Folder Version Prune"
});
}
};
return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions };
return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId };
};

View File

@@ -111,37 +111,8 @@ export const secretVersionDALFactory = (db: TDbClient) => {
}
};
const pruneExcessVersions = async () => {
try {
await db(TableName.SecretVersion)
.with("version_cte", (qb) => {
void qb
.from(TableName.SecretVersion)
.select(
"id",
"folderId",
db.raw(
`ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num`
)
);
})
.join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`)
.join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
.join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
.join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`)
.whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
.delete();
} catch (error) {
throw new DatabaseError({
error,
name: "Secret Version Prune"
});
}
};
return {
...secretVersionOrm,
pruneExcessVersions,
findLatestVersionMany,
bulkUpdate,
findLatestVersionByFolderId,

View File

@@ -20,7 +20,6 @@ import {
TUpdateWorkspaceIdentityRoleDTO,
TUpdateWorkspaceUserRoleDTO,
UpdateEnvironmentDTO,
UpdatePitVersionLimitDTO,
Workspace
} from "./types";
@@ -250,21 +249,6 @@ export const useToggleAutoCapitalization = () => {
});
};
export const useUpdateWorkspaceVersionLimit = () => {
const queryClient = useQueryClient();
return useMutation<{}, {}, UpdatePitVersionLimitDTO>({
mutationFn: ({ projectSlug, pitVersionLimit }) => {
return apiRequest.put(`/api/v1/workspace/${projectSlug}/version-limit`, {
pitVersionLimit
});
},
onSuccess: () => {
queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);
}
});
};
export const useDeleteWorkspace = () => {
const queryClient = useQueryClient();

View File

@@ -16,7 +16,6 @@ export type Workspace = {
upgradeStatus: string | null;
autoCapitalization: boolean;
environments: WorkspaceEnv[];
pitVersionLimit: number;
slug: string;
};
@@ -49,7 +48,6 @@ export type CreateWorkspaceDTO = {
};
export type RenameWorkspaceDTO = { workspaceID: string; newWorkspaceName: string };
export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number };
export type ToggleAutoCapitalizationDTO = { workspaceID: string; state: boolean };
export type DeleteWorkspaceDTO = { workspaceID: string };
@@ -130,4 +128,4 @@ export type TUpdateWorkspaceGroupRoleDTO = {
temporaryAccessStartTime: string;
}
)[];
};
};

View File

@@ -454,12 +454,12 @@ export const SecretOverviewPage = () => {
const filteredSecretNames = secKeys
?.filter((name) => name.toUpperCase().includes(searchFilter.toUpperCase()))
.sort((a, b) => (sortDir === "asc" ? a.localeCompare(b) : b.localeCompare(a)));
const filteredFolderNames = folderNames?.filter((name) =>
name.toLowerCase().includes(searchFilter.toLowerCase())
);
const filteredDynamicSecrets = dynamicSecretNames?.filter((name) =>
name.toLowerCase().includes(searchFilter.toLowerCase())
);
const filteredFolderNames = folderNames
?.filter((name) => name.toLowerCase().includes(searchFilter.toLowerCase()))
.sort((a, b) => (sortDir === "asc" ? a.localeCompare(b) : b.localeCompare(a)));
const filteredDynamicSecrets = dynamicSecretNames
?.filter((name) => name.toLowerCase().includes(searchFilter.toLowerCase()))
.sort((a, b) => (sortDir === "asc" ? a.localeCompare(b) : b.localeCompare(a)));
const isTableEmpty =
!(

View File

@@ -1,92 +0,0 @@
import { Controller, useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { z } from "zod";
import { createNotification } from "@app/components/notifications";
import { Button, FormControl, Input } from "@app/components/v2";
import { useProjectPermission, useWorkspace } from "@app/context";
import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
import { useUpdateWorkspaceVersionLimit } from "@app/hooks/api/workspace/queries";
const formSchema = z.object({
pitVersionLimit: z.coerce.number().min(1).max(100)
});
type TForm = z.infer<typeof formSchema>;
export const PointInTimeVersionLimitSection = () => {
const { mutateAsync: updatePitVersion } = useUpdateWorkspaceVersionLimit();
const { currentWorkspace } = useWorkspace();
const { membership } = useProjectPermission();
const {
control,
formState: { isSubmitting, isDirty },
handleSubmit
} = useForm<TForm>({
resolver: zodResolver(formSchema),
values: {
pitVersionLimit: currentWorkspace?.pitVersionLimit || 10
}
});
if (!currentWorkspace) return null;
const handleVersionLimitSubmit = async ({ pitVersionLimit }: TForm) => {
try {
await updatePitVersion({
pitVersionLimit,
projectSlug: currentWorkspace.slug
});
createNotification({
text: "Successfully updated version limit",
type: "success"
});
} catch (err) {
createNotification({
text: "Failed updating project's version limit",
type: "error"
});
}
};
const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin);
return (
<div className="mb-6 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
<div className="flex w-full items-center justify-between">
<p className="text-xl font-semibold">Version Retention</p>
</div>
<p className="mb-4 mt-2 max-w-2xl text-sm text-gray-400">
This defines the maximum number of recent secret versions to keep per folder. Excess versions will be removed at midnight (UTC) each day.
</p>
<form onSubmit={handleSubmit(handleVersionLimitSubmit)} autoComplete="off">
<div className="max-w-xs">
<Controller
control={control}
defaultValue={0}
name="pitVersionLimit"
render={({ field, fieldState: { error } }) => (
<FormControl
isError={Boolean(error)}
errorText={error?.message}
label="Recent versions to keep"
>
<Input {...field} type="number" min={1} step={1} isDisabled={!isAdmin} />
</FormControl>
)}
/>
</div>
<Button
colorSchema="secondary"
type="submit"
isLoading={isSubmitting}
disabled={!isAdmin || !isDirty}
>
Save
</Button>
</form>
</div>
);
};

View File

@@ -1 +0,0 @@
export { PointInTimeVersionLimitSection } from "./PointInTimeVersionLimitSection";

View File

@@ -3,7 +3,6 @@ import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSect
import { DeleteProjectSection } from "../DeleteProjectSection";
import { E2EESection } from "../E2EESection";
import { EnvironmentSection } from "../EnvironmentSection";
import { PointInTimeVersionLimitSection } from "../PointInTimeVersionLimitSection";
import { ProjectNameChangeSection } from "../ProjectNameChangeSection";
import { SecretTagsSection } from "../SecretTagsSection";
@@ -15,7 +14,6 @@ export const ProjectGeneralTab = () => {
<SecretTagsSection />
<AutoCapitalizationSection />
<E2EESection />
<PointInTimeVersionLimitSection />
<BackfillSecretReferenceSecretion />
<DeleteProjectSection />
</div>