Compare commits

...

10 Commits

Author SHA1 Message Date
Daniel Hougaard
b94ffb8a82 Fix: UA Token being overwritten by INFISICAL_TOKEN env variable 2024-04-23 22:00:32 +02:00
vmatsiiako
b067751027 Merge pull request #1720 from Infisical/docs/amplify-patch
docs: added -y flag in infisical cli installation in amplify doc to skip confirmation prompt
2024-04-22 22:54:38 -07:00
Akhil Mohan
f2b3b7b726 docs: added -y flag in infisical cli installation in amplify doc to skip confirmation prompt 2024-04-23 11:23:03 +05:30
Maidul Islam
1ba7a31e0d Merge pull request #1719 from Infisical/daniel/migration-fix
Fix: Duplicate org membership migration
2024-04-22 19:51:21 -04:00
Daniel Hougaard
233a4f7d77 Update 20240405000045_org-memberships-unique-constraint.ts 2024-04-23 01:49:44 +02:00
Daniel Hougaard
44ff1abd74 Update 20240405000045_org-memberships-unique-constraint.ts 2024-04-23 01:49:26 +02:00
Maidul Islam
08cb105fe4 Merge pull request #1712 from akhilmhdh/feat/batch-raw-secrets-api
Batch raw secrets api
2024-04-22 18:48:14 -04:00
Akhil Mohan
aa5cd0fd0f feat(server): switched from workspace id to project slug 2024-04-22 21:19:06 +05:30
Akhil Mohan
a339c473d5 docs: updated api doc with bulk raw secret ops 2024-04-19 20:54:41 +05:30
Akhil Mohan
718cabe49b feat(server): added batch raw bulk secret ops api 2024-04-19 20:53:54 +05:30
12 changed files with 576 additions and 6 deletions

View File

@@ -942,6 +942,113 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }]
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual([]);
});
test.each(testRawSecrets)("Bulk create secret raw in path $path", async ({ path, secret }) => {
const createSecretReqBody = {
projectSlug: seedData1.project.slug,
environment: seedData1.environment.slug,
secretPath: path,
secrets: [
{
secretKey: secret.key,
secretValue: secret.value,
secretComment: secret.comment
}
]
};
const createSecRes = await testServer.inject({
method: "POST",
url: `/api/v3/secrets/batch/raw`,
headers: {
authorization: `Bearer ${authToken}`
},
body: createSecretReqBody
});
expect(createSecRes.statusCode).toBe(200);
const createdSecretPayload = JSON.parse(createSecRes.payload);
expect(createdSecretPayload).toHaveProperty("secrets");
// fetch secrets
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual(
expect.arrayContaining([
expect.objectContaining({
key: secret.key,
value: secret.value,
type: SecretType.Shared
})
])
);
await deleteRawSecret({ path, key: secret.key });
});
test.each(testRawSecrets)("Bulk update secret raw in path $path", async ({ secret, path }) => {
await createRawSecret({ path, ...secret });
const updateSecretReqBody = {
projectSlug: seedData1.project.slug,
environment: seedData1.environment.slug,
secretPath: path,
secrets: [
{
secretValue: "new-value",
secretKey: secret.key
}
]
};
const updateSecRes = await testServer.inject({
method: "PATCH",
url: `/api/v3/secrets/batch/raw`,
headers: {
authorization: `Bearer ${authToken}`
},
body: updateSecretReqBody
});
expect(updateSecRes.statusCode).toBe(200);
const updatedSecretPayload = JSON.parse(updateSecRes.payload);
expect(updatedSecretPayload).toHaveProperty("secrets");
// fetch secrets
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual(
expect.arrayContaining([
expect.objectContaining({
key: secret.key,
value: "new-value",
version: 2,
type: SecretType.Shared
})
])
);
await deleteRawSecret({ path, key: secret.key });
});
test.each(testRawSecrets)("Bulk delete secret raw in path $path", async ({ path, secret }) => {
await createRawSecret({ path, ...secret });
const deletedSecretReqBody = {
projectSlug: seedData1.project.slug,
environment: seedData1.environment.slug,
secretPath: path,
secrets: [{ secretKey: secret.key }]
};
const deletedSecRes = await testServer.inject({
method: "DELETE",
url: `/api/v3/secrets/batch/raw`,
headers: {
authorization: `Bearer ${authToken}`
},
body: deletedSecretReqBody
});
expect(deletedSecRes.statusCode).toBe(200);
const deletedSecretPayload = JSON.parse(deletedSecRes.payload);
expect(deletedSecretPayload).toHaveProperty("secrets");
// fetch secrets
const secrets = await getSecrets(seedData1.environment.slug, path);
expect(secrets).toEqual([]);
});
}
);

View File

@@ -42,6 +42,7 @@ export async function up(knex: Knex): Promise<void> {
await knex.transaction(async (tx) => {
const duplicateRows = await tx(TableName.OrgMembership)
.select("userId", "orgId") // Select the userId and orgId so we can group by them
.whereNotNull("userId") // Ensure that the userId is not null
.count("* as cnt") // Count the number of rows for each userId and orgId, so we can make sure there are more than 1 row (a duplicate)
.groupBy("userId", "orgId")
.havingRaw("count(*) > ?", [1]); // Using havingRaw for direct SQL expressions

View File

@@ -282,6 +282,7 @@ export const RAW_SECRETS = {
},
CREATE: {
secretName: "The name of the secret to create.",
projectSlug: "The slug of the project to create the secret in.",
environment: "The slug of the environment to create the secret in.",
secretComment: "Attach a comment to the secret.",
secretPath: "The path to create the secret in.",
@@ -301,11 +302,13 @@ export const RAW_SECRETS = {
},
UPDATE: {
secretName: "The name of the secret to update.",
secretComment: "Update comment to the secret.",
environment: "The slug of the environment where the secret is located.",
secretPath: "The path of the secret to update",
secretValue: "The new value of the secret.",
skipMultilineEncoding: "Skip multiline encoding for the secret value.",
type: "The type of the secret to update.",
projectSlug: "The slug of the project to update the secret in.",
workspaceId: "The ID of the project to update the secret in."
},
DELETE: {
@@ -313,6 +316,7 @@ export const RAW_SECRETS = {
environment: "The slug of the environment where the secret is located.",
secretPath: "The path of the secret.",
type: "The type of the secret to delete.",
projectSlug: "The slug of the project to delete the secret in.",
workspaceId: "The ID of the project where the secret is located."
}
} as const;

View File

@@ -1656,4 +1656,263 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
return { secrets };
}
});
server.route({
method: "POST",
url: "/batch/raw",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Create many secrets",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().describe(RAW_SECRETS.CREATE.projectSlug),
environment: z.string().trim().describe(RAW_SECRETS.CREATE.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(RAW_SECRETS.CREATE.secretPath),
secrets: z
.object({
secretKey: z.string().trim().describe(RAW_SECRETS.CREATE.secretName),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()))
.describe(RAW_SECRETS.CREATE.secretValue),
secretComment: z.string().trim().optional().default("").describe(RAW_SECRETS.CREATE.secretComment),
skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.CREATE.skipMultilineEncoding)
})
.array()
.min(1)
}),
response: {
200: z.object({
secrets: secretRawSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body;
const secrets = await server.services.secret.createManySecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
secretPath,
environment,
projectSlug,
secrets: inputSecrets
});
await server.services.auditLog.createAuditLog({
projectId: secrets[0].workspace,
...req.auditLogInfo,
event: {
type: EventType.CREATE_SECRETS,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secrets: secrets.map((secret, i) => ({
secretId: secret.id,
secretKey: inputSecrets[i].secretKey,
secretVersion: secret.version
}))
}
}
});
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretCreated,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: secrets[0].workspace,
environment: req.body.environment,
secretPath: req.body.secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
return { secrets };
}
});
server.route({
method: "PATCH",
url: "/batch/raw",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Update many secrets",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().describe(RAW_SECRETS.UPDATE.projectSlug),
environment: z.string().trim().describe(RAW_SECRETS.UPDATE.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(RAW_SECRETS.UPDATE.secretPath),
secrets: z
.object({
secretKey: z.string().trim().describe(RAW_SECRETS.UPDATE.secretName),
secretValue: z
.string()
.transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()))
.describe(RAW_SECRETS.UPDATE.secretValue),
secretComment: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.secretComment),
skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding)
})
.array()
.min(1)
}),
response: {
200: z.object({
secrets: secretRawSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body;
const secrets = await server.services.secret.updateManySecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
secretPath,
environment,
projectSlug,
secrets: inputSecrets
});
await server.services.auditLog.createAuditLog({
projectId: secrets[0].workspace,
...req.auditLogInfo,
event: {
type: EventType.UPDATE_SECRETS,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secrets: secrets.map((secret, i) => ({
secretId: secret.id,
secretKey: inputSecrets[i].secretKey,
secretVersion: secret.version
}))
}
}
});
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretUpdated,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: secrets[0].workspace,
environment: req.body.environment,
secretPath: req.body.secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
return { secrets };
}
});
server.route({
method: "DELETE",
url: "/batch/raw",
config: {
rateLimit: secretsLimit
},
schema: {
description: "Delete many secrets",
security: [
{
bearerAuth: []
}
],
body: z.object({
projectSlug: z.string().trim().describe(RAW_SECRETS.DELETE.projectSlug),
environment: z.string().trim().describe(RAW_SECRETS.DELETE.environment),
secretPath: z
.string()
.trim()
.default("/")
.transform(removeTrailingSlash)
.describe(RAW_SECRETS.DELETE.secretPath),
secrets: z
.object({
secretKey: z.string().trim().describe(RAW_SECRETS.DELETE.secretName)
})
.array()
.min(1)
}),
response: {
200: z.object({
secrets: secretRawSchema.array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { environment, projectSlug, secretPath, secrets: inputSecrets } = req.body;
const secrets = await server.services.secret.deleteManySecretsRaw({
actorId: req.permission.id,
actor: req.permission.type,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
environment,
projectSlug,
secretPath,
secrets: inputSecrets
});
await server.services.auditLog.createAuditLog({
projectId: secrets[0].workspace,
...req.auditLogInfo,
event: {
type: EventType.DELETE_SECRETS,
metadata: {
environment: req.body.environment,
secretPath: req.body.secretPath,
secrets: secrets.map((secret, i) => ({
secretId: secret.id,
secretKey: inputSecrets[i].secretKey,
secretVersion: secret.version
}))
}
}
});
await server.services.telemetry.sendPostHogEvents({
event: PostHogEventTypes.SecretDeleted,
distinctId: getTelemetryDistinctId(req),
properties: {
numberOfSecrets: secrets.length,
workspaceId: secrets[0].workspace,
environment: req.body.environment,
secretPath: req.body.secretPath,
channel: getUserAgentType(req.headers["user-agent"]),
...req.auditLogInfo
}
});
return { secrets };
}
});
};

View File

@@ -33,9 +33,11 @@ import { TSecretQueueFactory } from "./secret-queue";
import {
TAttachSecretTagsDTO,
TCreateBulkSecretDTO,
TCreateManySecretRawDTO,
TCreateSecretDTO,
TCreateSecretRawDTO,
TDeleteBulkSecretDTO,
TDeleteManySecretRawDTO,
TDeleteSecretDTO,
TDeleteSecretRawDTO,
TFnSecretBlindIndexCheckV2,
@@ -46,6 +48,7 @@ import {
TGetSecretsRawDTO,
TGetSecretVersionsDTO,
TUpdateBulkSecretDTO,
TUpdateManySecretRawDTO,
TUpdateSecretDTO,
TUpdateSecretRawDTO
} from "./secret-types";
@@ -1036,6 +1039,143 @@ export const secretServiceFactory = ({
return decryptSecretRaw(secret, botKey);
};
const createManySecretsRaw = async ({
actorId,
projectSlug,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
secrets: inputSecrets = []
}: TCreateManySecretRawDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const secrets = await createManySecret({
projectId,
environment,
path: secretPath,
actor,
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag
};
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const updateManySecretsRaw = async ({
actorId,
projectSlug,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
secrets: inputSecrets = []
}: TUpdateManySecretRawDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const secrets = await updateManySecret({
projectId,
environment,
path: secretPath,
actor,
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(({ secretComment, secretKey, secretValue, skipMultilineEncoding }) => {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
return {
secretName: secretKey,
type: SecretType.Shared,
skipMultilineEncoding,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
secretKeyTag: secretKeyEncrypted.tag,
secretValueCiphertext: secretValueEncrypted.ciphertext,
secretValueIV: secretValueEncrypted.iv,
secretValueTag: secretValueEncrypted.tag,
secretCommentCiphertext: secretCommentEncrypted.ciphertext,
secretCommentIV: secretCommentEncrypted.iv,
secretCommentTag: secretCommentEncrypted.tag
};
})
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const deleteManySecretsRaw = async ({
actorId,
projectSlug,
environment,
actor,
actorOrgId,
actorAuthMethod,
secretPath,
secrets: inputSecrets = []
}: TDeleteManySecretRawDTO) => {
const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
if (!project) throw new BadRequestError({ message: "Project not found" });
const projectId = project.id;
const botKey = await projectBotService.getBotKey(projectId);
if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
const secrets = await deleteManySecret({
projectId,
environment,
path: secretPath,
actor,
actorId,
actorOrgId,
actorAuthMethod,
secrets: inputSecrets.map(({ secretKey }) => ({ secretName: secretKey, type: SecretType.Shared }))
});
await snapshotService.performSnapshot(secrets[0].folderId);
await secretQueueService.syncSecrets({ secretPath, projectId, environment });
return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
};
const getSecretVersions = async ({
actorId,
actor,
@@ -1280,6 +1420,9 @@ export const secretServiceFactory = ({
createSecretRaw,
updateSecretRaw,
deleteSecretRaw,
createManySecretsRaw,
updateManySecretsRaw,
deleteManySecretsRaw,
getSecretVersions,
// external services function
fnSecretBulkDelete,

View File

@@ -181,6 +181,39 @@ export type TDeleteSecretRawDTO = TProjectPermission & {
type: SecretType;
};
export type TCreateManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretPath: string;
projectSlug: string;
environment: string;
secrets: {
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding?: boolean;
}[];
};
export type TUpdateManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretPath: string;
projectSlug: string;
environment: string;
secrets: {
secretKey: string;
secretValue: string;
secretComment?: string;
skipMultilineEncoding?: boolean;
}[];
};
export type TDeleteManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
secretPath: string;
projectSlug: string;
environment: string;
secrets: {
secretKey: string;
}[];
};
export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
limit?: number;
offset?: number;

View File

@@ -7,6 +7,7 @@ import (
"crypto/sha256"
"encoding/base64"
"fmt"
"os"
"regexp"
"sort"
"strings"
@@ -204,8 +205,10 @@ var secretsSetCmd = &cobra.Command{
// decrypt workspace key
plainTextEncryptionKey := crypto.DecryptAsymmetric(encryptedWorkspaceKey, encryptedWorkspaceKeyNonce, encryptedWorkspaceKeySenderPublicKey, currentUsersPrivateKey)
infisicalTokenEnv := os.Getenv(util.INFISICAL_TOKEN_NAME)
// pull current secrets
secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, SecretsPath: secretsPath}, "")
secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, SecretsPath: secretsPath, InfisicalToken: infisicalTokenEnv}, "")
if err != nil {
util.HandleError(err, "unable to retrieve secrets")
}

View File

@@ -307,10 +307,6 @@ func FilterSecretsByTag(plainTextSecrets []models.SingleEnvironmentVariable, tag
}
func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectConfigFilePath string) ([]models.SingleEnvironmentVariable, error) {
if params.InfisicalToken == "" {
params.InfisicalToken = os.Getenv(INFISICAL_TOKEN_NAME)
}
isConnected := CheckIsConnectedToInternet()
var secretsToReturn []models.SingleEnvironmentVariable
// var serviceTokenDetails api.GetServiceTokenDetailsResponse

View File

@@ -0,0 +1,8 @@
---
title: "Bulk Create"
openapi: "POST /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -0,0 +1,8 @@
---
title: "Bulk Delete"
openapi: "DELETE /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -0,0 +1,8 @@
---
title: "Bulk Update"
openapi: "PATCH /api/v3/secrets/batch/raw"
---
<Tip>
This endpoint requires you to disable end-to-end encryption. For more information, you should consult this [note](https://infisical.com/docs/api-reference/overview/examples/note).
</Tip>

View File

@@ -33,7 +33,7 @@ This approach enables you to fetch secrets from Infisical during Amplify build t
preBuild:
commands:
- sudo curl -1sLf 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.rpm.sh' | sudo -E bash
- sudo yum install infisical
- sudo yum -y install infisical
```
</Step>
<Step title="Modify the build command">