Merge pull request #171 from Infisical/secret-versioning

This commit is contained in:
BlackMagiq
2022-12-26 17:55:12 -05:00
committed by GitHub
24 changed files with 638 additions and 70 deletions

View File

@ -1,5 +1,6 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { patchRouterParam } = require('./utils/patchAsyncRoutes');
import { patchRouterParam } from './utils/patchAsyncRoutes';
import express from 'express';
import helmet from 'helmet';
import cors from 'cors';
@ -10,6 +11,11 @@ dotenv.config();
import { PORT, NODE_ENV, SITE_URL } from './config';
import { apiLimiter } from './helpers/rateLimiter';
import {
workspace as eeWorkspaceRouter,
secret as eeSecretRouter
} from './ee/routes';
import {
signup as signupRouter,
auth as authRouter,
@ -29,12 +35,13 @@ import {
integration as integrationRouter,
integrationAuth as integrationAuthRouter
} from './routes';
import { getLogger } from './utils/logger';
import { RouteNotFoundError } from './utils/errors';
import { requestErrorHandler } from './middleware/requestErrorHandler';
//* Patch Async route params to handle Promise Rejections
patchRouterParam()
// patch async route params to handle Promise Rejections
patchRouterParam();
export const app = express();
@ -56,6 +63,10 @@ if (NODE_ENV === 'production') {
app.use(helmet());
}
// /ee routers
app.use('/api/v1/secret', eeSecretRouter);
app.use('/api/v1/workspace', eeWorkspaceRouter);
// routers
app.use('/api/v1/signup', signupRouter);
app.use('/api/v1/auth', authRouter);

View File

@ -41,6 +41,7 @@ const STRIPE_PUBLISHABLE_KEY = process.env.STRIPE_PUBLISHABLE_KEY!;
const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY!;
const STRIPE_WEBHOOK_SECRET = process.env.STRIPE_WEBHOOK_SECRET!;
const TELEMETRY_ENABLED = process.env.TELEMETRY_ENABLED! !== 'false' && true;
const LICENSE_KEY = process.env.LICENSE_KEY!;
export {
PORT,
@ -83,5 +84,6 @@ export {
STRIPE_PUBLISHABLE_KEY,
STRIPE_SECRET_KEY,
STRIPE_WEBHOOK_SECRET,
TELEMETRY_ENABLED
TELEMETRY_ENABLED,
LICENSE_KEY
};

View File

@ -1,6 +1,6 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { Key } from '../models';
import { Key, Secret } from '../models';
import {
pushSecrets as push,
pullSecrets as pull,
@ -169,9 +169,6 @@ export const pullSecrets = async (req: Request, res: Response) => {
* @returns
*/
export const pullSecretsServiceToken = async (req: Request, res: Response) => {
// get (encrypted) secrets from workspace with id [workspaceId]
// service token route
let secrets;
let key;
try {
@ -225,4 +222,4 @@ export const pullSecretsServiceToken = async (req: Request, res: Response) => {
secrets: reformatPullSecrets({ secrets }),
key
});
};
};

View File

@ -7,7 +7,7 @@ import {
Integration,
IntegrationAuth,
IUser,
ServiceToken
ServiceToken,
} from '../models';
import {
createWorkspace as create,

View File

@ -1,5 +1,9 @@
import * as stripeController from './stripeController';
import * as secretController from './secretController';
import * as workspaceController from './workspaceController';
export {
stripeController
stripeController,
secretController,
workspaceController
}

View File

@ -0,0 +1,35 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { SecretVersion } from '../models';
/**
* Return secret versions for secret with id [secretId]
* @param req
* @param res
*/
export const getSecretVersions = async (req: Request, res: Response) => {
let secretVersions;
try {
const { secretId } = req.params;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
secretVersions = await SecretVersion.find({
secret: secretId
})
.skip(offset)
.limit(limit);
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get secret versions'
});
}
return res.status(200).send({
secretVersions
});
}

View File

@ -0,0 +1,35 @@
import { Request, Response } from 'express';
import * as Sentry from '@sentry/node';
import { SecretSnapshot } from '../models';
/**
* Return secret snapshots for workspace with id [workspaceId]
* @param req
* @param res
*/
export const getWorkspaceSecretSnapshots = async (req: Request, res: Response) => {
let secretSnapshots;
try {
const { workspaceId } = req.params;
const offset: number = parseInt(req.query.offset as string);
const limit: number = parseInt(req.query.limit as string);
secretSnapshots = await SecretSnapshot.find({
workspace: workspaceId
})
.skip(offset)
.limit(limit);
} catch (err) {
Sentry.setUser({ email: req.user.email });
Sentry.captureException(err);
return res.status(400).send({
message: 'Failed to get secret snapshots'
});
}
return res.status(200).send({
secretSnapshots
});
}

View File

@ -1,21 +0,0 @@
/**
* @param {Object} obj
* @param {Object} obj.licenseKey - Infisical license key
*/
const checkLicenseKey = ({
licenseKey
}: {
licenseKey: string
}) => {
try {
// TODO
} catch (err) {
}
}
export {
checkLicenseKey
}

View File

@ -0,0 +1,74 @@
import * as Sentry from '@sentry/node';
import {
Secret
} from '../../models';
import {
SecretSnapshot,
SecretVersion,
ISecretVersion
} from '../models';
/**
* Save a copy of the current state of secrets in workspace with id
* [workspaceId] under a new snapshot with incremented version under the
* secretsnapshots collection.
* @param {Object} obj
* @param {String} obj.workspaceId
*/
const takeSecretSnapshotHelper = async ({
workspaceId
}: {
workspaceId: string;
}) => {
try {
const secrets = await Secret.find({
workspace: workspaceId
});
const latestSecretSnapshot = await SecretSnapshot.findOne({
workspace: workspaceId
}).sort({ version: -1 });
if (!latestSecretSnapshot) {
// case: no snapshots exist for workspace -> create first snapshot
await new SecretSnapshot({
workspace: workspaceId,
version: 1,
secrets
}).save();
return;
}
// case: snapshots exist for workspace
await new SecretSnapshot({
workspace: workspaceId,
version: latestSecretSnapshot.version + 1,
secrets
}).save();
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to take a secret snapshot');
}
}
const addSecretVersionsHelper = async ({
secretVersions
}: {
secretVersions: ISecretVersion[]
}) => {
try {
await SecretVersion.insertMany(secretVersions);
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
throw new Error('Failed to add secret versions');
}
}
export {
takeSecretSnapshotHelper,
addSecretVersionsHelper
}

View File

@ -0,0 +1,9 @@
import SecretSnapshot, { ISecretSnapshot } from "./secretSnapshot";
import SecretVersion, { ISecretVersion } from "./secretVersion";
export {
SecretSnapshot,
ISecretSnapshot,
SecretVersion,
ISecretVersion
}

View File

@ -0,0 +1,109 @@
import { Schema, model, Types } from 'mongoose';
import {
SECRET_SHARED,
SECRET_PERSONAL,
ENV_DEV,
ENV_TESTING,
ENV_STAGING,
ENV_PROD
} from '../../variables';
export interface ISecretSnapshot {
workspace: Types.ObjectId;
version: number;
secrets: {
version: number;
workspace: Types.ObjectId;
type: string;
user: Types.ObjectId;
environment: string;
secretKeyCiphertext: string;
secretKeyIV: string;
secretKeyTag: string;
secretKeyHash: string;
secretValueCiphertext: string;
secretValueIV: string;
secretValueTag: string;
secretValueHash: string;
}[]
}
const secretSnapshotSchema = new Schema<ISecretSnapshot>(
{
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',
required: true
},
version: {
type: Number,
required: true
},
secrets: [{
version: {
type: Number,
default: 1,
required: true
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',
required: true
},
type: {
type: String,
enum: [SECRET_SHARED, SECRET_PERSONAL],
required: true
},
user: {
// user associated with the personal secret
type: Schema.Types.ObjectId,
ref: 'User'
},
environment: {
type: String,
enum: [ENV_DEV, ENV_TESTING, ENV_STAGING, ENV_PROD],
required: true
},
secretKeyCiphertext: {
type: String,
required: true
},
secretKeyIV: {
type: String, // symmetric
required: true
},
secretKeyTag: {
type: String, // symmetric
required: true
},
secretKeyHash: {
type: String,
required: true
},
secretValueCiphertext: {
type: String,
required: true
},
secretValueIV: {
type: String, // symmetric
required: true
},
secretValueTag: {
type: String, // symmetric
required: true
},
secretValueHash: {
type: String,
required: true
}
}]
},
{
timestamps: true
}
);
const SecretSnapshot = model<ISecretSnapshot>('SecretSnapshot', secretSnapshotSchema);
export default SecretSnapshot;

View File

@ -0,0 +1,75 @@
import { Schema, model, Types } from 'mongoose';
export interface ISecretVersion {
_id?: Types.ObjectId;
secret: Types.ObjectId;
version: number;
isDeleted: boolean;
secretKeyCiphertext: string;
secretKeyIV: string;
secretKeyTag: string;
secretKeyHash: string;
secretValueCiphertext: string;
secretValueIV: string;
secretValueTag: string;
secretValueHash: string;
}
const secretVersionSchema = new Schema<ISecretVersion>(
{
secret: { // could be deleted
type: Schema.Types.ObjectId,
ref: 'Secret',
required: true
},
version: {
type: Number,
default: 1,
required: true
},
isDeleted: {
type: Boolean,
default: false,
required: true
},
secretKeyCiphertext: {
type: String,
required: true
},
secretKeyIV: {
type: String, // symmetric
required: true
},
secretKeyTag: {
type: String, // symmetric
required: true
},
secretKeyHash: {
type: String,
required: true
},
secretValueCiphertext: {
type: String,
required: true
},
secretValueIV: {
type: String, // symmetric
required: true
},
secretValueTag: {
type: String, // symmetric
required: true
},
secretValueHash: {
type: String,
required: true
}
},
{
timestamps: true
}
);
const SecretVersion = model<ISecretVersion>('SecretVersion', secretVersionSchema);
export default SecretVersion;

View File

@ -0,0 +1,7 @@
import secret from './secret';
import workspace from './workspace';
export {
secret,
workspace
}

View File

@ -0,0 +1,26 @@
import express from 'express';
const router = express.Router();
import {
requireAuth,
requireWorkspaceAuth,
validateRequest
} from '../../middleware';
import { body, query, param } from 'express-validator';
import { secretController } from '../controllers';
import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../../variables';
router.get(
'/:secretId/secret-versions',
requireAuth,
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
acceptedStatuses: [COMPLETED, GRANTED]
}),
param('secretId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
validateRequest,
secretController.getSecretVersions
);
export default router;

View File

@ -0,0 +1,27 @@
import express from 'express';
const router = express.Router();
import {
requireAuth,
requireWorkspaceAuth,
validateRequest
} from '../../middleware';
import { param, query } from 'express-validator';
import { ADMIN, MEMBER, GRANTED } from '../../variables';
import { workspaceController } from '../controllers';
router.get(
'/:workspaceId/secret-snapshots',
requireAuth,
requireWorkspaceAuth({
acceptedRoles: [ADMIN, MEMBER],
acceptedStatuses: [GRANTED]
}),
param('workspaceId').exists().trim(),
query('offset').exists().isInt(),
query('limit').exists().isInt(),
validateRequest,
workspaceController.getWorkspaceSecretSnapshots
);
export default router;

View File

@ -0,0 +1,19 @@
import { LICENSE_KEY } from '../../config';
/**
* Class to handle Enterprise Edition license actions
*/
class EELicenseService {
private readonly _isLicenseValid: boolean;
constructor(licenseKey: string) {
this._isLicenseValid = true;
}
public get isLicenseValid(): boolean {
return this._isLicenseValid;
}
}
export default new EELicenseService(LICENSE_KEY);

View File

@ -0,0 +1,47 @@
import { ISecretVersion } from '../models';
import {
takeSecretSnapshotHelper,
addSecretVersionsHelper
} from '../helpers/secret';
import EELicenseService from './EELicenseService';
/**
* Class to handle Enterprise Edition secret actions
*/
class EESecretService {
/**
* Save a copy of the current state of secrets in workspace with id
* [workspaceId] under a new snapshot with incremented version under the
* SecretSnapshot collection.
* Requires a valid license key [licenseKey]
* @param {Object} obj
* @param {String} obj.workspaceId
*/
static async takeSecretSnapshot({
workspaceId
}: {
workspaceId: string;
}) {
if (!EELicenseService.isLicenseValid) return;
await takeSecretSnapshotHelper({ workspaceId });
}
/**
* Adds secret versions [secretVersions] to the SecretVersion collection.
* @param {Object} obj
* @param {SecretVersion} obj.secretVersions
*/
static async addSecretVersions({
secretVersions
}: {
secretVersions: ISecretVersion[];
}) {
if (!EELicenseService.isLicenseValid) return;
await addSecretVersionsHelper({
secretVersions
});
}
}
export default EESecretService;

View File

@ -0,0 +1,7 @@
import EELicenseService from "./EELicenseService";
import EESecretService from "./EESecretService";
export {
EELicenseService,
EESecretService
}

View File

@ -1,10 +1,20 @@
import * as Sentry from '@sentry/node';
import {
Secret,
ISecret
ISecret,
} from '../models';
import {
EESecretService
} from '../ee/services';
import {
SecretVersion
} from '../ee/models';
import {
takeSecretSnapshotHelper
} from '../ee/helpers/secret';
import { decryptSymmetric } from '../utils/crypto';
import { SECRET_SHARED, SECRET_PERSONAL } from '../variables';
import { LICENSE_KEY } from '../config';
interface PushSecret {
ciphertextKey: string;
@ -19,7 +29,7 @@ interface PushSecret {
}
interface Update {
[index: string]: string;
[index: string]: any;
}
type DecryptSecretType = 'text' | 'object' | 'expanded';
@ -46,6 +56,7 @@ const pushSecrets = async ({
environment: string;
secrets: PushSecret[];
}): Promise<void> => {
// TODO: clean up function and fix up types
try {
// construct useful data structures
const oldSecrets = await pullSecrets({
@ -53,74 +64,124 @@ const pushSecrets = async ({
workspaceId,
environment
});
const oldSecretsObj: any = oldSecrets.reduce((accumulator, s: any) => {
return { ...accumulator, [s.secretKeyHash]: s };
}, {});
const newSecretsObj = secrets.reduce((accumulator, s) => {
return { ...accumulator, [s.hashKey]: s };
}, {});
const oldSecretsObj: any = oldSecrets.reduce((accumulator, s: any) =>
({ ...accumulator, [`${s.type}-${s.secretKeyHash}`]: s })
, {});
const newSecretsObj: any = secrets.reduce((accumulator, s) =>
({ ...accumulator, [`${s.type}-${s.hashKey}`]: s })
, {});
// handle deleting secrets
const toDelete = oldSecrets.filter(
(s: ISecret) => !(s.secretKeyHash in newSecretsObj)
);
const toDelete = oldSecrets
.filter(
(s: ISecret) => !(`${s.type}-${s.secretKeyHash}` in newSecretsObj)
)
.map((s) => s._id);
if (toDelete.length > 0) {
await Secret.deleteMany({
_id: { $in: toDelete.map((s) => s._id) }
_id: { $in: toDelete }
});
await SecretVersion.updateMany({
secret: { $in: toDelete }
}, {
isDeleted: true
});
}
// handle modifying secrets where type or value changed
const operations = secrets
const toUpdate = oldSecrets
.filter((s) => {
if (s.hashKey in oldSecretsObj) {
if (s.hashValue !== oldSecretsObj[s.hashKey].secretValueHash) {
if (`${s.type}-${s.secretKeyHash}` in newSecretsObj) {
if (s.secretValueHash !== newSecretsObj[`${s.type}-${s.secretKeyHash}`].hashValue) {
// case: filter secrets where value changed
return true;
}
if (s.type !== oldSecretsObj[s.hashKey].type) {
// case: filter secrets where type changed
if (!s.version) {
// case: filter (legacy) secrets that were not versioned
return true;
}
}
return false;
})
});
const operations = toUpdate
.map((s) => {
const {
ciphertextValue,
ivValue,
tagValue,
hashValue
} = newSecretsObj[`${s.type}-${s.secretKeyHash}`];
const update: Update = {
type: s.type,
secretValueCiphertext: s.ciphertextValue,
secretValueIV: s.ivValue,
secretValueTag: s.tagValue,
secretValueHash: s.hashValue
};
secretValueCiphertext: ciphertextValue,
secretValueIV: ivValue,
secretValueTag: tagValue,
secretValueHash: hashValue
}
if (!s.version) {
// case: (legacy) secret was not versioned
update.version = 1;
} else {
update['$inc'] = {
version: 1
}
}
if (s.type === SECRET_PERSONAL) {
// attach user assocaited with the personal secret
// attach user associated with the personal secret
update['user'] = userId;
}
return {
updateOne: {
filter: {
workspace: workspaceId,
_id: oldSecretsObj[s.hashKey]._id
_id: oldSecretsObj[`${s.type}-${s.secretKeyHash}`]._id
},
update
}
};
});
const a = await Secret.bulkWrite(operations as any);
await Secret.bulkWrite(operations as any);
// (EE) add secret versions for updated secrets
await EESecretService.addSecretVersions({
secretVersions: toUpdate.map(({
_id,
version,
type,
secretKeyHash,
}) => {
const newSecret = newSecretsObj[`${type}-${secretKeyHash}`];
return ({
secret: _id,
version: version ? version + 1 : 1,
isDeleted: false,
secretKeyCiphertext: newSecret.ciphertextKey,
secretKeyIV: newSecret.ivKey,
secretKeyTag: newSecret.tagKey,
secretKeyHash: newSecret.hashKey,
secretValueCiphertext: newSecret.ciphertextValue,
secretValueIV: newSecret.ivValue,
secretValueTag: newSecret.tagValue,
secretValueHash: newSecret.hashValue
})
})
});
// handle adding new secrets
const toAdd = secrets.filter((s) => !(s.hashKey in oldSecretsObj));
const toAdd = secrets.filter((s) => !(`${s.type}-${s.hashKey}` in oldSecretsObj));
if (toAdd.length > 0) {
// add secrets
await Secret.insertMany(
const newSecrets = await Secret.insertMany(
toAdd.map((s, idx) => {
let obj: any = {
const obj: any = {
version: 1,
workspace: workspaceId,
type: toAdd[idx].type,
environment,
@ -141,7 +202,39 @@ const pushSecrets = async ({
return obj;
})
);
// (EE) add secret versions for new secrets
EESecretService.addSecretVersions({
secretVersions: newSecrets.map(({
_id,
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
}) => ({
secret: _id,
version: 1,
isDeleted: false,
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretKeyHash,
secretValueCiphertext,
secretValueIV,
secretValueTag,
secretValueHash
}))
});
}
// (EE) take a secret snapshot
await EESecretService.takeSecretSnapshot({
workspaceId
})
} catch (err) {
Sentry.setUser(null);
Sentry.captureException(err);
@ -295,6 +388,8 @@ const decryptSecrets = ({
return content;
};
export {
pushSecrets,
pullSecrets,

View File

@ -10,6 +10,7 @@ import {
export interface ISecret {
_id: Types.ObjectId;
version: number;
workspace: Types.ObjectId;
type: string;
user: Types.ObjectId;
@ -26,6 +27,10 @@ export interface ISecret {
const secretSchema = new Schema<ISecret>(
{
version: {
type: Number,
required: true
},
workspace: {
type: Schema.Types.ObjectId,
ref: 'Workspace',

View File

@ -7,8 +7,8 @@ import {
validateRequest
} from '../middleware';
import { body, query, param } from 'express-validator';
import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../variables';
import { secretController } from '../controllers';
import { ADMIN, MEMBER, COMPLETED, GRANTED } from '../variables';
router.post(
'/:workspaceId',

View File

@ -1,6 +1,6 @@
import express from 'express';
const router = express.Router();
import { body, param } from 'express-validator';
import { body, param, query } from 'express-validator';
import {
requireAuth,
requireWorkspaceAuth,

View File

@ -45,7 +45,7 @@ function wrap(fn) {
return copyFnProps(fn, newFn);
}
export function patchRouterParam() {
function patchRouterParam() {
const originalParam = Router.prototype.constructor.param;
Router.prototype.constructor.param = function param(name, fn) {
fn = wrap(fn);
@ -62,4 +62,8 @@ Object.defineProperty(Layer.prototype, 'handle', {
fn = wrap(fn);
this.__handle = fn;
},
});
});
module.exports = {
patchRouterParam
};

View File

@ -28,6 +28,7 @@ Configuring Infisical requires setting some environment variables. There is a fi
| `SMTP_FROM_ADDRESS` | ❗️ Email address to be used for sending emails (e.g. `team@infisical.com`) | `None` |
| `SMTP_FROM_NAME` | Name label to be used in From field (e.g. `Team`) | `Infisical` |
| `TELEMETRY_ENABLED` | `true` or `false`. [More](../overview). | `true` |
| `LICENSE_KEY` | License key if using Infisical Enterprise Edition | `true` |
| `CLIENT_ID_HEROKU` | OAuth2 client ID for Heroku integration | `None` |
| `CLIENT_ID_VERCEL` | OAuth2 client ID for Vercel integration | `None` |
| `CLIENT_ID_NETLIFY` | OAuth2 client ID for Netlify integration | `None` |