feat: Add central error handler

Added:
- Added Midleware to capture and handle all errors
    - will catch error from `throw ...` and `next(...)`
- Added RequestError base error class to build consistent error objects
- Added common errors like `UnauthorizedRequestError`, `BadRequestError`
- Added consistent Logging solution using `winston`
    - Supports Loki Transporter using `winston-loki`
- Outputing Legal disclaimer to console when `TELEMETRY_ENABLED=true`

Changed:
- Changed console.log to getLogger() favor of using consistent logging

backend/environment.d.ts

@@ -14,6 +14,8 @@ declare global {
 			JWT_SIGNUP_SECRET: string;
       MONGO_URL: string;
       NODE_ENV: 'development' | 'staging' | 'testing' | 'production';
+      VERBOSE_ERROR_OUTPUT: boolean;
+      LOKI_HOST: string;
       CLIENT_ID_HEROKU: string;
       CLIENT_ID_VERCEL: string;
       CLIENT_ID_NETLIFY: string;

backend/package.json

@@ -25,7 +25,9 @@
     "stripe": "^10.7.0",
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
-    "typescript": "^4.9.3"
+    "typescript": "^4.9.3",
+    "winston": "^3.8.2",
+    "winston-loki": "^6.0.6"
   },
   "name": "infisical-api",
   "version": "1.0.0",

backend/src/app.ts

@@ -1,6 +1,5 @@
-/* eslint-disable no-console */
 
-import express from 'express';
+import express, { NextFunction, Request, Response } from 'express';
 import helmet from 'helmet';
 import cors from 'cors';
 import cookieParser from 'cookie-parser';
@@ -29,6 +28,9 @@ import {
   integration as integrationRouter,
   integrationAuth as integrationAuthRouter
 } from './routes';
+import { getLogger } from './utils/logger';
+import RequestError from './utils/requestError';
+import { InternalServerError } from './utils/errors';
 
 export const app = express();
 
@@ -50,6 +52,19 @@ if (NODE_ENV === 'production') {
   app.use(helmet());
 }
 
+//* Error Handling Middleware
+app.use((error: RequestError|Error, req: Request, res: Response, next: NextFunction)=>{
+  if(res.headersSent) return next();
+  
+  if(!(error instanceof RequestError)){
+      error = InternalServerError({context: {exception: error.message}, stack: error.stack})
+      getLogger('backend-main').log((<RequestError>error).levelName.toLowerCase(), (<RequestError>error).message)
+  }
+  res.status((<RequestError>error).statusCode).json((<RequestError>error).format(req))
+  next()
+})
+
+
 // routers
 app.use('/api/v1/signup', signupRouter);
 app.use('/api/v1/auth', authRouter);
@@ -70,5 +85,5 @@ app.use('/api/v1/integration', integrationRouter);
 app.use('/api/v1/integration-auth', integrationAuthRouter);
 
 export const server = app.listen(PORT, () => {
-  console.log(`Listening on PORT ${[PORT]}`);
+  getLogger("backend-main").info(`Server started listening at port ${PORT}`)
 });

backend/src/config/index.ts

@@ -10,6 +10,8 @@ const JWT_SIGNUP_LIFETIME = process.env.JWT_SIGNUP_LIFETIME! || '15m';
 const JWT_SIGNUP_SECRET = process.env.JWT_SIGNUP_SECRET!;
 const MONGO_URL = process.env.MONGO_URL!;
 const NODE_ENV = process.env.NODE_ENV! || 'production';
+const VERBOSE_ERROR_OUTPUT = process.env.VERBOSE_ERROR_OUTPUT || false;
+const LOKI_HOST = process.env.LOKI_HOST || undefined;
 const CLIENT_SECRET_HEROKU = process.env.CLIENT_SECRET_HEROKU!;
 const CLIENT_ID_HEROKU = process.env.CLIENT_ID_HEROKU!;
 const CLIENT_ID_VERCEL = process.env.CLIENT_ID_VERCEL!;
@@ -51,6 +53,8 @@ export {
   JWT_SIGNUP_SECRET,
   MONGO_URL,
   NODE_ENV,
+  VERBOSE_ERROR_OUTPUT,
+  LOKI_HOST,
   CLIENT_ID_HEROKU,
   CLIENT_ID_VERCEL,
   CLIENT_ID_NETLIFY,

backend/src/services/PostHogClient.ts

@@ -5,8 +5,16 @@ import {
   POSTHOG_PROJECT_API_KEY,
   TELEMETRY_ENABLED
 } from '../config';
+import { getLogger } from '../utils/logger';
 
-console.log('TELEMETRY_ENABLED: ', TELEMETRY_ENABLED);
+if(TELEMETRY_ENABLED){
+  getLogger("backend-main").info([
+    "",
+    "Infisical collects telemetry data about general usage.",
+    "The data helps us understand how the product is doing and guide our product development to create the best possible platform; it also helps us demonstrate growth for investors as we support Infisical as open-source software.",
+    "To opt out of telemetry, you can set `TELEMETRY_ENABLED=false` within the environment variables",
+  ].join('\n'))
+}
 
 let postHogClient: any;
 if (NODE_ENV === 'production' && TELEMETRY_ENABLED) {

backend/src/services/database.ts

@@ -1,10 +1,10 @@
-/* eslint-disable no-console */
 import mongoose from 'mongoose';
+import { getLogger } from '../utils/logger';
 
 export const initDatabase = (MONGO_URL: string) => {
   mongoose
     .connect(MONGO_URL)
-    .then(() => console.log('Successfully connected to DB'))
-    .catch((e) => console.log('Failed to connect to DB ', e));
+    .then(() => getLogger("database").info("Database connection established"))
+    .catch((e) => getLogger("database").error(`Unable to establish Database connection due to the error.\n${e}`));
   return mongoose.connection;
 };

backend/src/services/health.ts

@@ -1,10 +1,10 @@
-/* eslint-disable no-console */
 import mongoose from 'mongoose';
 import { createTerminus } from '@godaddy/terminus';
+import { getLogger } from '../utils/logger';
 
 export const setUpHealthEndpoint = <T>(server: T) => {
   const onSignal = () => {
-    console.log('Server is starting clean-up');
+    getLogger('backend-main').info('Server is starting clean-up');
     return Promise.all([
       new Promise((resolve) => {
         if (mongoose.connection && mongoose.connection.readyState == 1) {

backend/src/types/express/index.d.ts

@@ -1,5 +1,6 @@
 import * as express from 'express';
 
+
 // TODO: fix (any) types
 declare global {
 	namespace Express {

backend/src/utils/errors.ts

@@ -0,0 +1,64 @@
+import RequestError, { LogLevel, RequestErrorContext } from "./requestError"
+
+export const RouteNotFoundError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.INFO,
+    statusCode: error?.statusCode ?? 404,
+    type: error?.type ?? 'route_not_found',
+    message: error?.message ?? 'The requested source was not found',
+    context: error?.context,
+    stack: error?.stack
+})
+
+export const MethodNotAllowedError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.INFO,
+    statusCode: error?.statusCode ?? 405,
+    type: error?.type ?? 'method_not_allowed',
+    message: error?.message ?? 'The requested method is not allowed for the resource',
+    context: error?.context,
+    stack: error?.stack
+})
+
+export const UnauthorizedRequestError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.INFO,
+    statusCode: error?.statusCode ?? 401,
+    type: error?.type ?? 'unauthorized',
+    message: error?.message ?? 'You are not authorized to access this resource',
+    context: error?.context,
+    stack: error?.stack
+})
+  
+export const ForbiddenRequestError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.INFO,
+    statusCode: error?.statusCode ?? 403,
+    type: error?.type ?? 'forbidden',
+    message: error?.message ?? 'You are not allowed to access this resource',
+    context: error?.context,
+    stack: error?.stack
+})
+
+export const BadRequestError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.INFO,
+    statusCode: error?.statusCode ?? 400,
+    type: error?.type ?? 'bad_request',
+    message: error?.message ?? 'The request is invalid or cannot be served',
+    context: error?.context,
+    stack: error?.stack
+})
+
+export const InternalServerError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.ERROR,
+    statusCode: error?.statusCode ?? 500,
+    type: error?.type ?? 'internal_server_error',
+    message: error?.message ?? 'The server encountered an error while processing the request',
+    context: error?.context,
+    stack: error?.stack
+})
+
+export const ServiceUnavailableError = (error?: Partial<RequestErrorContext>) => new RequestError({
+    logLevel: error?.logLevel ?? LogLevel.ERROR,
+    statusCode: error?.statusCode ?? 503,
+    type: error?.type ?? 'service_unavailable',
+    message: error?.message ?? 'The service is currently unavailable. Please try again later.',
+    context: error?.context,
+    stack: error?.stack
+})

backend/src/utils/logger.ts

@@ -0,0 +1,65 @@
+/* eslint-disable no-console */
+import { createLogger, format, transports } from 'winston';
+import LokiTransport from 'winston-loki';
+import { LOKI_HOST, NODE_ENV } from '../config';
+
+const { combine, colorize, label, printf, splat, timestamp } = format;
+
+const logFormat = (prefix: string) => combine(
+  timestamp(),
+  splat(),
+  label({ label: prefix }),
+  printf((info) => `${info.timestamp} ${info.label} ${info.level}: ${info.message}`)
+);
+
+const createLoggerWithLabel = (level: string, label: string) => {
+  const _level = level.toLowerCase() || 'info'
+  //* Always add Console output to transports
+  const _transports: any[] = [
+    new transports.Console({
+      format: combine(
+        colorize(),
+        logFormat(label),
+        // format.json()
+      )
+    })
+  ]
+  //* Add LokiTransport if it's enabled
+  if(LOKI_HOST !== undefined){
+    _transports.push(
+      new LokiTransport({
+        host: LOKI_HOST,
+        handleExceptions: true,
+        handleRejections: true,
+        batching: true,
+        level: _level,
+        timeout: 30000,
+        format: format.combine(
+          format.json()
+        ),
+        labels: {app: process.env.npm_package_name, version: process.env.npm_package_version, environment: NODE_ENV},
+        onConnectionError: (err: Error)=> console.error('Connection error while connecting to Loki Server.\n', err)
+      })
+    )
+  }
+  
+  
+  return createLogger({
+    level: _level,
+    transports: _transports,
+    format: format.combine(
+      logFormat(label),
+      format.metadata({ fillExcept: ['message', 'level', 'timestamp', 'label'] })
+    )
+  });
+}
+
+const DEFAULT_LOGGERS = {
+  "backend-main": createLoggerWithLabel('info', '[IFSC:backend-main]'),
+  "database": createLoggerWithLabel('info', '[IFSC:database]'),
+}
+type LoggerNames = keyof typeof DEFAULT_LOGGERS
+
+export const getLogger = (loggerName: LoggerNames) => {
+  return DEFAULT_LOGGERS[loggerName]
+} 

backend/src/utils/requestError.ts

@@ -0,0 +1,113 @@
+import { Request } from 'express'
+import { VERBOSE_ERROR_OUTPUT } from '../config'
+
+export enum LogLevel {
+    DEBUG = 100,
+    INFO = 200,
+    NOTICE = 250,
+    WARNING = 300,
+    ERROR = 400,
+    CRITICAL = 500,
+    ALERT = 550,
+    EMERGENCY = 600,
+}
+
+export type RequestErrorContext =  {
+	logLevel?: LogLevel,
+	statusCode: number,
+	type: string,
+	message: string,
+	context?: Record<string, unknown>,
+	stack?: string|undefined
+}
+
+export default class RequestError extends Error{
+
+    private _logLevel: LogLevel
+    private _logName: string
+    statusCode: number
+    type: string
+    context: Record<string, unknown>
+    extra: Record<string, string|number|symbol>[]
+    private stacktrace: string|undefined|string[]
+
+    constructor(
+        {logLevel, statusCode, type, message, context, stack} : RequestErrorContext
+        ){
+        super(message)
+        this._logLevel = logLevel || LogLevel.INFO
+        this._logName = LogLevel[this._logLevel]
+        this.statusCode = statusCode
+        this.type = type
+        this.context = context || {}
+        this.extra = []
+
+        if(stack) this.stack = stack
+        else Error.captureStackTrace(this, this.constructor)
+        this.stacktrace = this.stack?.split('\n')
+    }
+
+    static convertFrom(error: Error) {
+        //This error was not handled by error handler. Please report this incident to the staff.
+        return new RequestError({
+            logLevel: LogLevel.ERROR,
+            statusCode: 500,
+            type: 'internal_server_error',
+            message: 'This error was not handled by error handler. Please report this incident to the staff',
+            context: {
+                message: error.message,
+                name: error.name
+            },
+            stack: error.stack
+        })
+    }
+
+    get level(){ return this._logLevel }
+    get levelName(){ return this._logName }
+
+    withTags(...tags: string[]|number[]){
+        this.context['tags'] = Object.assign(tags, this.context['tags'])
+        return this
+    }
+
+    withExtras(...extras: Record<string, string|boolean|number>[]){
+        this.extra = Object.assign(extras, this.extra)
+        return this
+    }
+
+    private _omit(obj: any, keys: string[]): typeof obj{
+        const exclude = new Set(keys)
+        obj = Object.fromEntries(Object.entries(obj).filter(e => !exclude.has(e[0])))
+        return obj
+    }
+
+    public format(req: Request){
+        let _context = Object.assign({
+            stacktrace: this.stacktrace
+        }, this.context)
+
+        //* Omit sensitive information from context that can leak internal workings of this program if user is not developer
+        if(!VERBOSE_ERROR_OUTPUT){
+            _context = this._omit(_context, [
+                'stacktrace',
+                'exception',
+            ])
+        }
+
+        const formatObject = {
+            type: this.type,
+            message: this.message,
+            context: _context,
+            level: this.level,
+            level_name: this.levelName,
+            status_code: this.statusCode,
+            datetime_iso: new Date().toISOString(),
+            application: process.env.npm_package_name || 'unknown',
+            request_id: req.headers["Request-Id"],
+            extra: this.extra
+        }
+
+        return formatObject
+
+    }
+}