From a4495becb86667fe5063fd94e1f70d5ced3162e0 Mon Sep 17 00:00:00 2001 From: Sergey Ukustov Date: Sat, 6 Jul 2024 13:35:41 +0300 Subject: [PATCH] Move auth logic from api gateway lambda to cas codebase (#1234) * wip * wip * wip * drop network * get auth lambda * wip * Metrics * Metrics * parse allowed dids list * default dids * reasons to disallow * relaxed label * Update src/auth/auth.middleware.ts Co-authored-by: Mohsin Zaidi <2236875+smrz2001@users.noreply.github.com> --------- Co-authored-by: Mohsin Zaidi <2236875+smrz2001@users.noreply.github.com> --- config/default.json | 6 +- config/env/dev.json | 6 +- config/env/prod.json | 6 +- config/env/test.json | 6 +- src/auth/__tests__/auth.middleware.test.ts | 246 +++++++++++++++++++++ src/auth/auth.middleware.ts | 154 +++++++++++++ src/auth/index.ts | 65 ------ src/server.ts | 12 +- src/settings.ts | 4 + 9 files changed, 433 insertions(+), 72 deletions(-) create mode 100644 src/auth/__tests__/auth.middleware.test.ts create mode 100644 src/auth/auth.middleware.ts delete mode 100644 src/auth/index.ts diff --git a/config/default.json b/config/default.json index 95b5787b..206544b7 100644 --- a/config/default.json +++ b/config/default.json @@ -11,7 +11,6 @@ "merkleDepthLimit": 0, "minStreamCount": 1024, "readyRetryIntervalMS": 300000, - "requireAuth": false, "schedulerIntervalMS": 300000, "schedulerStopAfterNoOp": false, "pubsubResponderWindowMs": 8035200000, @@ -101,5 +100,10 @@ "s3Endpoint": "", "maxTimeToHoldMessageSec": 21600, "waitTimeForMessageSec": 0 + }, + "auth": { + "required": false, + "dids": "", + "relaxed": true } } diff --git a/config/env/dev.json b/config/env/dev.json index 2557853f..a584e6a4 100644 --- a/config/env/dev.json +++ b/config/env/dev.json @@ -11,7 +11,6 @@ "merkleDepthLimit": "@@MERKLE_DEPTH_LIMIT", "minStreamCount": "@@MIN_STREAM_COUNT", "readyRetryIntervalMS": "@@READY_RETRY_INTERVAL_MS", - "requireAuth": "@@REQUIRE_AUTH", "schedulerIntervalMS": "@@SCHEDULER_INTERVAL_MS", "schedulerStopAfterNoOp": "@@SCHEDULER_STOP_AFTER_NO_OP", "pubsubResponderWindowMs": "@@PUBSUB_RESPONDER_WINDOW_MS", @@ -93,5 +92,10 @@ "s3Endpoint": "@@S3_ENDPOINT", "maxTimeToHoldMessageSec": "@@MAX_TIME_TO_HOLD_MESSAGE_SEC", "waitTimeForMessageSec": "@@WAIT_TIME_FOR_MESSAGE_SEC" + }, + "auth": { + "required": "@@REQUIRE_AUTH", + "dids": "@@AUTH_DIDS_ALLOWED", + "relaxed": "@@AUTH_RELAXED" } } diff --git a/config/env/prod.json b/config/env/prod.json index 2557853f..a584e6a4 100644 --- a/config/env/prod.json +++ b/config/env/prod.json @@ -11,7 +11,6 @@ "merkleDepthLimit": "@@MERKLE_DEPTH_LIMIT", "minStreamCount": "@@MIN_STREAM_COUNT", "readyRetryIntervalMS": "@@READY_RETRY_INTERVAL_MS", - "requireAuth": "@@REQUIRE_AUTH", "schedulerIntervalMS": "@@SCHEDULER_INTERVAL_MS", "schedulerStopAfterNoOp": "@@SCHEDULER_STOP_AFTER_NO_OP", "pubsubResponderWindowMs": "@@PUBSUB_RESPONDER_WINDOW_MS", @@ -93,5 +92,10 @@ "s3Endpoint": "@@S3_ENDPOINT", "maxTimeToHoldMessageSec": "@@MAX_TIME_TO_HOLD_MESSAGE_SEC", "waitTimeForMessageSec": "@@WAIT_TIME_FOR_MESSAGE_SEC" + }, + "auth": { + "required": "@@REQUIRE_AUTH", + "dids": "@@AUTH_DIDS_ALLOWED", + "relaxed": "@@AUTH_RELAXED" } } diff --git a/config/env/test.json b/config/env/test.json index 64117e35..e0bc3766 100644 --- a/config/env/test.json +++ b/config/env/test.json @@ -3,7 +3,6 @@ "expirationPeriod": 0, "loadStreamTimeoutMs": 1000, "readyRetryIntervalMS": 10000, - "requireAuth": false, "schedulerIntervalMS": 10000, "carStorage": { "mode": "s3", @@ -74,5 +73,10 @@ "s3BucketName": "ceramic-tnet-cas", "maxTimeToHoldMessageSec": 10800, "waitTimeForMessageSec": 10 + }, + "auth": { + "required": false, + "dids": "", + "relaxed": true } } diff --git a/src/auth/__tests__/auth.middleware.test.ts b/src/auth/__tests__/auth.middleware.test.ts new file mode 100644 index 00000000..4380c47c --- /dev/null +++ b/src/auth/__tests__/auth.middleware.test.ts @@ -0,0 +1,246 @@ +import { test, describe, expect, beforeAll } from '@jest/globals' +import express, { Express } from 'express' +import { auth } from '../auth.middleware.js' +import supertest from 'supertest' +import { DID } from 'dids' +import { Ed25519Provider } from 'key-did-provider-ed25519' +import KeyDIDResolver from 'key-did-resolver' +import { CARFactory } from 'cartonne' +import bodyParser from 'body-parser' +import { logger } from '../../logger/index.js' + +const carFactory = new CARFactory() + +async function createDidKey( + seed: Uint8Array = crypto.getRandomValues(new Uint8Array(32)) +): Promise { + const did = new DID({ + provider: new Ed25519Provider(seed), + resolver: KeyDIDResolver.getResolver(), + }) + await did.authenticate() + return did +} + +async function makeJWS(did: DID, payload: object): Promise { + const dagJWS = await did.createJWS(payload) + const signature = dagJWS.signatures[0] + if (!signature) throw new Error(`No signature`) + return `${signature.protected}.${dagJWS.payload}.${signature.signature}` +} + +describe('Authorization header: strict', () => { + let app: Express + let did: DID + let disallowedDID: DID + + beforeAll(async () => { + did = await createDidKey() + disallowedDID = await createDidKey() + app = express().use(express.json()) + app.use(bodyParser.raw({ inflate: true, type: 'application/vnd.ipld.car', limit: '1mb' })) + app.use(bodyParser.json({ type: 'application/json' })) + app.use(bodyParser.urlencoded({ extended: true, type: 'application/x-www-form-urlencoded' })) + app.use( + auth({ + allowedDIDs: new Set([did.id]), + isRelaxed: false, + logger: logger, + }) + ) + app.post('/', (req, res) => { + res.json({ hello: 'world' }) + }) + }) + + test('allowed DID, valid digest', async () => { + const carFile = carFactory.build() + const cid = carFile.put({ hello: 'world' }, { isRoot: true }) + const jws = await makeJWS(did, { nonce: '1234567890', digest: cid.toString() }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(200) + }) + test('allowed DID, invalid digest', async () => { + const carFile = carFactory.build() + const jws = await makeJWS(did, { nonce: '1234567890', digest: `Invalid` }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(403) + }) + test('disallowed DID, valid digest', async () => { + const carFile = carFactory.build() + const cid = carFile.put({ hello: 'world' }, { isRoot: true }) + const jws = await makeJWS(disallowedDID, { nonce: '1234567890', digest: cid.toString() }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(403) + }) + test('disallowed DID, invalid digest', async () => { + const carFile = carFactory.build() + const jws = await makeJWS(disallowedDID, { nonce: '1234567890', digest: `Invalid` }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(403) + }) +}) + +describe('Authorization header: relaxed', () => { + let app: Express + let disallowedDID: DID + let did: DID + + beforeAll(async () => { + disallowedDID = await createDidKey() + did = await createDidKey() + app = express().use(express.json()) + app.use(bodyParser.raw({ inflate: true, type: 'application/vnd.ipld.car', limit: '1mb' })) + app.use(bodyParser.json({ type: 'application/json' })) + app.use(bodyParser.urlencoded({ extended: true, type: 'application/x-www-form-urlencoded' })) + app.use( + auth({ + allowedDIDs: new Set([did.id]), + isRelaxed: true, + logger: logger, + }) + ) + app.post('/', (req, res) => { + res.json({ hello: 'world' }) + }) + }) + + test('disallowed DID, valid digest', async () => { + const carFile = carFactory.build() + const cid = carFile.put({ hello: 'world' }, { isRoot: true }) + const jws = await makeJWS(disallowedDID, { nonce: '1234567890', digest: cid.toString() }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(200) + }) + test('disallowed DID, invalid digest', async () => { + const carFile = carFactory.build() + const jws = await makeJWS(disallowedDID, { nonce: '1234567890', digest: `Invalid` }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(403) + }) +}) + +describe('Auth lambda', () => { + let app: Express + let did: DID + + beforeAll(async () => { + did = await createDidKey() + app = express().use(express.json()) + app.use(bodyParser.raw({ inflate: true, type: 'application/vnd.ipld.car', limit: '1mb' })) + app.use(bodyParser.json({ type: 'application/json' })) + app.use(bodyParser.urlencoded({ extended: true, type: 'application/x-www-form-urlencoded' })) + app.use( + auth({ + allowedDIDs: new Set(), + isRelaxed: false, + logger: logger, + }) + ) + app.post('/', (req, res) => { + res.json({ hello: 'world' }) + }) + }) + + test('valid digest', async () => { + const carFile = carFactory.build() + const cid = carFile.put({ hello: 'world' }, { isRoot: true }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('did', did.id) + .set('digest', cid.toString()) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(200) + }) + test('invalid digest', async () => { + const carFile = carFactory.build() + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('did', did.id) + .set('digest', 'INVALID') + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(403) + }) +}) + +describe('empty allowed dids list', () => { + let app: Express + let did: DID + + beforeAll(async () => { + did = await createDidKey() + app = express().use(express.json()) + app.use(bodyParser.raw({ inflate: true, type: 'application/vnd.ipld.car', limit: '1mb' })) + app.use(bodyParser.json({ type: 'application/json' })) + app.use(bodyParser.urlencoded({ extended: true, type: 'application/x-www-form-urlencoded' })) + app.use( + auth({ + allowedDIDs: new Set(), + isRelaxed: false, + logger: logger, + }) + ) + app.post('/', (req, res) => { + res.json({ hello: 'world' }) + }) + }) + + test('pass Authorization header check', async () => { + const carFile = carFactory.build() + const cid = carFile.put({ hello: 'world' }, { isRoot: true }) + const jws = await makeJWS(did, { nonce: '1234567890', digest: cid.toString() }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('Authorization', `Bearer ${jws}`) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(200) + }) + test('use Auth Lambda check: ok', async () => { + const carFile = carFactory.build() + const cid = carFile.put({ hello: 'world' }, { isRoot: true }) + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('did', did.id) + .set('digest', cid.toString()) + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(200) + }) + test('use Auth Lambda check: invalid digest', async () => { + const carFile = carFactory.build() + const response = await supertest(app) + .post('/') + .set('Content-Type', 'application/vnd.ipld.car') + .set('did', did.id) + .set('digest', 'INVALID') + .send(Buffer.from(carFile.bytes)) // Supertest quirk + expect(response.status).toBe(403) + }) +}) diff --git a/src/auth/auth.middleware.ts b/src/auth/auth.middleware.ts new file mode 100644 index 00000000..dbee7de9 --- /dev/null +++ b/src/auth/auth.middleware.ts @@ -0,0 +1,154 @@ +import { NextFunction, Request, Response, Handler } from 'express' +import * as DAG_JOSE from 'dag-jose' +import * as sha256 from '@stablelib/sha256' +import * as u8a from 'uint8arrays' +import { CARFactory, CAR } from 'cartonne' +import { DiagnosticsLogger } from '@ceramicnetwork/common' +import { DID } from 'dids' +import KeyDIDResolver from 'key-did-resolver' +import { ServiceMetrics } from '@ceramicnetwork/observability' +import { METRIC_NAMES } from '../settings.js' + +export type AuthOpts = { + logger?: DiagnosticsLogger + allowedDIDs: Set + isRelaxed: boolean +} + +export const AUTH_BEARER_REGEXP = new RegExp(/Bearer (.*)/) +const CAR_FACTORY = new CARFactory() +CAR_FACTORY.codecs.add(DAG_JOSE) + +const VERIFIER = new DID({ resolver: KeyDIDResolver.getResolver() }) + +enum DISALLOW_REASON { + LAMBDA_INVALID_DIGEST = 'lambda-invalid-digest', + DID_ALLOWLIST_NO_HEADER = 'did-allowlist-no-header', + DID_ALLOWLIST_NO_DID = 'did-allowlist-no-did', + DID_ALLOWLIST_NO_FIELDS = 'did-allowlist-no-fields', + DID_ALLOWLIST_REJECTED = 'did-allowlist-rejected', + DID_ALLOWLIST_INVALID_DIGEST = 'did-allowlist-invalid-digest', +} + +export function parseAllowedDIDs(dids: string | undefined): Set { + if (dids) { + const parts = dids.split(',') + return new Set(parts) + } else { + return new Set() + } +} + +export function auth(opts: AuthOpts): Handler { + const hasAllowedDIDsList = opts.allowedDIDs.size > 0 + + /** + * @dev If the request has a did header, it means we have already confirmed the did + * is registered. If the request has no did, it means we have already + * confirmed the IP address making the request is on our allowlist. If the + * request contains a body, it means we have already verified the digest + * header can be trusted here. + * All of this logic mentioned above lives outside of this app. + * Notice that the absense of a did header or body bypasses any checks below + * this app will still work if the logice above is not in place. + */ + return async function (req: Request, res: Response, next: NextFunction) { + const logger = opts.logger + + // Use auth lambda + const didFromHeader = req.header('did') + if (didFromHeader && req.body) { + const digest = buildBodyDigest(req.header('Content-Type'), req.body) + if (req.header('digest') === digest) { + ServiceMetrics.count(METRIC_NAMES.AUTH_ALLOWED, 1, { did: didFromHeader }) + return next() + } else { + logger?.verbose(`Disallowed: Auth lambda: Invalid digest`) + return disallow(res, DISALLOW_REASON.LAMBDA_INVALID_DIGEST) + } + } + + // Authorization Header + if (hasAllowedDIDsList) { + const authorizationHeader = req.header('Authorization') || '' + const bearerTokenMatch = AUTH_BEARER_REGEXP.exec(authorizationHeader) + const jws = bearerTokenMatch?.[1] + if (!jws) { + logger?.verbose(`Disallowed: No authorization header`) + return disallow(res, DISALLOW_REASON.DID_ALLOWLIST_NO_HEADER) + } + const verifyJWSResult = await VERIFIER.verifyJWS(jws) + const did = verifyJWSResult.didResolutionResult.didDocument?.id + if (!did) { + logger?.verbose(`Disallowed: No DID`) + return disallow(res, DISALLOW_REASON.DID_ALLOWLIST_NO_DID) + } + const nonce = verifyJWSResult.payload?.['nonce'] + const digest = verifyJWSResult.payload?.['digest'] + if (!nonce || !digest) { + logger?.verbose(`Disallowed: No nonce or No digest`) + return disallow(res, DISALLOW_REASON.DID_ALLOWLIST_NO_FIELDS) + } + if (!isAllowedDID(did, opts)) { + logger?.verbose(`Disallowed: ${did}`) + return disallow(res, DISALLOW_REASON.DID_ALLOWLIST_REJECTED) + } + + const body = req.body + const contentType = req.header('Content-Type') + const digestCalculated = buildBodyDigest(contentType, body) + if (digestCalculated !== digest) { + logger?.verbose(`Disallowed: Incorrect digest for DID ${did}`) + return disallow(res, DISALLOW_REASON.DID_ALLOWLIST_INVALID_DIGEST) + } + const relaxedLabel = opts.isRelaxed ? 1 : 0 + ServiceMetrics.count(METRIC_NAMES.AUTH_ALLOWED, 1, { did: did, relaxed: relaxedLabel }) + } + return next() + } +} + +function disallow(res: Response, reason: DISALLOW_REASON): Response { + ServiceMetrics.count(METRIC_NAMES.AUTH_DISALLOWED, 1, { reason: reason }) + return res.status(403).json({ error: 'Unauthorized' }) +} + +function isAllowedDID(did: string, opts: AuthOpts): boolean { + if (opts.isRelaxed) { + opts.logger?.verbose(`Allowed: Relaxed: ${did}`) + return true + } else { + return opts.allowedDIDs.has(did) + } +} + +function buildBodyDigest(contentType: string | undefined, body: any): string | undefined { + if (!body) return + + let hash: Uint8Array | undefined + + if (contentType) { + if (contentType.includes('application/vnd.ipld.car')) { + let car: CAR + try { + car = CAR_FACTORY.fromBytes(body) + } catch (e) { + return undefined + } + const root = car.roots[0] + if (!root) { + return undefined + } + return root.toString() + } else if (contentType.includes('application/json')) { + hash = sha256.hash(u8a.fromString(JSON.stringify(body))) + } + } + + if (!hash) { + // Default to hashing stringified body + hash = sha256.hash(u8a.fromString(JSON.stringify(body))) + } + + return `0x${u8a.toString(hash, 'base16')}` +} diff --git a/src/auth/index.ts b/src/auth/index.ts deleted file mode 100644 index 1b24e956..00000000 --- a/src/auth/index.ts +++ /dev/null @@ -1,65 +0,0 @@ -import * as sha256 from '@stablelib/sha256' -import { CARFactory } from 'cartonne' -import { NextFunction, Request, Response } from 'express' -import * as u8a from 'uint8arrays' -import * as DAG_JOSE from 'dag-jose' - -export const auth = buildExpressMiddleware() -function buildExpressMiddleware() { - /** - * @dev If the request has a did header, it means we have already confirmed the did - * is registered. If the request has no did, it means we have already - * confirmed the IP address making the request is on our allowlist. If the - * request contains a body, it means we have already verified the digest - * header can be trusted here. - * All of this logic mentioned above lives outside of this app. - * Notice that the absense of a did header or body bypasses any checks below - * this app will still work if the logice above is not in place. - */ - return function (req: Request, _res: Response, next: NextFunction) { - if (req.headers) { - if (req.headers['did'] && req.body) { - if (Object.keys(req.body).length > 0) { - const digest = buildBodyDigest(req.headers['content-type'], req.body) - if (req.headers['digest'] == digest) { - return next() - } else { - throw Error('Body digest verification failed') - } - } - } - } - return next() - } -} - -function buildBodyDigest(contentType: string | undefined, body: any): string | undefined { - if (!body) return - - let hash: Uint8Array | undefined - - if (contentType) { - if (contentType.includes('application/vnd.ipld.car')) { - const carFactory = new CARFactory() - carFactory.codecs.add(DAG_JOSE) - // console.log('Will build a car file from req.body', body) - // try { - // console.log('Will build a car file from req.body (as utf8 string)', u8a.toString(body, 'base64')) - // } catch(e) { - // console.log('Couldn\'t convert req.body to string: ', e) - // } - const car = carFactory.fromBytes(body) - if (!car.roots[0]) throw Error('Missing CAR root') - return car.roots[0].toString() - } else if (contentType.includes('application/json')) { - hash = sha256.hash(u8a.fromString(JSON.stringify(body))) - } - } - - if (!hash) { - // Default to hashing stringified body - hash = sha256.hash(u8a.fromString(JSON.stringify(body))) - } - - return `0x${u8a.toString(hash, 'base16')}` -} diff --git a/src/server.ts b/src/server.ts index a2418147..25ef5b14 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,6 +1,6 @@ import bodyParser from 'body-parser' import { Server } from '@overnightjs/core' -import { auth } from './auth/index.js' +import { auth, parseAllowedDIDs } from './auth/auth.middleware.js' import { expressLoggers, logger, expressErrorLogger } from './logger/index.js' import { Config } from 'node-config-ts' import { multiprocess, type Multiprocess } from './ancillary/multiprocess.js' @@ -20,8 +20,14 @@ export class CeramicAnchorServer extends Server { bodyParser.urlencoded({ extended: true, type: 'application/x-www-form-urlencoded' }) ) this.app.use(expressLoggers) - if (config.requireAuth == true) { - this.app.use(auth) + if (config.auth.required) { + this.app.use( + auth({ + allowedDIDs: parseAllowedDIDs(config.auth.dids), + isRelaxed: config.auth.relaxed, + logger: logger, + }) + ) } this.addControllers(controllers) diff --git a/src/settings.ts b/src/settings.ts index 27b5ebb0..29febf05 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -89,4 +89,8 @@ export enum METRIC_NAMES { CTRL_ERROR_CREATING_REQUEST = 'ctrl_error_creating_request', CTRL_REQUEST_NOT_FOUND = 'ctrl_request_not_found', CTRL_REQUEST_FOUND = 'ctrl_request_found', + + // Auth + AUTH_ALLOWED = 'auth_allowed', + AUTH_DISALLOWED = 'auth_disallowed', }