From a64936ce612816c3a712d5f940ea53d62859c44e Mon Sep 17 00:00:00 2001 From: dakota002 Date: Fri, 26 Apr 2024 14:41:02 -0400 Subject: [PATCH] Basic kafka admin stuff New routes and full form, successful creation of ACLs Cleanup, and testing acl verification method Simplify some functions, fix form --- app.arc | 5 + app/lib/kafka.server.ts | 157 ++++++++++++++++++++++++++++++ app/root.tsx | 8 ++ app/root/header/Header.tsx | 8 +- app/routes/admin.kafka._index.tsx | 134 +++++++++++++++++++++++++ app/routes/admin.kafka.edit.tsx | 95 ++++++++++++++++++ app/routes/admin.kafka.tsx | 91 +++++++++++++++++ package-lock.json | 144 +++++++++++++++++++++++++++ package.json | 1 + sandbox-seed.json | 14 +++ 10 files changed, 656 insertions(+), 1 deletion(-) create mode 100644 app/lib/kafka.server.ts create mode 100644 app/routes/admin.kafka._index.tsx create mode 100644 app/routes/admin.kafka.edit.tsx create mode 100644 app/routes/admin.kafka.tsx diff --git a/app.arc b/app.arc index 28dd2106b7..10fe316f50 100644 --- a/app.arc +++ b/app.arc @@ -94,6 +94,11 @@ legacy_users email *String PointInTimeRecovery true +kafka_acls + topicName *String + group **String + PointInTimeRecovery true + @tables-indexes email_notification_subscription topic *String diff --git a/app/lib/kafka.server.ts b/app/lib/kafka.server.ts new file mode 100644 index 0000000000..f7dd89e0e5 --- /dev/null +++ b/app/lib/kafka.server.ts @@ -0,0 +1,157 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import { tables } from '@architect/functions' +import { paginateScan } from '@aws-sdk/lib-dynamodb' +import type { DynamoDBDocument } from '@aws-sdk/lib-dynamodb' +import { Kafka } from 'gcn-kafka' +import type { AclEntry } from 'kafkajs' +import { + AclOperationTypes, + AclPermissionTypes, + AclResourceTypes, + ResourcePatternTypes, +} from 'kafkajs' + +import { getEnvOrDie } from './env.server' +import type { User } from '~/routes/_auth/user.server' + +export type KafkaACL = { + topicName: string + permissionType: PermissionType + group: string + prefixed: boolean +} + +export type PermissionType = 'producer' | 'consumer' + +export const adminGroup = 'gcn.nasa.gov/gcn-admin' + +const consumerOperations = [AclOperationTypes.READ, AclOperationTypes.DESCRIBE] +const producerOperations = [ + AclOperationTypes.CREATE, + AclOperationTypes.WRITE, + AclOperationTypes.DESCRIBE, +] + +const admin_client_id = getEnvOrDie('KAFKA_ADMIN_CLIENT_ID') +const admin_client_secret = getEnvOrDie('KAFKA_ADMIN_CLIENT_SECRET') +const adminClient = new Kafka({ + client_id: admin_client_id, + client_secret: admin_client_secret, + domain: 'dev.gcn.nasa.gov', // TODO: replace w/ useDomain +}).admin() + +function validateUser(user: User) { + if (!user.groups.includes(adminGroup)) + throw new Response(null, { status: 403 }) +} + +// Not sure if this is a useful method, but may be helpful if we +// want to verify that our table matches the defined kafka acls +export async function verifyKafkaACL(acl: KafkaACL) { + const operations = + acl.permissionType == 'producer' ? producerOperations : consumerOperations + + const promises = operations.map((operation) => + adminClient.describeAcls({ + resourceName: acl.topicName, + resourceType: AclResourceTypes.TOPIC, + host: '*', + permissionType: AclPermissionTypes.ALLOW, + operation, + resourcePatternType: ResourcePatternTypes.LITERAL, + }) + ) + + const results = await Promise.all(promises) + console.log(results) +} + +export async function createKafkaACL(user: User, acl: KafkaACL) { + validateUser(user) + // Save to db + const db = await tables() + await db.kafka_acls.put(acl) + + // Add to Kafka + await adminClient.connect() + await adminClient.createTopics({ + topics: [ + { + topic: acl.topicName, + }, + ], + }) + const acls = + acl.permissionType == 'producer' + ? createProducerAcls(acl) + : createConsumerAcls(acl) + await adminClient.createAcls({ acl: acls }) + await adminClient.disconnect() +} + +export async function getKafkaACLByTopicName(user: User, topicName: string) { + validateUser(user) + const db = await tables() + return (await db.kafka_acls.get({ topicName })) as KafkaACL +} + +export async function getKafkaACLs(user: User) { + validateUser(user) + const db = await tables() + const client = db._doc as unknown as DynamoDBDocument + const TableName = db.name('kafka_acls') + const pages = paginateScan({ client }, { TableName }) + const acls: KafkaACL[] = [] + for await (const page of pages) { + const newACL = page.Items as KafkaACL[] + if (newACL) acls.push(...newACL) + } + return acls +} + +export async function deleteKafkaACL(user: User, acl: KafkaACL) { + validateUser(user) + const db = await tables() + await db.kafka_acls.delete({ topicName: acl.topicName, group: acl.group }) + + const acls = + acl.permissionType == 'producer' + ? createProducerAcls(acl) + : createConsumerAcls(acl) + + await adminClient.connect() + await adminClient.deleteAcls({ filters: acls }) + await adminClient.disconnect() +} + +function createProducerAcls(acl: KafkaACL): AclEntry[] { + // Create, Write, and Describe operations + return mapAclAndOperations(acl, producerOperations) +} + +function createConsumerAcls(acl: KafkaACL): AclEntry[] { + // Read and Describe operations + return mapAclAndOperations(acl, consumerOperations) +} + +function mapAclAndOperations(acl: KafkaACL, operations: AclOperationTypes[]) { + return operations.map((operation) => { + return { + resourceType: AclResourceTypes.TOPIC, + resourceName: acl.topicName, + resourcePatternType: acl.prefixed + ? ResourcePatternTypes.PREFIXED + : ResourcePatternTypes.LITERAL, + principal: `User:${acl.group}`, + host: '*', + operation, + permissionType: AclPermissionTypes.ALLOW, + } + }) +} diff --git a/app/root.tsx b/app/root.tsx index 05d71d0eaa..582629bb95 100644 --- a/app/root.tsx +++ b/app/root.tsx @@ -46,6 +46,7 @@ import { useSpinDelay } from 'spin-delay' import invariant from 'tiny-invariant' import { features, getEnvOrDieInProduction, origin } from './lib/env.server' +import { adminGroup } from './lib/kafka.server' import { DevBanner } from './root/DevBanner' import { Footer } from './root/Footer' import NewsBanner from './root/NewsBanner' @@ -116,6 +117,7 @@ export async function loader({ request }: LoaderFunctionArgs) { const recaptchaSiteKey = getEnvOrDieInProduction('RECAPTCHA_SITE_KEY') const userIsMod = user?.groups.includes(moderatorGroup) const userIsVerifiedSubmitter = user?.groups.includes(group) + const userIsAdmin = user?.groups.includes(adminGroup) return { origin, @@ -126,6 +128,7 @@ export async function loader({ request }: LoaderFunctionArgs) { idp, userIsMod, userIsVerifiedSubmitter, + userIsAdmin, } } @@ -165,6 +168,11 @@ export function useSubmitterStatus() { return userIsVerifiedSubmitter } +export function useAdminStatus() { + const { userIsAdmin } = useLoaderDataRoot() + return userIsAdmin +} + export function useRecaptchaSiteKey() { const { recaptchaSiteKey } = useLoaderDataRoot() return recaptchaSiteKey diff --git a/app/root/header/Header.tsx b/app/root/header/Header.tsx index dde3947adc..1e55d4c8a1 100644 --- a/app/root/header/Header.tsx +++ b/app/root/header/Header.tsx @@ -17,7 +17,7 @@ import { useEffect, useState } from 'react' import { useClickAnyWhere, useWindowSize } from 'usehooks-ts' import { Meatball } from '~/components/meatball/Meatball' -import { useEmail, useUserIdp } from '~/root' +import { useAdminStatus, useEmail, useUserIdp } from '~/root' import styles from './header.module.css' @@ -74,6 +74,7 @@ export function Header() { const [expanded, setExpanded] = useState(false) const [userMenuIsOpen, setUserMenuIsOpen] = useState(false) const isMobile = useWindowSize().width < 1024 + const userIsAdmin = useAdminStatus() function toggleMobileNav() { setExpanded((expanded) => !expanded) @@ -162,6 +163,11 @@ export function Header() { Profile , + userIsAdmin && ( + + Admin + + ), Peer Endorsements , diff --git a/app/routes/admin.kafka._index.tsx b/app/routes/admin.kafka._index.tsx new file mode 100644 index 0000000000..0261d1348d --- /dev/null +++ b/app/routes/admin.kafka._index.tsx @@ -0,0 +1,134 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import type { LoaderFunctionArgs } from '@remix-run/node' +import { useFetcher, useLoaderData } from '@remix-run/react' +import type { ModalRef } from '@trussworks/react-uswds' +import { + Button, + Grid, + Icon, + Modal, + ModalFooter, + ModalHeading, + ModalToggleButton, +} from '@trussworks/react-uswds' +import { useRef } from 'react' + +import { getUser } from './_auth/user.server' +import HeadingWithAddButton from '~/components/HeadingWithAddButton' +import SegmentedCards from '~/components/SegmentedCards' +import { getGroups } from '~/lib/cognito.server' +import type { KafkaACL } from '~/lib/kafka.server' +import { getKafkaACLs } from '~/lib/kafka.server' + +export async function loader({ request }: LoaderFunctionArgs) { + const user = await getUser(request) + if (!user) throw new Response(null, { status: 403 }) + const aclData = await getKafkaACLs(user) + const userGroups = (await getGroups()) + .filter((group) => group.GroupName?.startsWith('gcn.nasa.gov/')) + .map((group) => group.GroupName) + + return { aclData, userGroups } +} + +export default function Index() { + const { aclData } = useLoaderData() + + return ( + <> + Kafka Admin +

Kafka ACLs

+ + {aclData.map((x, index) => ( + + ))} + + + ) +} + +function KafkaAclCard({ acl }: { acl: KafkaACL }) { + const ref = useRef(null) + const fetcher = useFetcher() + const disabled = fetcher.state !== 'idle' + + return ( + <> + +
+
+ + Topic: {acl.topicName} + +
+
+ + Permission Type: {acl.permissionType} + +
+
+ + Group: {acl.group} + +
+
+
+ + + Delete + +
+ + + + + + + + Delete Kafka ACL + + + + + Cancel + + + + + + + ) +} diff --git a/app/routes/admin.kafka.edit.tsx b/app/routes/admin.kafka.edit.tsx new file mode 100644 index 0000000000..7d9af231a1 --- /dev/null +++ b/app/routes/admin.kafka.edit.tsx @@ -0,0 +1,95 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import type { LoaderFunctionArgs } from '@remix-run/node' +import { Form, useLoaderData } from '@remix-run/react' +import { + Button, + Checkbox, + Label, + Select, + TextInput, +} from '@trussworks/react-uswds' + +import { getUser } from './_auth/user.server' +import { getGroups } from '~/lib/cognito.server' + +export async function loader({ request }: LoaderFunctionArgs) { + const user = await getUser(request) + if (!user) throw new Response(null, { status: 403 }) + const userGroups = (await getGroups()) + .filter((group) => group.GroupName?.startsWith('gcn.nasa.gov/')) + .map((group) => group.GroupName) + + return { userGroups } +} + +export default function Kafka() { + const { userGroups } = useLoaderData() + return +} + +function KafkaAclForm({ groups }: { groups: string[] }) { + return ( + <> +

Create Kafka ACLs

+
+ + console.log(e.target.value)} + /> + + + + Producer will generate ACLs for the Create, Write, and Describe + operations. Consumer will generate ACLs for the Read and Describe + operations + + + + +
+ + If yes, submission will also trigger th generation of ACLs for the + provided topic name as a PREFIXED topic with a period included at + the end. For example, if checked, a topic of `gcn.notices.icecube` + will result in ACLs for both `gcn.notices.icecube` (literal) and + `gcn.notices.icecube.` (prefixed). + +
+ + + + ) +} diff --git a/app/routes/admin.kafka.tsx b/app/routes/admin.kafka.tsx new file mode 100644 index 0000000000..35ade3e434 --- /dev/null +++ b/app/routes/admin.kafka.tsx @@ -0,0 +1,91 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import type { ActionFunctionArgs } from '@remix-run/node' +import { NavLink, Outlet } from '@remix-run/react' +import { GridContainer, SideNav } from '@trussworks/react-uswds' + +import { getUser } from './_auth/user.server' +import type { PermissionType } from '~/lib/kafka.server' +import { createKafkaACL, deleteKafkaACL } from '~/lib/kafka.server' +import { getFormDataString } from '~/lib/utils' + +export async function action({ request }: ActionFunctionArgs) { + const user = await getUser(request) + if (!user) throw new Response(null, { status: 403 }) + const data = await request.formData() + const intent = getFormDataString(data, 'intent') + const topicName = getFormDataString(data, 'topicName') + const permissionType = getFormDataString( + data, + 'permissionType' + ) as PermissionType + const group = getFormDataString(data, 'group') + const includePrefixed = getFormDataString(data, 'includePrefixed') + if (!topicName || !permissionType || !group) + throw new Response(null, { status: 400 }) + const promises = [] + + switch (intent) { + case 'delete': + promises.push( + deleteKafkaACL(user, { + topicName, + permissionType, + group, + prefixed: false, + }) + ) + break + case 'create': + promises.push( + createKafkaACL(user, { + topicName, + permissionType, + group, + prefixed: false, + }) + ) + + if (includePrefixed) + promises.push( + createKafkaACL(user, { + topicName: `${topicName}.`, + permissionType, + group, + prefixed: true, + }) + ) + break + default: + break + } + await Promise.all(promises) + + return null +} + +export default function Kafka() { + return ( + +
+
+ + Kafka + , + ]} + /> +
+
+ +
+
+ + ) +} diff --git a/package-lock.json b/package-lock.json index 01fc44adde..0f1c093971 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "diff": "^5.2.0", "downshift": "^7.2.1", "email-validator": "^2.0.4", + "gcn-kafka": "^0.2.1", "github-slugger": "^2.0.0", "hast-util-find-and-replace": "^5.0.1", "hastscript": "^8.0.0", @@ -7011,6 +7012,128 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/@mongodb-js/zstd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.0.tgz", + "integrity": "sha512-sKHsJU2MXsp822IFXOHw/4mpFulScNHpZzVy1Zi5k5wBsdiAPx1QramyOXZkpacla+2QPEC/s7TxPlEhG/HuNQ==", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mongodb-js/zstd-darwin-arm64": "1.2.0", + "@mongodb-js/zstd-darwin-x64": "1.2.0", + "@mongodb-js/zstd-linux-arm64-gnu": "1.2.0", + "@mongodb-js/zstd-linux-arm64-musl": "1.2.0", + "@mongodb-js/zstd-linux-x64-gnu": "1.2.0", + "@mongodb-js/zstd-linux-x64-musl": "1.2.0", + "@mongodb-js/zstd-win32-x64-msvc": "1.2.0" + } + }, + "node_modules/@mongodb-js/zstd-darwin-arm64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.2.0.tgz", + "integrity": "sha512-QWgW6IkWp3ErBXOvlOj9lw3lwMfey7eXh/p/Srb/7sEiu1e0yEO+LQ8IctmDWh8bfznKXmwUC0h7LKDbYR30yw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-darwin-x64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.2.0.tgz", + "integrity": "sha512-VnxYO8P2SWubdnydGId5+6veO6Ki6nxCr/pTaDZd8s4Urn6bDdXSX6YsZ0r42dO3Fa0FVYzrlcVAuNB67e2b6w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-arm64-gnu": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.2.0.tgz", + "integrity": "sha512-TYF0XgNJW6UrvtY2u4Uuo5HiVWNgWNZ/ae2BhVp8hNsDhwFqb/YNoyiZqBei6whUwr8hecMy0UaHAXm3h+O2+Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-arm64-musl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-musl/-/zstd-linux-arm64-musl-1.2.0.tgz", + "integrity": "sha512-e2ClmJI1BvJq23VSLH14hgjjjcMOad3R/Ap7Q7dTa1uiVSJG4xKd2CmrWQgX1Az4/EfUMWEI7pb4yuanbdd2AQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-x64-gnu": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.2.0.tgz", + "integrity": "sha512-JuoK8lxUlkFPDBfsBUJKnLxpXA5ar+v7G43lIUlBKgjOp5aEWO/qQp5sNgCRnYA7x6PItYqIkEJjsays4N6JOA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-x64-musl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-musl/-/zstd-linux-x64-musl-1.2.0.tgz", + "integrity": "sha512-pSb1iUF3Gc/qrJuP/Mi5ry4YFAUdUVFKNRZh1KTDDhSWyRCLd9gKcNdRnXqJjIdeGGEKf4bhtZAbYw4i/g0foA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-win32-x64-msvc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.2.0.tgz", + "integrity": "sha512-iz4Yl+WK3yr/4Yg6F4tKz3X9+yMZDK6pyBMA0CdXydSDZs6o2XQ2I0ZSu3oSk/ACfaZX3SNfRi3XTGgAM1eKZA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nasa-gcn/architect-functions-search": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@nasa-gcn/architect-functions-search/-/architect-functions-search-1.0.0.tgz", @@ -14785,6 +14908,19 @@ "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", "dev": true }, + "node_modules/gcn-kafka": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/gcn-kafka/-/gcn-kafka-0.2.1.tgz", + "integrity": "sha512-zFZyqxk4VUHRY78wLPyKZet3xrbd3wJBRa2DxRTQTZXeORzTl21pb+qpHNXNQTPgufQJ9vyFpKd2sB6/gTnYfA==", + "dependencies": { + "@mongodb-js/zstd": "^1.2.0", + "kafkajs": "^2.0.2", + "openid-client": "^5.1.6" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/generic-names": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/generic-names/-/generic-names-4.0.0.tgz", @@ -18125,6 +18261,14 @@ "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", "dev": true }, + "node_modules/kafkajs": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", + "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/keyboardevent-key-polyfill": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keyboardevent-key-polyfill/-/keyboardevent-key-polyfill-1.1.0.tgz", diff --git a/package.json b/package.json index 853fa2cff6..62b92beb07 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "diff": "^5.2.0", "downshift": "^7.2.1", "email-validator": "^2.0.4", + "gcn-kafka": "^0.2.1", "github-slugger": "^2.0.0", "hast-util-find-and-replace": "^5.0.1", "hastscript": "^8.0.0", diff --git a/sandbox-seed.json b/sandbox-seed.json index 8be8e4b9b0..c0a53f3f3b 100644 --- a/sandbox-seed.json +++ b/sandbox-seed.json @@ -5125,5 +5125,19 @@ "affiliation": "Example", "submit": 1 } + ], + "kafka_acls": [ + { + "topicName": "test_topic_created_from_website", + "permissionType": "consumer", + "group": "gcn.nasa.gov/kafka-gcn-test-consumer", + "prefixed": false + }, + { + "topicName": "test_topic_created_from_website", + "permissionType": "producer", + "group": "gcn.nasa.gov/kafka-gcn-test-producer", + "prefixed": false + } ] }