From cf051bb75c684bfddcaa8688dfab769162aa479c Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 22 Oct 2024 14:38:04 -0500 Subject: [PATCH 01/86] Decrement page if empty on Cardlist If you removed an item from the cardlist so that the page is empty afterwards then, and only then, change the page Closes https://github.com/PelicanPlatform/pelican/issues/1473 --- web_ui/frontend/components/CardList.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web_ui/frontend/components/CardList.tsx b/web_ui/frontend/components/CardList.tsx index c4be45577..98e463e9d 100644 --- a/web_ui/frontend/components/CardList.tsx +++ b/web_ui/frontend/components/CardList.tsx @@ -24,9 +24,11 @@ export function CardList({ data, Card, cardProps }: CardListProps) { const PAGE_SIZE = 5; const [page, setPage] = useState(1); - // Reset the page on data length change + // Minus the page if the data length changes useEffect(() => { - setPage(1); + if (data?.length && page > Math.ceil(data.length / PAGE_SIZE)) { + setPage(Math.max(1, Math.ceil(data.length / PAGE_SIZE))); + } }, [data?.length]); const count = useMemo(() => { From 7acaea00442f6271ac49195860b196cde95d6772 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Fri, 25 Oct 2024 08:31:04 -0500 Subject: [PATCH 02/86] Remove the use of index.d.ts files Allows us to have more consistent styles with this code and the other code that we write that is compiled for js consumption. If we need a .d.ts file we can let the compiler create it. --- .../director/components/DirectorDropdown.tsx | 2 +- .../CustomRegistrationField/BooleanField.tsx | 4 +-- .../EnumerationField.tsx | 4 +-- .../EpochTimeField.tsx | 4 +-- .../CustomRegistrationField/IntegerField.tsx | 4 +-- .../CustomRegistrationField/PubkeyField.tsx | 4 +-- .../CustomRegistrationField/StringField.tsx | 4 +-- .../CustomRegistrationField/index.d.ts | 28 ------------------- .../CustomRegistrationField/index.tsx | 22 ++++++++++++--- .../frontend/app/registry/components/Form.tsx | 6 ++-- .../app/registry/components/PostPage.tsx | 2 +- .../app/registry/components/PutPage.tsx | 2 +- .../frontend/app/registry/components/index.ts | 5 ++++ .../frontend/app/registry/components/util.tsx | 7 ++++- .../Namespace/InformationDropdown.tsx | 3 +- .../frontend/components/Namespace/index.d.tsx | 21 -------------- .../frontend/components/Namespace/index.tsx | 23 +++++++++++++++ web_ui/frontend/{index.d.ts => index.ts} | 8 ++++-- 18 files changed, 76 insertions(+), 77 deletions(-) delete mode 100644 web_ui/frontend/app/registry/components/CustomRegistrationField/index.d.ts create mode 100644 web_ui/frontend/app/registry/components/index.ts delete mode 100644 web_ui/frontend/components/Namespace/index.d.tsx rename web_ui/frontend/{index.d.ts => index.ts} (89%) diff --git a/web_ui/frontend/app/director/components/DirectorDropdown.tsx b/web_ui/frontend/app/director/components/DirectorDropdown.tsx index 15d7bcb03..60d1c433d 100644 --- a/web_ui/frontend/app/director/components/DirectorDropdown.tsx +++ b/web_ui/frontend/app/director/components/DirectorDropdown.tsx @@ -107,7 +107,7 @@ const directoryListToTreeHelper = ( tree[path[0]] = {}; } - tree[path[0]] = directoryListToTreeHelper(path.slice(1), tree[path[0]]); + tree[path[0]] = directoryListToTreeHelper(path.slice(1), tree[path[0]] as StringTree); return tree; }; diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/BooleanField.tsx b/web_ui/frontend/app/registry/components/CustomRegistrationField/BooleanField.tsx index f541024ff..b3bbfe61f 100644 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/BooleanField.tsx +++ b/web_ui/frontend/app/registry/components/CustomRegistrationField/BooleanField.tsx @@ -9,7 +9,7 @@ import React, { ChangeEvent, ReactNode, SyntheticEvent, useMemo } from 'react'; import { createId } from '@/components/configuration/util'; import FormHelperText from '@mui/material/FormHelperText'; -import type { CustomRegistrationFieldProps } from './index.d'; +import type { BaseCustomRegistrationFieldProps } from './index'; const BooleanField = ({ onChange, @@ -18,7 +18,7 @@ const BooleanField = ({ required, description, value, -}: CustomRegistrationFieldProps) => { +}: BaseCustomRegistrationFieldProps) => { const id = useMemo(() => createId(name), [name]); const labelId = useMemo(() => `${id}-label`, [id]); diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/EnumerationField.tsx b/web_ui/frontend/app/registry/components/CustomRegistrationField/EnumerationField.tsx index d0f6b699b..ec43e79e1 100644 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/EnumerationField.tsx +++ b/web_ui/frontend/app/registry/components/CustomRegistrationField/EnumerationField.tsx @@ -1,7 +1,7 @@ import { Autocomplete, TextField } from '@mui/material'; import React, { useMemo } from 'react'; -import type { CustomRegistrationFieldProps } from './index.d'; +import type { BaseCustomRegistrationFieldProps } from './index'; const EnumerationField = ({ onChange, @@ -11,7 +11,7 @@ const EnumerationField = ({ description, value, options, -}: CustomRegistrationFieldProps) => { +}: BaseCustomRegistrationFieldProps) => { const textValue = useMemo( () => options?.find((option) => option.id === value), [value, options] diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/EpochTimeField.tsx b/web_ui/frontend/app/registry/components/CustomRegistrationField/EpochTimeField.tsx index 38d8c7a22..141aefc42 100644 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/EpochTimeField.tsx +++ b/web_ui/frontend/app/registry/components/CustomRegistrationField/EpochTimeField.tsx @@ -6,7 +6,7 @@ import FormControl from '@mui/material/FormControl'; import FormHelperText from '@mui/material/FormHelperText'; import { DateTime } from 'luxon'; -import type { CustomRegistrationFieldProps } from './index.d'; +import type { BaseCustomRegistrationFieldProps } from './index'; const EpochTimeField = ({ onChange, @@ -15,7 +15,7 @@ const EpochTimeField = ({ required, description, value, -}: CustomRegistrationFieldProps) => { +}: BaseCustomRegistrationFieldProps) => { return ( { if (value && isNaN(Number(value))) { @@ -17,7 +17,7 @@ const IntegerField = ({ required, description, value, -}: CustomRegistrationFieldProps) => { +}: BaseCustomRegistrationFieldProps) => { const [error, setError] = React.useState(undefined); // Check that the value is a number or undefined throwing error if not diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx b/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx index 703e0d15e..6d7d5135b 100644 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx +++ b/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { StringField } from './StringField'; -import type { CustomRegistrationFieldProps } from './index.d'; +import type { BaseCustomRegistrationFieldProps } from './index'; const JWKPlaceholder = { keys: [ @@ -25,7 +25,7 @@ const pubkeyValidator = (value: string) => { } }; -const PubkeyField = ({ ...props }: CustomRegistrationFieldProps) => { +const PubkeyField = ({ ...props }: BaseCustomRegistrationFieldProps) => { return ( & - CustomRegistrationFieldProps; + BaseCustomRegistrationFieldProps; interface StringFieldProps extends TextFieldProps { validator?: (value: string) => string | undefined; diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/index.d.ts b/web_ui/frontend/app/registry/components/CustomRegistrationField/index.d.ts deleted file mode 100644 index e00428efe..000000000 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/index.d.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { CustomRegistrationField } from '@/components/configuration'; -import { Alert as AlertType, Namespace } from '@/index'; - -export interface NamespaceFormPage { - update: (data: Partial) => Promise; -} - -export interface CustomRegistrationProps extends CustomRegistrationField { - displayed_name: string; -} - -export type CustomRegistrationPropsEnum = - | (CustomRegistrationProps & { type: 'int' }) - | (CustomRegistrationProps & { type: 'string' }) - | (CustomRegistrationProps & { type: 'bool' }) - | (CustomRegistrationProps & { type: 'datetime' }) - | (CustomRegistrationProps & { type: 'enum' }); - -export interface CustomRegistrationFieldProps - extends CustomRegistrationProps { - onChange: (value: T | null) => void; - value?: T; -} - -export type CustomRegistrationFieldPropsEnum = - CustomRegistrationFieldProps & { - type: 'int' | 'string' | 'bool' | 'datetime' | 'enum'; - }; diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/index.tsx b/web_ui/frontend/app/registry/components/CustomRegistrationField/index.tsx index 7c64a464e..dfd07a814 100644 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/index.tsx +++ b/web_ui/frontend/app/registry/components/CustomRegistrationField/index.tsx @@ -1,17 +1,31 @@ -import type { CustomRegistrationField } from '@/components/configuration/index'; + import { BooleanField } from './BooleanField'; -import { ErrorField } from './ErrorField'; import { StringField } from './StringField'; import { IntegerField } from './IntegerField'; import PubkeyField from './PubkeyField'; -import { CustomRegistrationFieldPropsEnum } from './index.d'; import EpochTimeField from '@/app/registry/components/CustomRegistrationField/EpochTimeField'; import EnumerationField from '@/app/registry/components/CustomRegistrationField/EnumerationField'; +import { CustomRegistrationField as CustomRegistrationFieldConfiguration } from '@/components/configuration'; +import type { CustomRegistrationField } from '@/components/configuration'; + +export type CustomRegistrationFieldProps = + | (BaseCustomRegistrationFieldProps & { type: 'int' }) + | (BaseCustomRegistrationFieldProps & { type: 'string' }) + | (BaseCustomRegistrationFieldProps & { type: 'bool' }) + | (BaseCustomRegistrationFieldProps & { type: 'datetime' }) + | (BaseCustomRegistrationFieldProps & { type: 'enum' }); + +export interface BaseCustomRegistrationFieldProps + extends CustomRegistrationFieldConfiguration { + onChange: (value: T | null) => void; + value?: T; + displayed_name: string; +} const CustomRegistrationField = ({ ...props -}: CustomRegistrationFieldPropsEnum) => { +}: CustomRegistrationFieldProps) => { // If the field is the pubkey field, render the pubkey field if (props.type == 'string' && props.name === 'pubkey') { return ; diff --git a/web_ui/frontend/app/registry/components/Form.tsx b/web_ui/frontend/app/registry/components/Form.tsx index 24f8fa128..37326f65c 100644 --- a/web_ui/frontend/app/registry/components/Form.tsx +++ b/web_ui/frontend/app/registry/components/Form.tsx @@ -11,7 +11,7 @@ import { populateKey, submitNamespaceForm, } from '@/app/registry/components/util'; -import { CustomRegistrationPropsEnum } from './CustomRegistrationField/index.d'; +import { CustomRegistrationFieldProps } from './CustomRegistrationField'; import { getErrorMessage } from '@/helpers/util'; interface FormProps { @@ -20,7 +20,7 @@ interface FormProps { } const getRegistrationFields = async (): Promise< - CustomRegistrationPropsEnum[] + Omit[] > => { const response = await fetch('/api/v1.0/registry_ui/namespaces', { method: 'OPTIONS', @@ -57,7 +57,7 @@ const Form = ({ namespace, onSubmit }: FormProps) => { namespace || {} ); - const { data: fields, error } = useSWR( + const { data: fields, error } = useSWR[]>( 'getRegistrationFields', getRegistrationFields, { fallbackData: [] } diff --git a/web_ui/frontend/app/registry/components/PostPage.tsx b/web_ui/frontend/app/registry/components/PostPage.tsx index 6c8ad7be2..f5d6991ee 100644 --- a/web_ui/frontend/app/registry/components/PostPage.tsx +++ b/web_ui/frontend/app/registry/components/PostPage.tsx @@ -25,7 +25,7 @@ import { Alert as AlertType, Namespace } from '@/index'; import Form from '@/app/registry/components/Form'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { submitNamespaceForm } from '@/app/registry/components/util'; -import type { NamespaceFormPage } from './CustomRegistrationField/index.d'; +import { NamespaceFormPage } from '@/app/registry/components'; const PostPage = ({ update }: NamespaceFormPage) => { const [fromUrl, setFromUrl] = useState(undefined); diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index b5e1ebce1..813aa1acb 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -41,7 +41,7 @@ import { getNamespace, submitNamespaceForm, } from '@/app/registry/components/util'; -import type { NamespaceFormPage } from './CustomRegistrationField/index.d'; +import { NamespaceFormPage } from '@/app/registry/components' const PutPage = ({ update }: NamespaceFormPage) => { const [id, setId] = useState(undefined); diff --git a/web_ui/frontend/app/registry/components/index.ts b/web_ui/frontend/app/registry/components/index.ts new file mode 100644 index 000000000..86214d7a3 --- /dev/null +++ b/web_ui/frontend/app/registry/components/index.ts @@ -0,0 +1,5 @@ +import { Alert as AlertType, Namespace } from '@/index'; + +export interface NamespaceFormPage { + update: (data: Partial) => Promise; +} diff --git a/web_ui/frontend/app/registry/components/util.tsx b/web_ui/frontend/app/registry/components/util.tsx index 0eb3398ae..9a55fd031 100644 --- a/web_ui/frontend/app/registry/components/util.tsx +++ b/web_ui/frontend/app/registry/components/util.tsx @@ -26,7 +26,12 @@ export const calculateKeys = (key: string) => { return [key]; }; -export const getValue = (o: any, key: string[]): string | undefined => { +/** + * Get the value of a key in an object + * @param o Object to get the value from + * @param key List of keys to traverse + */ +export const getValue = (o: Record | undefined, key: string[]): any => { if (o === undefined) { return undefined; } diff --git a/web_ui/frontend/components/Namespace/InformationDropdown.tsx b/web_ui/frontend/components/Namespace/InformationDropdown.tsx index db87fc751..752fe37b6 100644 --- a/web_ui/frontend/components/Namespace/InformationDropdown.tsx +++ b/web_ui/frontend/components/Namespace/InformationDropdown.tsx @@ -1,7 +1,6 @@ import { Box, Tooltip, Collapse, Grid, Typography } from '@mui/material'; import React from 'react'; -import { NamespaceAdminMetadata } from './index.d'; -import { Dropdown, InformationSpan } from '@/components'; +import { Dropdown, InformationSpan, NamespaceAdminMetadata } from '@/components'; interface InformationDropdownProps { adminMetadata: NamespaceAdminMetadata; diff --git a/web_ui/frontend/components/Namespace/index.d.tsx b/web_ui/frontend/components/Namespace/index.d.tsx deleted file mode 100644 index c7292df96..000000000 --- a/web_ui/frontend/components/Namespace/index.d.tsx +++ /dev/null @@ -1,21 +0,0 @@ -import { PendingCardProps } from './PendingCard'; -import { CardProps } from './Card'; - -export interface NamespaceAdminMetadata { - user_id: string; - description: string; - site_name: string; - institution: string; - security_contact_user_id: string; - status: 'Pending' | 'Approved' | 'Denied' | 'Unknown'; - approver_id: number; - approved_at: string; - created_at: string; - updated_at: string; -} - -export interface FlatObject { - [key: string]: Exclude; -} - -export type NamespaceCardProps = CardProps & PendingCardProps; diff --git a/web_ui/frontend/components/Namespace/index.tsx b/web_ui/frontend/components/Namespace/index.tsx index 77dbe881d..7f7e4ac4b 100644 --- a/web_ui/frontend/components/Namespace/index.tsx +++ b/web_ui/frontend/components/Namespace/index.tsx @@ -15,6 +15,29 @@ export { NamespaceIcon, }; +import { PendingCardProps } from './PendingCard'; +import { CardProps } from './Card'; + +export interface NamespaceAdminMetadata { + user_id: string; + description: string; + site_name: string; + institution: string; + security_contact_user_id: string; + status: 'Pending' | 'Approved' | 'Denied' | 'Unknown'; + approver_id: number; + approved_at: string; + created_at: string; + updated_at: string; +} + +export interface FlatObject { + [key: string]: Exclude; +} + +export type NamespaceCardProps = CardProps & PendingCardProps; + + export const getServerType = (namespace: Namespace) => { // If the namespace is empty the value is undefined if (namespace?.prefix == null || namespace.prefix == '') { diff --git a/web_ui/frontend/index.d.ts b/web_ui/frontend/index.ts similarity index 89% rename from web_ui/frontend/index.d.ts rename to web_ui/frontend/index.ts index 7be168325..dfe6b1e9c 100644 --- a/web_ui/frontend/index.d.ts +++ b/web_ui/frontend/index.ts @@ -34,9 +34,11 @@ export interface Capabilities { DirectReads: boolean; } -export type StringTree = Record; +export interface StringTree { + [key: string]: StringTree | true; +} -interface Alert { +export interface Alert { severity: 'error' | 'warning' | 'info' | 'success'; message: string; } @@ -50,7 +52,7 @@ export interface Namespace { custom_fields?: Record; } -interface Institution { +export interface Institution { id: string; name: string; } From 0b672326b75cd2bb39c1ff97d050e1af9371c375 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Fri, 25 Oct 2024 16:20:35 -0500 Subject: [PATCH 03/86] Uniform Error Handling --- .../app/(login)/initialization/code/page.tsx | 50 ++-- .../(login)/initialization/password/page.tsx | 61 ++-- web_ui/frontend/app/(login)/login/page.tsx | 46 ++- .../app/director/components/DirectorCard.tsx | 107 ++----- web_ui/frontend/app/director/page.tsx | 22 +- web_ui/frontend/app/layout.tsx | 7 +- .../frontend/app/registry/cache/edit/page.tsx | 3 +- .../app/registry/cache/register/page.tsx | 5 +- .../frontend/app/registry/components/Form.tsx | 32 +-- .../app/registry/components/PostPage.tsx | 43 +-- .../app/registry/components/PutPage.tsx | 64 +++-- .../frontend/app/registry/components/index.ts | 2 +- .../frontend/app/registry/components/util.tsx | 52 +--- web_ui/frontend/app/registry/denied/page.tsx | 76 ++--- .../app/registry/namespace/edit/page.tsx | 2 +- .../app/registry/namespace/register/page.tsx | 2 +- .../app/registry/origin/edit/page.tsx | 4 +- .../app/registry/origin/register/page.tsx | 4 +- web_ui/frontend/app/registry/page.tsx | 81 ++---- web_ui/frontend/app/test/page.tsx | 55 ++++ web_ui/frontend/components/AlertPortal.tsx | 32 ++- web_ui/frontend/components/AlertProvider.tsx | 72 +++++ web_ui/frontend/components/CodeBlock.tsx | 25 ++ web_ui/frontend/components/Namespace/Card.tsx | 33 +-- .../components/Namespace/DeniedCard.tsx | 108 ++----- .../components/Namespace/PendingCard.tsx | 81 ++---- .../ThemeProvider.tsx} | 6 +- web_ui/frontend/helpers/api.ts | 267 ++++++++++++++++++ .../frontend/helpers/{login.tsx => login.ts} | 0 web_ui/frontend/helpers/{util.tsx => util.ts} | 74 ++++- 30 files changed, 815 insertions(+), 601 deletions(-) create mode 100644 web_ui/frontend/app/test/page.tsx create mode 100644 web_ui/frontend/components/AlertProvider.tsx create mode 100644 web_ui/frontend/components/CodeBlock.tsx rename web_ui/frontend/{public/theme.tsx => components/ThemeProvider.tsx} (92%) create mode 100644 web_ui/frontend/helpers/api.ts rename web_ui/frontend/helpers/{login.tsx => login.ts} (100%) rename web_ui/frontend/helpers/{util.tsx => util.ts} (50%) diff --git a/web_ui/frontend/app/(login)/initialization/code/page.tsx b/web_ui/frontend/app/(login)/initialization/code/page.tsx index 05f784bf6..18791c9b9 100644 --- a/web_ui/frontend/app/(login)/initialization/code/page.tsx +++ b/web_ui/frontend/app/(login)/initialization/code/page.tsx @@ -18,13 +18,16 @@ 'use client'; -import { Box, Typography, Grow } from '@mui/material'; +import { Box, Typography } from '@mui/material'; import { useRouter } from 'next/navigation'; -import { useState } from 'react'; +import { useContext, useState } from 'react'; import CodeInput, { Code } from '../../components/CodeInput'; import LoadingButton from '../../components/LoadingButton'; -import { getErrorMessage } from '@/helpers/util'; +import { initLogin } from '@/helpers/api'; +import { alertOnError } from '@/helpers/util'; +import { AlertDispatchContext } from '@/components/AlertProvider'; + export default function Home() { const router = useRouter(); @@ -37,11 +40,11 @@ export default function Home() { undefined, ]); let [loading, setLoading] = useState(false); - let [error, setError] = useState(undefined); + + const dispatch = useContext(AlertDispatchContext) const setCode = (code: Code) => { _setCode(code); - setError(undefined); if (!code.includes(undefined)) { submit(code.map((x) => x!.toString()).join('')); @@ -51,26 +54,15 @@ export default function Home() { async function submit(code: string) { setLoading(true); - try { - let response = await fetch('/api/v1.0/auth/initLogin', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - code: code, - }), - }); - - if (response.ok) { - router.push('../password/'); - } else { - setLoading(false); - setError(await getErrorMessage(response)); - } - } catch { + let response = await alertOnError( + async () => await initLogin(code), + 'Could not login', + dispatch + ) + if(response) { + router.push('../password/'); + } else { setLoading(false); - setError('Could not connect to server'); } } @@ -97,16 +89,6 @@ export default function Home() {
- - - {error} - - (''); let [confirmPassword, _setConfirmPassword] = useState(''); let [loading, setLoading] = useState(false); - let [error, setError] = useState(undefined); + async function submit(password: string) { setLoading(true); - try { - let response = await fetch('/api/v1.0/auth/resetLogin', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - password: password, - }), - }); - - if (response.ok) { - router.push('/'); - } else { - setLoading(false); - setError(await getErrorMessage(response)); - } - } catch { + let response = await alertOnError( + async () => await resetLogin(password), + 'Could not login', + dispatch + ) + if(response) { + router.push('../password/'); + } else { setLoading(false); - setError('Could not connect to server'); } } @@ -66,7 +60,16 @@ export default function Home() { if (password == confirmPassword) { submit(password); } else { - setError('Passwords do not match'); + dispatch({ + type: 'openAlert', + payload: { + alertProps: { + severity: 'warning' + }, + message: 'Passwords do not match', + onClose: () => dispatch({ type: 'closeAlert' }) + } + }) } } @@ -89,7 +92,6 @@ export default function Home() { InputProps: { onChange: (e) => { _setPassword(e.target.value); - setError(undefined); }, }, }} @@ -102,7 +104,6 @@ export default function Home() { InputProps: { onChange: (e) => { _setConfirmPassword(e.target.value); - setError(undefined); }, }, error: password != confirmPassword, @@ -112,16 +113,6 @@ export default function Home() { /> - - - {error} - - { + + const dispatch = useContext(AlertDispatchContext); + const router = useRouter(); const { mutate } = useSWR('getUser', getUser); @@ -68,34 +74,20 @@ const AdminLogin = () => { async function submit(password: string) { setLoading(true); - let response; - try { - response = await fetch('/api/v1.0/auth/login', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - user: 'admin', - password: password, - }), - }); - - if (response.ok) { - await mutate(getUser); - - const url = new URL(window.location.href); - let returnUrl = url.searchParams.get('returnURL') || ''; - returnUrl = returnUrl.replace(`/view`, ''); - router.push(returnUrl ? returnUrl : '../'); - } else { - setLoading(false); - setError(await getErrorMessage(response)); - } - } catch (e) { - console.error(e); + const response = await alertOnError( + async () => await login(password), + "Could not login", + dispatch + ) + if (response) { + await mutate(getUser); + + const url = new URL(window.location.href); + let returnUrl = url.searchParams.get('returnURL') || ''; + returnUrl = returnUrl.replace(`/view`, ''); + router.push(returnUrl ? returnUrl : '../'); + } else { setLoading(false); - setError('Could not connect to server'); } } diff --git a/web_ui/frontend/app/director/components/DirectorCard.tsx b/web_ui/frontend/app/director/components/DirectorCard.tsx index 72419fd18..305ee0950 100644 --- a/web_ui/frontend/app/director/components/DirectorCard.tsx +++ b/web_ui/frontend/app/director/components/DirectorCard.tsx @@ -1,5 +1,5 @@ import { Authenticated, secureFetch } from '@/helpers/login'; -import React, { useRef, useState } from 'react'; +import React, { useContext, useRef, useState } from 'react'; import { Avatar, Box, @@ -21,8 +21,10 @@ import { NamespaceIcon } from '@/components/Namespace/index'; import useSWR from 'swr'; import Link from 'next/link'; import { User } from '@/index'; -import { getErrorMessage } from '@/helpers/util'; +import { alertOnError, getErrorMessage } from '@/helpers/util'; import { DirectorDropdown } from '@/app/director/components/DirectorDropdown'; +import { allowServer, filterServer } from '@/helpers/api'; +import { AlertDispatchContext } from '@/components/AlertProvider'; export interface DirectorCardProps { server: Server; @@ -30,11 +32,11 @@ export interface DirectorCardProps { } export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { - const [filtered, setFiltered] = useState(server.filtered); - const [error, setError] = useState(undefined); const [disabled, setDisabled] = useState(false); const [dropdownOpen, setDropdownOpen] = useState(false); + const dispatch = useContext(AlertDispatchContext); + const { mutate } = useSWR('getServers'); return ( @@ -75,38 +77,34 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { { - x.stopPropagation(); + onClick={async (e) => { + e.stopPropagation(); // Disable the switch setDisabled(true); - // Provide optimistic feedback - setFiltered(!filtered); - // Update the server - let error; - if (filtered) { - error = await allowServer(server.name); - } else { - error = await filterServer(server.name); - } + await alertOnError( + async () => { + if (server.filtered) { + await allowServer(server.name); + } else { + await filterServer(server.name); + } + }, + 'Failed to toggle server status', + dispatch + ) - // Revert if we were too optimistic - if (error) { - setFiltered(!filtered); - setError(error); - } else { - mutate(); - } + mutate(); setDisabled(false); }} /> } - label={!filtered ? 'Active' : 'Disabled'} + label={server.filtered ? 'Disabled' : 'Active'} /> @@ -127,69 +125,8 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { - - setError(undefined)} - > - setError(undefined)} - severity='error' - variant='filled' - sx={{ width: '100%' }} - > - {error} -
- If this error persists on reload, please file a ticket via the (?) - in the bottom left. - - - ); }; -const filterServer = async (name: string): Promise => { - try { - const response = await secureFetch( - `/api/v1.0/director_ui/servers/filter/${name}`, - { - method: 'PATCH', - } - ); - if (response.ok) { - return; - } else { - return await getErrorMessage(response); - } - } catch (e) { - if (e instanceof Error) { - return e.message; - } - return 'Could not connect to server'; - } -}; - -const allowServer = async (name: string): Promise => { - try { - const response = await secureFetch( - `/api/v1.0/director_ui/servers/allow/${name}`, - { - method: 'PATCH', - } - ); - if (response.ok) { - return; - } else { - return await getErrorMessage(response); - } - } catch (e) { - if (e instanceof Error) { - return e.message; - } - return 'Could not connect to server'; - } -}; - export default DirectorCard; diff --git a/web_ui/frontend/app/director/page.tsx b/web_ui/frontend/app/director/page.tsx index 3b1f64c1a..ef7bfde10 100644 --- a/web_ui/frontend/app/director/page.tsx +++ b/web_ui/frontend/app/director/page.tsx @@ -19,23 +19,33 @@ 'use client'; import { Box, Grid, Skeleton, Typography } from '@mui/material'; -import { useMemo } from 'react'; +import { useContext, useMemo } from 'react'; import useSWR from 'swr'; import { Server } from '@/index'; import { DirectorCardList, - DirectorCard, - DirectorCardProps, } from './components'; import { getUser } from '@/helpers/login'; import FederationOverview from '@/components/FederationOverview'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { PaddedContent } from '@/components/layout'; +import { fetchApi } from '@/helpers/api'; +import { alertOnError } from '@/helpers/util'; +import { AlertDispatchContext } from '@/components/AlertProvider'; export default function Page() { - const { data } = useSWR('getServers', getServers); - const { data: user, error } = useSWR('getUser', getUser); + const dispatch = useContext(AlertDispatchContext); + + const { data } = useSWR( + 'getServers', + async () => await alertOnError(getServers, 'Failed to fetch servers', dispatch) + ); + + const { data: user, error } = useSWR( + 'getUser', + () => alertOnError(getUser, "Failed to fetch user", dispatch) + ); const cacheData = useMemo(() => { return data?.filter((server) => server.type === 'Cache'); @@ -97,7 +107,7 @@ export default function Page() { const getServers = async () => { const url = new URL('/api/v1.0/director_ui/servers', window.location.origin); - let response = await fetch(url); + let response = await fetchApi(async () => await fetch(url)); if (response.ok) { const responseData: Server[] = await response.json(); responseData.sort((a, b) => a.name.localeCompare(b.name)); diff --git a/web_ui/frontend/app/layout.tsx b/web_ui/frontend/app/layout.tsx index bc4da3fc9..10f5ca8b3 100644 --- a/web_ui/frontend/app/layout.tsx +++ b/web_ui/frontend/app/layout.tsx @@ -17,7 +17,8 @@ ***************************************************************/ import { LocalizationProvider } from '@/clientComponents'; -import { ThemeProviderClient } from '@/public/theme'; +import { ThemeProviderClient } from '@/components/ThemeProvider'; +import { AlertProvider } from '@/components/AlertProvider'; import './globals.css'; export const metadata = { @@ -34,7 +35,9 @@ export default function RootLayout({ - {children} + + {children} + diff --git a/web_ui/frontend/app/registry/cache/edit/page.tsx b/web_ui/frontend/app/registry/cache/edit/page.tsx index 66ce2d284..6c06ab12b 100644 --- a/web_ui/frontend/app/registry/cache/edit/page.tsx +++ b/web_ui/frontend/app/registry/cache/edit/page.tsx @@ -21,11 +21,12 @@ import { PutPage } from '@/app/registry/components/PutPage'; import { namespaceToCache, - putGeneralNamespace, } from '@/app/registry/components/util'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; +import { putGeneralNamespace } from '@/helpers/api'; + export default function Page() { const putCache = async (data: any) => { const cache = namespaceToCache(structuredClone(data)); diff --git a/web_ui/frontend/app/registry/cache/register/page.tsx b/web_ui/frontend/app/registry/cache/register/page.tsx index 6221ddb01..70386a7db 100644 --- a/web_ui/frontend/app/registry/cache/register/page.tsx +++ b/web_ui/frontend/app/registry/cache/register/page.tsx @@ -19,12 +19,13 @@ 'use client'; import { - namespaceToCache, - postGeneralNamespace, + namespaceToCache } from '@/app/registry/components/util'; import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; +import { postGeneralNamespace } from '@/helpers/api'; + export default function Page() { const postCache = async (data: any) => { diff --git a/web_ui/frontend/app/registry/components/Form.tsx b/web_ui/frontend/app/registry/components/Form.tsx index 37326f65c..688f92e79 100644 --- a/web_ui/frontend/app/registry/components/Form.tsx +++ b/web_ui/frontend/app/registry/components/Form.tsx @@ -1,5 +1,5 @@ import { Box, Button, Alert } from '@mui/material'; -import React, { useEffect, useState, Dispatch, SetStateAction } from 'react'; +import React, { useEffect, useState, Dispatch, SetStateAction, useContext } from 'react'; import useSWR from 'swr'; import { Namespace } from '@/index'; @@ -12,26 +12,15 @@ import { submitNamespaceForm, } from '@/app/registry/components/util'; import { CustomRegistrationFieldProps } from './CustomRegistrationField'; -import { getErrorMessage } from '@/helpers/util'; +import { alertOnError, getErrorMessage } from '@/helpers/util'; +import { optionsNamespaceRegistrationFields } from '@/helpers/api'; +import { AlertDispatchContext } from '@/components/AlertProvider'; interface FormProps { namespace?: Namespace; onSubmit: (data: Partial) => Promise; } -const getRegistrationFields = async (): Promise< - Omit[] -> => { - const response = await fetch('/api/v1.0/registry_ui/namespaces', { - method: 'OPTIONS', - }); - if (response.ok) { - return await response.json(); - } else { - throw new Error(await getErrorMessage(response)); - } -}; - const onChange = ( name: string, value: string | number | boolean | null, @@ -53,13 +42,20 @@ const onChange = ( }; const Form = ({ namespace, onSubmit }: FormProps) => { + + const dispatch = useContext(AlertDispatchContext); + const [data, setData] = useState | undefined>( namespace || {} ); - const { data: fields, error } = useSWR[]>( - 'getRegistrationFields', - getRegistrationFields, + const { data: fields, error } = useSWR[] | undefined>( + 'optionsNamespaceRegistrationFields', + async () => await alertOnError( + optionsNamespaceRegistrationFields, + "Couldn't fetch registration fields", + dispatch + ), { fallbackData: [] } ); diff --git a/web_ui/frontend/app/registry/components/PostPage.tsx b/web_ui/frontend/app/registry/components/PostPage.tsx index f5d6991ee..91ecf2478 100644 --- a/web_ui/frontend/app/registry/components/PostPage.tsx +++ b/web_ui/frontend/app/registry/components/PostPage.tsx @@ -19,44 +19,49 @@ 'use client'; import { Box, Grid, Collapse, Alert, Skeleton } from '@mui/material'; -import React, { useEffect, useState } from 'react'; - -import { Alert as AlertType, Namespace } from '@/index'; +import React, { useContext, useEffect, useState } from 'react'; import Form from '@/app/registry/components/Form'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { submitNamespaceForm } from '@/app/registry/components/util'; import { NamespaceFormPage } from '@/app/registry/components'; +import { alertOnError } from '@/helpers/util'; +import { AlertDispatchContext } from '@/components/AlertProvider'; const PostPage = ({ update }: NamespaceFormPage) => { + + const dispatch = useContext(AlertDispatchContext); + const [fromUrl, setFromUrl] = useState(undefined); - const [alert, setAlert] = useState(undefined); useEffect(() => { - const urlParams = new URLSearchParams(window.location.search); - const fromUrl = urlParams.get('fromUrl'); + (async () => { + const urlParams = new URLSearchParams(window.location.search); + const fromUrl = urlParams.get('fromUrl'); - try { if (fromUrl != undefined) { - const parsedUrl = new URL(fromUrl); - setFromUrl(parsedUrl); + const parsedUrl = await alertOnError( + () => new URL(fromUrl), + "Failed to parse URL", + dispatch + ); + if (parsedUrl) { + setFromUrl(parsedUrl); + } } - } catch (e) { - setAlert({ severity: 'error', message: 'Invalid fromUrl provided' }); - } + })() }, []); return ( - - - {alert?.message} - - { - setAlert(await submitNamespaceForm(data, fromUrl, update)); + onSubmit={async (namespace) => { + await alertOnError( + async () => await submitNamespaceForm(namespace, fromUrl, update), + "Failed to update namespace", + dispatch + ) }} /> diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index 813aa1acb..4b3a8c820 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -28,7 +28,7 @@ import { } from '@mui/material'; import React, { ReactNode, - Suspense, + Suspense, useContext, useEffect, useMemo, useState, @@ -38,16 +38,19 @@ import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { Namespace, Alert as AlertType } from '@/index'; import Form from '@/app/registry/components/Form'; import { - getNamespace, submitNamespaceForm, } from '@/app/registry/components/util'; +import { getNamespace } from '@/helpers/api'; import { NamespaceFormPage } from '@/app/registry/components' +import { AlertDispatchContext } from '@/components/AlertProvider'; +import { alertOnError } from '@/helpers/util'; const PutPage = ({ update }: NamespaceFormPage) => { const [id, setId] = useState(undefined); const [fromUrl, setFromUrl] = useState(undefined); const [namespace, setNamespace] = useState(undefined); - const [alert, setAlert] = useState(undefined); + + const dispatch = useContext(AlertDispatchContext); useEffect(() => { const urlParams = new URLSearchParams(window.location.search); @@ -55,7 +58,14 @@ const PutPage = ({ update }: NamespaceFormPage) => { const fromUrl = urlParams.get('fromUrl'); if (id === null) { - setAlert({ severity: 'error', message: 'No Namespace ID Provided' }); + dispatch({ + type: "openAlert", + payload: { + title: "No Namespace ID Provided", + message: "Your URL should contain a query parameter 'id' with the ID of the namespace you want to edit", + onClose: () => dispatch({ type: "closeAlert" }) + } + }) return; } @@ -65,23 +75,46 @@ const PutPage = ({ update }: NamespaceFormPage) => { setFromUrl(parsedUrl); } } catch (e) { - setAlert({ severity: 'error', message: 'Invalid fromUrl provided' }); + dispatch({ + type: "openAlert", + payload: { + title: "Invalid fromUrl provided", + message: "The `fromUrl` parameter provided is not a valid URL, this will only impact your redirection on completion of this form", + alertProps: { + severity: "warning" + }, + onClose: () => dispatch({ type: "closeAlert" }) + } + }) } try { setId(parseInt(id)); } catch (e) { - setAlert({ severity: 'error', message: 'Invalid Namespace ID Provided' }); + dispatch({ + type: "openAlert", + payload: { + title: "Invalid Namespace ID provided", + message: "The Namespace Id provided is not a valid number. Please report this issue, as well as what link directed you here.", + alertProps: { + severity: "error" + }, + onClose: () => dispatch({ type: "closeAlert" }) + } + }) } }, []); useEffect(() => { (async () => { if (id !== undefined) { - try { - setNamespace(await getNamespace(id)); - } catch (e) { - setAlert({ severity: 'error', message: e as string }); + const response = await alertOnError( + async () => await getNamespace(id), + "Couldn't get namespace", + dispatch + ) + if(response){ + setNamespace(response); } } })(); @@ -91,17 +124,16 @@ const PutPage = ({ update }: NamespaceFormPage) => { - - - {alert?.message} - - {namespace ? ( { let namespace = { ...data, id: id }; - setAlert(await submitNamespaceForm(namespace, fromUrl, update)); + await alertOnError( + async () => await submitNamespaceForm(namespace, fromUrl, update), + "Failed to update namespace", + dispatch + ) }} /> ) : ( diff --git a/web_ui/frontend/app/registry/components/index.ts b/web_ui/frontend/app/registry/components/index.ts index 86214d7a3..30e5853be 100644 --- a/web_ui/frontend/app/registry/components/index.ts +++ b/web_ui/frontend/app/registry/components/index.ts @@ -1,5 +1,5 @@ import { Alert as AlertType, Namespace } from '@/index'; export interface NamespaceFormPage { - update: (data: Partial) => Promise; + update: (data: Partial) => Promise; } diff --git a/web_ui/frontend/app/registry/components/util.tsx b/web_ui/frontend/app/registry/components/util.tsx index 9a55fd031..e6c595942 100644 --- a/web_ui/frontend/app/registry/components/util.tsx +++ b/web_ui/frontend/app/registry/components/util.tsx @@ -98,61 +98,15 @@ export const namespaceToOrigin = (data: Namespace) => { return data; }; -export const getNamespace = async ( - id: string | number -): Promise => { - const url = new URL( - `/api/v1.0/registry_ui/namespaces/${id}`, - window.location.origin - ); - const response = await fetch(url); - if (response.ok) { - return await response.json(); - } else { - throw new Error(await getErrorMessage(response)); - } -}; - -export const postGeneralNamespace = async ( - data: Namespace -): Promise => { - return await handleRequestAlert('/api/v1.0/registry_ui/namespaces', { - body: JSON.stringify(data), - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - }); -}; - -export const putGeneralNamespace = async ( - data: Namespace -): Promise => { - return await handleRequestAlert( - `/api/v1.0/registry_ui/namespaces/${data.id}`, - { - body: JSON.stringify(data), - method: 'PUT', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - } - ); -}; - export const submitNamespaceForm = async ( data: Partial, toUrl: URL | undefined, - handleSubmit: (data: Partial) => Promise + handleSubmit: (data: Partial) => Promise ) => { - const submitAlert = await handleSubmit(data); + const response = await handleSubmit(data); // Clear the form on successful submit - if (submitAlert == undefined) { + if (response != undefined) { window.location.href = toUrl ? toUrl.toString() : '/view/registry/'; } - - return submitAlert; }; diff --git a/web_ui/frontend/app/registry/denied/page.tsx b/web_ui/frontend/app/registry/denied/page.tsx index f1bee0e5e..00c792651 100644 --- a/web_ui/frontend/app/registry/denied/page.tsx +++ b/web_ui/frontend/app/registry/denied/page.tsx @@ -20,69 +20,46 @@ import { Box, - Button, Grid, Typography, - Paper, Alert, - Collapse, - IconButton, + Collapse } from '@mui/material'; -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useContext, useMemo } from 'react'; import { - PendingCard, - Card, - CardSkeleton, - CreateNamespaceCard, + CardSkeleton } from '@/components/Namespace'; -import Link from 'next/link'; -import { Namespace, Alert as AlertType } from '@/index'; import { getUser } from '@/helpers/login'; -import { Add } from '@mui/icons-material'; import NamespaceCardList from '@/components/Namespace/NamespaceCardList'; import useSWR from 'swr'; import { CardProps } from '@/components/Namespace/Card'; -import { PendingCardProps } from '@/components/Namespace/PendingCard'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import DeniedCard from '@/components/Namespace/DeniedCard'; +import { getExtendedNamespaces } from '@/helpers/api'; +import { AlertDispatchContext } from '@/components/AlertProvider'; +import { alertOnError } from '@/helpers/util'; -const getData = async () => { - let data: { namespace: Namespace }[] = []; - - const url = new URL( - '/api/v1.0/registry_ui/namespaces', - window.location.origin - ); - - const response = await fetch(url); - if (response.ok) { - const responseData: Namespace[] = await response.json(); - responseData.sort((a, b) => (a.id > b.id ? 1 : -1)); - responseData.forEach((namespace) => { - if (namespace.prefix.startsWith('/caches/')) { - namespace.type = 'cache'; - namespace.prefix = namespace.prefix.replace('/caches/', ''); - } else if (namespace.prefix.startsWith('/origins/')) { - namespace.type = 'origin'; - namespace.prefix = namespace.prefix.replace('/origins/', ''); - } else { - namespace.type = 'namespace'; - } - }); - - // Convert data to Partial CardProps - data = responseData.map((d) => { - return { namespace: d }; - }); - } +export default function Home() { - return data; -}; + const dispatch = useContext(AlertDispatchContext) -export default function Home() { - const { data } = useSWR('getNamespaces', getData); - const { data: user, error } = useSWR('getUser', getUser); + const { data } = useSWR( + 'getNamespaces', + async () => alertOnError( + getExtendedNamespaces, + "Couldn't fetch namespaces", + dispatch + ) + ) + const { data: user, error } = useSWR( + 'getUser', + async () => alertOnError( + getUser, + "Couldn't fetch user", + dispatch + ) + ); const deniedNamespaces = useMemo( () => @@ -97,11 +74,6 @@ export default function Home() { Namespace Registry - - - {error?.toString()} - - diff --git a/web_ui/frontend/app/registry/namespace/edit/page.tsx b/web_ui/frontend/app/registry/namespace/edit/page.tsx index 2abb51461..757897925 100644 --- a/web_ui/frontend/app/registry/namespace/edit/page.tsx +++ b/web_ui/frontend/app/registry/namespace/edit/page.tsx @@ -19,7 +19,7 @@ 'use client'; import { PutPage } from '@/app/registry/components/PutPage'; -import { putGeneralNamespace } from '@/app/registry/components/util'; +import { putGeneralNamespace } from '@/helpers/api'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; diff --git a/web_ui/frontend/app/registry/namespace/register/page.tsx b/web_ui/frontend/app/registry/namespace/register/page.tsx index 12d60291f..6d51df66a 100644 --- a/web_ui/frontend/app/registry/namespace/register/page.tsx +++ b/web_ui/frontend/app/registry/namespace/register/page.tsx @@ -18,7 +18,7 @@ 'use client'; -import { postGeneralNamespace } from '@/app/registry/components/util'; +import { postGeneralNamespace } from '@/helpers/api'; import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; diff --git a/web_ui/frontend/app/registry/origin/edit/page.tsx b/web_ui/frontend/app/registry/origin/edit/page.tsx index 683381232..25629eb54 100644 --- a/web_ui/frontend/app/registry/origin/edit/page.tsx +++ b/web_ui/frontend/app/registry/origin/edit/page.tsx @@ -20,11 +20,11 @@ import { PutPage } from '@/app/registry/components/PutPage'; import { - namespaceToOrigin, - putGeneralNamespace, + namespaceToOrigin } from '@/app/registry/components/util'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; +import { putGeneralNamespace } from '@/helpers/api'; export default function Page() { const putCache = async (data: any) => { diff --git a/web_ui/frontend/app/registry/origin/register/page.tsx b/web_ui/frontend/app/registry/origin/register/page.tsx index a89e3f83d..59e52d4a7 100644 --- a/web_ui/frontend/app/registry/origin/register/page.tsx +++ b/web_ui/frontend/app/registry/origin/register/page.tsx @@ -19,12 +19,12 @@ 'use client'; import { - namespaceToOrigin, - postGeneralNamespace, + namespaceToOrigin } from '@/app/registry/components/util'; import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; +import { postGeneralNamespace } from '@/helpers/api'; export default function Page() { const postCache = async (data: any) => { diff --git a/web_ui/frontend/app/registry/page.tsx b/web_ui/frontend/app/registry/page.tsx index caf27b17a..746a45d6c 100644 --- a/web_ui/frontend/app/registry/page.tsx +++ b/web_ui/frontend/app/registry/page.tsx @@ -28,7 +28,7 @@ import { Collapse, IconButton, } from '@mui/material'; -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useEffect, useMemo, useState, useContext } from 'react'; import { PendingCard, @@ -44,19 +44,30 @@ import { Add } from '@mui/icons-material'; import useSWR from 'swr'; import { CardProps } from '@/components/Namespace/Card'; import { PendingCardProps } from '@/components/Namespace/PendingCard'; +import { AlertDispatchContext } from '@/components/AlertProvider'; +import { alertOnError } from '@/helpers/util'; +import { getExtendedNamespaces } from '@/helpers/api'; export default function Home() { - const [alert, setAlert] = useState(undefined); - const { data, mutate: mutateNamespaces } = useSWR<{ namespace: Namespace }[]>( + const dispatch = useContext(AlertDispatchContext); + + const { data, mutate: mutateNamespaces } = useSWR<{ namespace: Namespace }[] | undefined>( 'getNamespaces', - getData, + () => alertOnError( + getExtendedNamespaces, + 'Failed to fetch namespaces', + dispatch + ), { fallbackData: [], } ); - const { data: user, error } = useSWR('getUser', getUser); + const { data: user, error } = useSWR( + 'getUser', + async () => await alertOnError(getUser, "Error Getting User", dispatch) + ); const pendingData = useMemo(() => { return data?.filter( @@ -97,33 +108,7 @@ export default function Home() { return ( - - Namespace Registry - - - {alert?.message} - - - - {user == undefined || - (!user.authenticated && ( - - - Login to register new namespaces. - - - - - - ))} {pendingData && pendingData.length > 0 && ( setAlert(a), onUpdate: () => mutateNamespaces(), }} /> @@ -263,36 +247,3 @@ export default function Home() { ); } - -const getData = async () => { - let data: { namespace: Namespace }[] = []; - - const url = new URL( - '/api/v1.0/registry_ui/namespaces', - window.location.origin - ); - - const response = await fetch(url); - if (response.ok) { - const responseData: Namespace[] = await response.json(); - responseData.sort((a, b) => (a.id > b.id ? 1 : -1)); - responseData.forEach((namespace) => { - if (namespace.prefix.startsWith('/caches/')) { - namespace.type = 'cache'; - namespace.prefix = namespace.prefix.replace('/caches/', ''); - } else if (namespace.prefix.startsWith('/origins/')) { - namespace.type = 'origin'; - namespace.prefix = namespace.prefix.replace('/origins/', ''); - } else { - namespace.type = 'namespace'; - } - }); - - // Convert data to Partial CardProps - data = responseData.map((d) => { - return { namespace: d }; - }); - } - - return data; -}; diff --git a/web_ui/frontend/app/test/page.tsx b/web_ui/frontend/app/test/page.tsx new file mode 100644 index 000000000..96176f517 --- /dev/null +++ b/web_ui/frontend/app/test/page.tsx @@ -0,0 +1,55 @@ +'use client'; + +import { AlertDispatchContext } from '@/components/AlertProvider'; +import React, { useContext } from 'react'; +import { Box } from '@mui/material'; +import CodeBlock from '@/components/CodeBlock'; + + +const Page = () => { + const dispatch = useContext(AlertDispatchContext); + + return ( +
+ + +
+ ); +} + +export default Page; diff --git a/web_ui/frontend/components/AlertPortal.tsx b/web_ui/frontend/components/AlertPortal.tsx index 7fc32a48c..4c482e3a4 100644 --- a/web_ui/frontend/components/AlertPortal.tsx +++ b/web_ui/frontend/components/AlertPortal.tsx @@ -1,35 +1,43 @@ import { Portal } from '@mui/base'; -import React from 'react'; -import { Alert, SnackbarProps, Snackbar } from '@mui/material'; - -import { Alert as AlertType } from '@/index'; +import React, { ReactNode } from 'react'; +import { Alert, AlertProps, Snackbar, SnackbarProps, AlertTitle } from '@mui/material'; export interface AlertPortalProps { - alert?: AlertType; onClose: () => void; + title?: string; + autoHideDuration?: number; + message?: ReactNode | string; + alertProps?: Omit; snackBarProps?: SnackbarProps; } export const AlertPortal = ({ - alert, onClose, + title, + autoHideDuration, + message, + alertProps, snackBarProps, }: AlertPortalProps) => { + + if (autoHideDuration) { + setTimeout(() => onClose(), autoHideDuration) + } + return ( - {alert?.message} + { title && {title} } + {message} diff --git a/web_ui/frontend/components/AlertProvider.tsx b/web_ui/frontend/components/AlertProvider.tsx new file mode 100644 index 000000000..91177efa9 --- /dev/null +++ b/web_ui/frontend/components/AlertProvider.tsx @@ -0,0 +1,72 @@ +'use client'; + +import { createContext, Dispatch, useReducer } from 'react'; +import { AlertPortal, AlertPortalProps } from '@/components/AlertPortal'; +import CodeBlock from '@/components/CodeBlock'; + +const defaultAlertContext: AlertPortalProps | undefined = undefined + +export const AlertContext = + createContext(defaultAlertContext); + +export const AlertDispatchContext = createContext>( + () => {} +); + +export const AlertProvider = ({ children }: { children: React.ReactNode }) => { + const [state, dispatch] = useReducer(alertReducer, defaultAlertContext); + + return ( + + + {children} + { state && } + + + ); +}; + +const alertReducer = ( + state: AlertPortalProps | undefined, + action: AlertReducerAction +): AlertPortalProps | undefined => { + switch (action.type) { + case 'closeAlert': + return undefined; + case 'openErrorAlert': + const {title, error, onClose} = action.payload; + + return { + title, + onClose, + message: {error}, + alertProps: { + severity: 'error' + } + } + case 'openAlert': + return action.payload; + default: + return state; + } +} + +export type AlertReducerAction = closeAlertAction | openErrorAlertAction | openAlertAction; + +type closeAlertAction = { + type: 'closeAlert' +} + +type openErrorAlertAction = { + type: 'openErrorAlert'; + payload: { + title: string; + error: string; + onClose: () => void; + } +} + +type openAlertAction = { + type: 'openAlert', + payload: AlertPortalProps +} diff --git a/web_ui/frontend/components/CodeBlock.tsx b/web_ui/frontend/components/CodeBlock.tsx new file mode 100644 index 000000000..900dc58e6 --- /dev/null +++ b/web_ui/frontend/components/CodeBlock.tsx @@ -0,0 +1,25 @@ +import { stackoverflowLight } from 'react-syntax-highlighter/dist/cjs/styles/hljs'; +import SyntaxHighlighter from 'react-syntax-highlighter'; +import { Box } from '@mui/material'; + +/** + * CodeBlock component + * Copy onClick and darken onHover + * @param children + * @constructor + */ +export const CodeBlock = ({children}: {children: string | string[]}) => { + return + { + navigator.clipboard.writeText(children.toString()); + }} + > + {children} + + +} + +export default CodeBlock; diff --git a/web_ui/frontend/components/Namespace/Card.tsx b/web_ui/frontend/components/Namespace/Card.tsx index c873ec115..b2e9d0a30 100644 --- a/web_ui/frontend/components/Namespace/Card.tsx +++ b/web_ui/frontend/components/Namespace/Card.tsx @@ -1,5 +1,5 @@ import { Alert, Alert as AlertType, Namespace } from '@/index'; -import React, { useRef, useState } from 'react'; +import React, { useContext, useRef, useState } from 'react'; import { Avatar, Box, @@ -14,9 +14,11 @@ import Link from 'next/link'; import InformationDropdown from './InformationDropdown'; import { NamespaceIcon } from '@/components/Namespace/index'; import { User } from '@/index'; -import AlertPortal from '@/components/AlertPortal'; -import { deleteNamespace } from './DeniedCard'; +import { deleteNamespace } from '@/helpers/api'; import { useSWRConfig } from 'swr'; +import { AlertDispatchContext } from '@/components/AlertProvider'; +import CodeBlock from '@/components/CodeBlock'; +import { alertOnError } from '@/helpers/util'; export interface CardProps { namespace: Namespace; @@ -25,9 +27,9 @@ export interface CardProps { } export const Card = ({ namespace, authenticated, onUpdate }: CardProps) => { + const dispatch = useContext(AlertDispatchContext); const ref = useRef(null); const [transition, setTransition] = useState(false); - const [alert, setAlert] = useState(undefined); const { mutate } = useSWRConfig(); return ( <> @@ -102,20 +104,14 @@ export const Card = ({ namespace, authenticated, onUpdate }: CardProps) => { color={'error'} onClick={async (e) => { e.stopPropagation(); - try { - await deleteNamespace(namespace.id); - setAlert({ - severity: 'success', - message: 'Registration deleted', - }); - setTimeout(() => mutate('getNamespaces'), 600); - if (onUpdate) { - onUpdate(); - } - } catch (e) { - if (e instanceof Error) { - setAlert({ severity: 'error', message: e.message }); - } + await alertOnError( + async () => await deleteNamespace(namespace.id), + 'Could Not Delete Registration', + dispatch + ) + setTimeout(() => mutate('getNamespaces'), 600); + if (onUpdate) { + onUpdate(); } }} > @@ -135,7 +131,6 @@ export const Card = ({ namespace, authenticated, onUpdate }: CardProps) => { /> - setAlert(undefined)} /> ); }; diff --git a/web_ui/frontend/components/Namespace/DeniedCard.tsx b/web_ui/frontend/components/Namespace/DeniedCard.tsx index 348cb7a45..2db34baaa 100644 --- a/web_ui/frontend/components/Namespace/DeniedCard.tsx +++ b/web_ui/frontend/components/Namespace/DeniedCard.tsx @@ -1,14 +1,17 @@ -import React, { useMemo, useRef, useState } from 'react'; +import React, { useContext, useMemo, useRef, useState } from 'react'; import { green, red } from '@mui/material/colors'; import { Authenticated, secureFetch } from '@/helpers/login'; import { Avatar, Box, IconButton, Tooltip, Typography } from '@mui/material'; import { Block, Check, Delete, Edit, Person } from '@mui/icons-material'; -import { Alert as AlertType, Alert, Namespace } from '@/index'; +import { Alert, Namespace } from '@/index'; import InformationDropdown from './InformationDropdown'; import { getServerType, NamespaceIcon } from '@/components/Namespace/index'; +import { AlertContext, AlertDispatchContext } from '@/components/AlertProvider'; import { User } from '@/index'; -import AlertPortal from '@/components/AlertPortal'; import { useSWRConfig } from 'swr'; +import CodeBlock from '@/components/CodeBlock'; +import { approveNamespace, deleteNamespace } from '@/helpers/api'; +import { alertOnError } from '@/helpers/util'; export interface DeniedCardProps { namespace: Namespace; @@ -17,60 +20,11 @@ export interface DeniedCardProps { authenticated?: User; } -export const deleteNamespace = async (id: number) => { - const response = await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}`, { - method: 'DELETE', - }); - - if (!response.ok) { - let alertMessage; - try { - let data = await response.json(); - if (data?.msg) { - alertMessage = data?.msg; - } - alertMessage = 'Details not provided'; - } catch (e) { - if (e instanceof Error) { - alertMessage = e.message; - } - } - - throw new Error('Failed to delete namespace: ' + alertMessage); - } -}; - -const approveNamespace = async (id: number) => { - const response = await secureFetch( - `/api/v1.0/registry_ui/namespaces/${id}/approve`, - { - method: 'PATCH', - } - ); - - if (!response.ok) { - let alertMessage; - try { - let data = await response.json(); - if (data?.msg) { - alertMessage = data?.msg; - } - alertMessage = 'Details not provided'; - } catch (e) { - if (e instanceof Error) { - alertMessage = e.message; - } - } - - throw new Error('Failed to approve registration: ' + alertMessage); - } -}; - export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { const ref = useRef(null); const [transition, setTransition] = useState(false); - const [alert, setAlert] = useState(undefined); - + const dispatch = useContext(AlertDispatchContext); + const alert = useContext(AlertContext) const { mutate } = useSWRConfig(); return ( @@ -86,9 +40,9 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { borderRadius: transition ? '10px 10px 0px 0px' : 2, transition: 'background-color .3s ease-out', bgcolor: - alert?.severity == 'success' + alert?.alertProps?.severity == 'success' ? green[100] - : alert?.severity == 'error' + : alert?.alertProps?.severity == 'error' ? red[100] : 'inherit', '&:hover': { @@ -130,18 +84,11 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { color={'error'} onClick={async (e) => { e.stopPropagation(); - try { - await deleteNamespace(namespace.id); - setAlert({ - severity: 'success', - message: 'Registration deleted', - }); - setTimeout(() => mutate('getNamespaces'), 600); - } catch (e) { - if (e instanceof Error) { - setAlert({ severity: 'error', message: e.message }); - } - } + await alertOnError( + () => deleteNamespace(namespace.id), + 'Could Not Delete Registration', + dispatch + ) }} > @@ -153,18 +100,12 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { color={'success'} onClick={async (e) => { e.stopPropagation(); - try { - await approveNamespace(namespace.id); - setAlert({ - severity: 'success', - message: 'Registration Approved', - }); - setTimeout(() => mutate('getNamespaces'), 600); - } catch (e) { - if (e instanceof Error) { - setAlert({ severity: 'error', message: e.message }); - } - } + await alertOnError( + () => approveNamespace(namespace.id), + 'Could Not Approve Registration', + dispatch + ) + setTimeout(() => mutate('getNamespaces'), 600); }} > @@ -183,13 +124,6 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { /> - {alert?.severity == 'error' && ( - setAlert(undefined)} - /> - )} ); }; diff --git a/web_ui/frontend/components/Namespace/PendingCard.tsx b/web_ui/frontend/components/Namespace/PendingCard.tsx index 4257ef09a..f138ab3a4 100644 --- a/web_ui/frontend/components/Namespace/PendingCard.tsx +++ b/web_ui/frontend/components/Namespace/PendingCard.tsx @@ -1,4 +1,4 @@ -import React, { useMemo, useRef, useState } from 'react'; +import React, { useContext, useMemo, useRef, useState } from 'react'; import { Authenticated, secureFetch } from '@/helpers/login'; import { Avatar, Box, IconButton, Tooltip, Typography } from '@mui/material'; import { Block, Check, Edit, Person } from '@mui/icons-material'; @@ -8,6 +8,9 @@ import { Alert, Namespace } from '@/index'; import InformationDropdown from './InformationDropdown'; import { getServerType, NamespaceIcon } from '@/components/Namespace/index'; import { User } from '@/index'; +import { alertOnError } from '@/helpers/util'; +import { AlertDispatchContext } from '@/components/AlertProvider'; +import { approveNamespace, denyNamespace } from '@/helpers/api'; export interface PendingCardProps { namespace: Namespace; @@ -25,61 +28,7 @@ export const PendingCard = ({ const ref = useRef(null); const [transition, setTransition] = useState(false); - const approveNamespace = async (e: React.MouseEvent) => { - e.stopPropagation(); - - try { - const response = await secureFetch( - `/api/v1.0/registry_ui/namespaces/${namespace.id}/approve`, - { - method: 'PATCH', - } - ); - - if (!response.ok) { - onAlert({ - severity: 'error', - message: `Failed to approve ${namespace.type} registration: ${namespace.prefix}`, - }); - } else { - onUpdate(); - onAlert({ - severity: 'success', - message: `Successfully approved ${namespace.type} registration: ${namespace.prefix}`, - }); - } - } catch (error) { - console.error(error); - } - }; - - const denyNamespace = async (e: React.MouseEvent) => { - e.stopPropagation(); - - try { - const response = await secureFetch( - `/api/v1.0/registry_ui/namespaces/${namespace.id}/deny`, - { - method: 'PATCH', - } - ); - - if (!response.ok) { - onAlert({ - severity: 'error', - message: `Failed to deny ${namespace.type} registration: ${namespace.prefix}`, - }); - } else { - onUpdate(); - onAlert({ - severity: 'success', - message: `Successfully denied ${namespace.type} registration: ${namespace.prefix}`, - }); - } - } catch (error) { - console.error(error); - } - }; + const dispatch = useContext(AlertDispatchContext) return ( @@ -123,7 +72,15 @@ export const PendingCard = ({ denyNamespace(e)} + onClick={async (e) => { + e.stopPropagation() + await alertOnError( + () => denyNamespace(namespace.id), + "Couldn't deny namespace", + dispatch + ) + onUpdate() + }} > @@ -132,7 +89,15 @@ export const PendingCard = ({ approveNamespace(e)} + onClick={async (e) => { + e.stopPropagation() + await alertOnError( + () => approveNamespace(namespace.id), + "Couldn't approve namespace", + dispatch + ) + onUpdate() + }} > diff --git a/web_ui/frontend/public/theme.tsx b/web_ui/frontend/components/ThemeProvider.tsx similarity index 92% rename from web_ui/frontend/public/theme.tsx rename to web_ui/frontend/components/ThemeProvider.tsx index 9d4e8cfcd..85aba2a81 100644 --- a/web_ui/frontend/public/theme.tsx +++ b/web_ui/frontend/components/ThemeProvider.tsx @@ -30,7 +30,7 @@ const poppins = Poppins({ display: 'swap', }); -let theme = createTheme({ +let themeProvider = createTheme({ palette: { primary: { main: '#0885ff', @@ -78,7 +78,7 @@ let theme = createTheme({ }, }); -theme = responsiveFontSizes(theme, { factor: 3 }); +themeProvider = responsiveFontSizes(themeProvider, { factor: 3 }); interface ThemeProviderClientProps { children: React.ReactNode; @@ -87,5 +87,5 @@ interface ThemeProviderClientProps { export const ThemeProviderClient: FC = ({ children, }) => { - return {children}; + return {children}; }; diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts new file mode 100644 index 000000000..c51a6b0b5 --- /dev/null +++ b/web_ui/frontend/helpers/api.ts @@ -0,0 +1,267 @@ +import { secureFetch } from '@/helpers/login'; +import { getErrorMessage } from '@/helpers/util'; +import { Alert, Namespace } from '@/index'; +import { CustomRegistrationFieldProps } from '@/app/registry/components/CustomRegistrationField'; + +// TODO: Decide if we should standardize the output in all of these functions. Should they all be responses? + +/** + * Wraps an api request with error handling for both the request and the response if error + * @param fetchRequest The request to make to the api + * @returns The response from the api + */ +export async function fetchApi( + fetchRequest: () => Promise +): Promise { + try { + const response = await fetchRequest(); + if (!response.ok) { + let alertMessage; + try { + alertMessage = await getErrorMessage(response); + } catch (e) { + if (e instanceof Error) { + alertMessage = e.message; + } + } + throw new Error(alertMessage); + } + return response + } catch (e) { + if (e instanceof Error) { + throw Error("Fetch to API Failed", { cause: e }) + } else { + throw Error("Fetch to API Failed", { cause: e}) + } + } +} + + +/** + * Deletes a namespace + * @param id Namespace ID + */ +export const deleteNamespace = async (id: number) => { + return fetchApi( + async () => await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}`, { + method: 'DELETE', + }) + ) +}; + + +/** + * Approves a namespace + * @param id Namespace ID + */ +export const approveNamespace = async (id: number) => { + return fetchApi( + async () => await secureFetch( + `/api/v1.0/registry_ui/namespaces/${id}/approve`, + { + method: 'PATCH', + } + ) + ) +} + +/** + * Denies a namespace + * @param id Namespace ID + */ +export const denyNamespace = async (id: number) => { + return fetchApi( + async () => await secureFetch( + `/api/v1.0/registry_ui/namespaces/${id}/deny`, + { + method: 'PATCH', + } + ) + ) +} + +/** + * Enables a server on the director + * @param name Server name + */ +export const allowServer = async (name: string) => { + return fetchApi( + async () => await secureFetch( + `/api/v1.0/director_ui/servers/allow/${name}`, + { + method: 'PATCH', + } + ) + ) +} + +/** + * Filters ( Disables ) a server on the director + * @param name Server name + */ +export const filterServer = async (name: string) => { + return fetchApi( + async () => await secureFetch( + `/api/v1.0/director_ui/servers/filter/${name}`, + { + method: 'PATCH', + } + ) + ) +} + +/** + * Get extended namespaces + */ +export const getExtendedNamespaces = async (): Promise<{ namespace: Namespace }[]> => { + const response = await getNamespaces() + const data: Namespace[] = await response.json(); + data.sort((a, b) => (a.id > b.id ? 1 : -1)); + data.forEach((namespace) => { + if (namespace.prefix.startsWith('/caches/')) { + namespace.type = 'cache'; + namespace.prefix = namespace.prefix.replace('/caches/', ''); + } else if (namespace.prefix.startsWith('/origins/')) { + namespace.type = 'origin'; + namespace.prefix = namespace.prefix.replace('/origins/', ''); + } else { + namespace.type = 'namespace'; + } + }); + + // TODO: This extra should be done somewhere else, why is it done? + return data.map((d) => { + return { namespace: d }; + }); +} + +/** + * Get namespaces + */ +export const getNamespaces = async (): Promise => { + const url = new URL( + '/api/v1.0/registry_ui/namespaces', + window.location.origin + ); + + const response = await fetchApi(async () => await fetch(url)); + return await response.json(); +} + +/** + * Gets a namespace by ID + * @param id Namespace ID + */ +export const getNamespace = async ( + id: string | number +): Promise => { + const url = new URL( + `/api/v1.0/registry_ui/namespaces/${id}`, + window.location.origin + ); + const response = await fetchApi(async () => await fetch(url)); + return await response.json(); +}; + +export const postGeneralNamespace = async ( + data: Namespace +): Promise => { + return await fetchApi( + async () => + await fetch('/api/v1.0/registry_ui/namespaces', { + body: JSON.stringify(data), + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + }) + ) +}; + +export const putGeneralNamespace = async ( + data: Namespace +): Promise => { + return await fetchApi( + async () => { + return fetch(`/api/v1.0/registry_ui/namespaces/${data.id}`, + { + body: JSON.stringify(data), + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + } + ) + } + ) +}; + +/** + * Get registration fields from options for namespace + * // TODO: Complain about the misuse of options + */ +export const optionsNamespaceRegistrationFields = async (): Promise< + Omit[] +> => { + const response = await fetchApi(async () => await fetch('/api/v1.0/registry_ui/namespaces', { + method: 'OPTIONS', + })) + return response.json() +}; + +/** + * Initializes a login via terminal code + */ +export const initLogin = async (code: string): Promise => { + return await fetchApi( + async () => + await fetch('/api/v1.0/auth/initLogin', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + code: code, + }), + }) + ) +} + +/** + * Reset ( Do initial ) Login + */ +export const resetLogin = async (password: string): Promise => { + return await fetchApi( + async () => + await fetch('/api/v1.0/auth/resetLogin', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + password: password, + }), + }) + ) +} + +/** + * Login + */ +export const login = async (password: string, user: string = "admin"): Promise => { + return await fetchApi( + async () => + await fetch('/api/v1.0/auth/login', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + user: user, + password: password, + }), + }) + ) +} diff --git a/web_ui/frontend/helpers/login.tsx b/web_ui/frontend/helpers/login.ts similarity index 100% rename from web_ui/frontend/helpers/login.tsx rename to web_ui/frontend/helpers/login.ts diff --git a/web_ui/frontend/helpers/util.tsx b/web_ui/frontend/helpers/util.ts similarity index 50% rename from web_ui/frontend/helpers/util.tsx rename to web_ui/frontend/helpers/util.ts index e1c1e6be1..42bb56cba 100644 --- a/web_ui/frontend/helpers/util.tsx +++ b/web_ui/frontend/helpers/util.ts @@ -1,4 +1,6 @@ import { ServerType } from '@/index'; +import { Dispatch } from 'react'; +import { AlertReducerAction } from '@/components/AlertProvider'; const stringToTime = (time: string) => { return new Date(Date.parse(time)).toLocaleString(); @@ -36,6 +38,11 @@ export const getOauthEnabledServers = async () => { } }; +/** + * Extract the value from a object via a list of keys + * @param obj + * @param keys + */ export function getObjectValue(obj: any, keys: string[]): T | undefined { const currentValue = obj?.[keys[0]]; if (keys.length == 1) { @@ -44,21 +51,28 @@ export function getObjectValue(obj: any, keys: string[]): T | undefined { return getObjectValue(currentValue, keys.slice(1)); } +/** + * Get the error message from a response + * @param response + */ export const getErrorMessage = async (response: Response): Promise => { - let message; try { let data = await response.json(); - message = response.status + ': ' + data['msg']; + return response.status + ': ' + data['msg']; } catch (e) { - message = response.status + ': ' + response.statusText; + return response.status + ': ' + response.statusText; } - return message; }; export type TypeFunction = (x?: F) => T; export type TypeOrTypeFunction = T | TypeFunction; +/** + * Evaluate a function or return a value + * @param o Function or value + * @param functionProps Function properties + */ export function evaluateOrReturn( o: TypeOrTypeFunction, functionProps?: F @@ -70,6 +84,58 @@ export function evaluateOrReturn( return o as T; } + +/** + * Get the average of an array of numbers + * @param arr Array of numbers + */ export const average = (arr: number[]) => { return arr.reduce((a, b) => a + b, 0) / arr.length; }; + +type ErrorWithCause = Error & { cause?: Error }; + + +/** + * If an error is caught from f then display the error via an alert UI + */ +export async function alertOnError( + f: () => Promise | T | undefined, + title: string = 'Error', + dispatch: Dispatch +){ + try { + return await f(); + } catch (error) { + console.error(error); + if(error instanceof Error) { + dispatch({ + type: "openErrorAlert", + payload: { + title, + error: errorToString(error as ErrorWithCause), + onClose: () => dispatch({ type: "closeAlert" }) + } + }) + } + } +} + +/** + * Convert a error into a string + * @param error + */ +export const errorToString = (error: ErrorWithCause) : string => { + + if(error?.cause){ + + // Check that error is instance of Error + if(!(error?.cause instanceof Error)) { + console.error("Malformed error, cause is not an instance of Error", error) + } + + return `${error.message}\n↳ ${errorToString(error.cause as ErrorWithCause)}` + } + + return `${error.message}` +} From 85066067c290ee59789dd92d2c85718b0360ca0a Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Mon, 28 Oct 2024 08:44:16 -0500 Subject: [PATCH 04/86] Uniform Error Handling --- web_ui/frontend/app/(login)/login/page.tsx | 47 +++++++++++++--------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/web_ui/frontend/app/(login)/login/page.tsx b/web_ui/frontend/app/(login)/login/page.tsx index c0a1a412a..4642364b0 100644 --- a/web_ui/frontend/app/(login)/login/page.tsx +++ b/web_ui/frontend/app/(login)/login/page.tsx @@ -52,16 +52,23 @@ const AdminLogin = () => { let [password, setPassword] = useState(''); let [loading, setLoading] = useState(false); - let [error, setError] = useState(undefined); const [toggled, setToggled] = useState(false); - const { data: enabledServers } = useSWR( + const { data: enabledServers } = useSWR( 'getEnabledServers', - getEnabledServers + async () => await alertOnError( + getEnabledServers, + "Could not get enabled servers", + dispatch + ) ); - const { data: oauthServers } = useSWR( + const { data: oauthServers } = useSWR( 'getOauthEnabledServers', - getOauthEnabledServers, + async () => await alertOnError( + getOauthEnabledServers, + "Could not get oauth enabled servers", + dispatch + ), { fallbackData: [] } ); @@ -108,23 +115,12 @@ const AdminLogin = () => { sx: { width: '50%' }, onChange: (e) => { setPassword(e.target.value); - setError(undefined); }, }, }} /> - - - {error} - - { }; export default function Home() { + + const dispatch = useContext(AlertDispatchContext); + const [returnUrl, setReturnUrl] = useState(undefined); - const { data: enabledServers } = useSWR( + const { data: enabledServers } = useSWR( 'getEnabledServers', - getEnabledServers + async () => await alertOnError( + getEnabledServers, + "Could not get enabled servers", + dispatch + ) ); - const { data: oauthServers } = useSWR( + const { data: oauthServers } = useSWR( 'getOauthEnabledServers', - getOauthEnabledServers, + async () => await alertOnError( + getOauthEnabledServers, + "Could not determine if the active server had OAuth enabled", + dispatch + ), { fallbackData: [] } ); From 56c0958a24e6dc790e11557373ab5460ad371d53 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Mon, 28 Oct 2024 09:14:03 -0500 Subject: [PATCH 05/86] Uniform Error Handling --- web_ui/frontend/app/config/Config.tsx | 33 +++++------ .../components/FederationOverview.tsx | 56 +++++++++---------- web_ui/frontend/helpers/api.ts | 10 ++++ web_ui/frontend/helpers/get.ts | 3 +- 4 files changed, 56 insertions(+), 46 deletions(-) diff --git a/web_ui/frontend/app/config/Config.tsx b/web_ui/frontend/app/config/Config.tsx index 45039f54f..97a337b25 100644 --- a/web_ui/frontend/app/config/Config.tsx +++ b/web_ui/frontend/app/config/Config.tsx @@ -27,7 +27,7 @@ import { IconButton, Alert, } from '@mui/material'; -import React, { memo, useCallback, useEffect, useMemo, useState } from 'react'; +import React, { memo, useCallback, useContext, useEffect, useMemo, useState } from 'react'; import { AppRegistration, AssistantDirection, @@ -51,21 +51,30 @@ import StatusSnackBar, { StatusSnackBarProps, } from '@/components/StatusSnackBar'; import { ServerType } from '@/index'; -import { getEnabledServers } from '@/helpers/util'; +import { alertOnError, getEnabledServers } from '@/helpers/util'; import DownloadButton from '@/components/DownloadButton'; import { PaddedContent } from '@/components/layout'; import { ConfigDisplay, TableOfContents } from '@/app/config/components'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; +import { getConfig } from '@/helpers/api'; +import { AlertDispatchContext } from '@/components/AlertProvider'; function Config({ metadata }: { metadata: ParameterMetadataRecord }) { + + const dispatch = useContext(AlertDispatchContext); + const [status, setStatus] = useState( undefined ); const [patch, _setPatch] = useState({}); - const { data, mutate, error } = useSWR( + const { data, mutate, error } = useSWR( 'getConfig', - getConfig + async () => await alertOnError( + getConfigJson, + "Could not get config", + dispatch + ) ); const { data: enabledServers } = useSWR( 'getEnabledServers', @@ -94,8 +103,6 @@ function Config({ metadata }: { metadata: ParameterMetadataRecord }) { ); }, [serverConfig, patch]); - console.error(error, data); - return ( <> @@ -215,17 +222,11 @@ function Config({ metadata }: { metadata: ParameterMetadataRecord }) { ); } -const getConfig = async (): Promise => { - let response = await fetch('/api/v1.0/config'); - - if (!response.ok) { - if (response.status == 401) { - throw new Error('You must be logged in to view and access the config'); - } - throw new Error('Failed to fetch config'); +const getConfigJson = async (): Promise => { + const response = await getConfig() + if(response){ + return await response.json(); } - - return await response.json(); }; export default Config; diff --git a/web_ui/frontend/components/FederationOverview.tsx b/web_ui/frontend/components/FederationOverview.tsx index 40895926d..a47f1d1a5 100644 --- a/web_ui/frontend/components/FederationOverview.tsx +++ b/web_ui/frontend/components/FederationOverview.tsx @@ -7,6 +7,7 @@ import { Box, Typography } from '@mui/material'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import Link from 'next/link'; import { getErrorMessage, getObjectValue } from '@/helpers/util'; +import { getConfig } from '@/helpers/api'; const LinkBox = ({ href, text }: { href: string; text: string }) => { return ( @@ -30,15 +31,15 @@ const LinkBox = ({ href, text }: { href: string; text: string }) => { }; const UrlData = [ - { key: ['Federation', 'NamespaceUrl', 'Value'], text: 'Namespace Registry' }, - { key: ['Federation', 'DirectorUrl', 'Value'], text: 'Director' }, - { key: ['Federation', 'RegistryUrl', 'Value'], text: 'Registry' }, + { key: ['Federation', 'NamespaceUrl'], text: 'Namespace Registry' }, + { key: ['Federation', 'DirectorUrl'], text: 'Director' }, + { key: ['Federation', 'RegistryUrl'], text: 'Registry' }, { - key: ['Federation', 'TopologyNamespaceUrl', 'Value'], + key: ['Federation', 'TopologyNamespaceUrl'], text: 'Topology Namespace', }, - { key: ['Federation', 'DiscoveryUrl', 'Value'], text: 'Discovery' }, - { key: ['Federation', 'JwkUrl', 'Value'], text: 'JWK' }, + { key: ['Federation', 'DiscoveryUrl'], text: 'Discovery' }, + { key: ['Federation', 'JwkUrl'], text: 'JWK' }, ]; const FederationOverview = () => { @@ -46,35 +47,32 @@ const FederationOverview = () => { { text: string; url: string | undefined }[] >([]); - let getConfig = async () => { - let response = await fetch('/api/v1.0/config'); - if (response.ok) { - const responseData = (await response.json()) as Config; + let getConfigJson = async () => { + const response = await getConfig() + const responseData = (await response.json()) as Config; - const federationUrls = UrlData.map(({ key, text }) => { - let url = getObjectValue(responseData, key); - if ( - url && - !url?.startsWith('http://') && - !url?.startsWith('https://') - ) { - url = 'https://' + url; - } + const federationUrls = UrlData.map(({ key, text }) => { + let url = getObjectValue(responseData, key); + if ( + url && + !url?.startsWith('http://') && + !url?.startsWith('https://') + ) { + url = 'https://' + url; + } + + return { + text, + url, + }; + }); - return { - text, - url, - }; - }); + setConfig(federationUrls); - setConfig(federationUrls); - } else { - console.error(await getErrorMessage(response)); - } }; useEffect(() => { - getConfig(); + getConfigJson(); }, []); if (config === undefined) { diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts index c51a6b0b5..dbbfe8b9e 100644 --- a/web_ui/frontend/helpers/api.ts +++ b/web_ui/frontend/helpers/api.ts @@ -37,6 +37,16 @@ export async function fetchApi( } +/** + * Get config + */ +export const getConfig = async (): Promise => { + return fetchApi( + async () => await secureFetch('/api/v1.0/config') + ) +} + + /** * Deletes a namespace * @param id Namespace ID diff --git a/web_ui/frontend/helpers/get.ts b/web_ui/frontend/helpers/get.ts index ca2be9954..e9251c31a 100644 --- a/web_ui/frontend/helpers/get.ts +++ b/web_ui/frontend/helpers/get.ts @@ -1,8 +1,9 @@ import { Config, ParameterValueRecord } from '@/components/configuration'; +import { getConfig as getConfigResponse } from '@/helpers/api' import { flattenObject } from '@/app/config/util'; export const getConfig = async (): Promise => { - let response = await fetch('/api/v1.0/config'); + let response = await getConfigResponse(); let data = await response.json(); let flatData = flattenObject(data); return flatData; From 81b6d785d270cf482c83f1ccdd13a8a4d9c1cbb4 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 29 Oct 2024 08:55:59 -0500 Subject: [PATCH 06/86] Prettier Format --- .../app/(login)/initialization/code/page.tsx | 7 +- .../(login)/initialization/password/page.tsx | 16 +- web_ui/frontend/app/(login)/login/page.tsx | 50 +++--- web_ui/frontend/app/config/Config.tsx | 21 ++- .../app/director/components/DirectorCard.tsx | 2 +- .../director/components/DirectorDropdown.tsx | 5 +- web_ui/frontend/app/director/page.tsx | 13 +- .../frontend/app/registry/cache/edit/page.tsx | 4 +- .../app/registry/cache/register/page.tsx | 5 +- .../CustomRegistrationField/PubkeyField.tsx | 4 +- .../CustomRegistrationField/index.tsx | 1 - .../frontend/app/registry/components/Form.tsx | 24 ++- .../app/registry/components/PostPage.tsx | 12 +- .../app/registry/components/PutPage.tsx | 63 +++---- .../frontend/app/registry/components/util.tsx | 5 +- web_ui/frontend/app/registry/denied/page.tsx | 35 +--- .../app/registry/origin/edit/page.tsx | 4 +- .../app/registry/origin/register/page.tsx | 4 +- web_ui/frontend/app/registry/page.tsx | 18 +- web_ui/frontend/app/test/page.tsx | 10 +- web_ui/frontend/components/AlertPortal.tsx | 13 +- web_ui/frontend/components/AlertProvider.tsx | 38 +++-- web_ui/frontend/components/CodeBlock.tsx | 28 ++-- .../components/FederationOverview.tsx | 9 +- web_ui/frontend/components/Namespace/Card.tsx | 2 +- .../components/Namespace/DeniedCard.tsx | 6 +- .../Namespace/InformationDropdown.tsx | 6 +- .../components/Namespace/PendingCard.tsx | 14 +- .../frontend/components/Namespace/index.tsx | 1 - web_ui/frontend/helpers/api.ts | 154 +++++++++--------- web_ui/frontend/helpers/get.ts | 4 +- web_ui/frontend/helpers/util.ts | 35 ++-- 32 files changed, 301 insertions(+), 312 deletions(-) diff --git a/web_ui/frontend/app/(login)/initialization/code/page.tsx b/web_ui/frontend/app/(login)/initialization/code/page.tsx index 18791c9b9..904446ada 100644 --- a/web_ui/frontend/app/(login)/initialization/code/page.tsx +++ b/web_ui/frontend/app/(login)/initialization/code/page.tsx @@ -28,7 +28,6 @@ import { initLogin } from '@/helpers/api'; import { alertOnError } from '@/helpers/util'; import { AlertDispatchContext } from '@/components/AlertProvider'; - export default function Home() { const router = useRouter(); let [code, _setCode] = useState([ @@ -41,7 +40,7 @@ export default function Home() { ]); let [loading, setLoading] = useState(false); - const dispatch = useContext(AlertDispatchContext) + const dispatch = useContext(AlertDispatchContext); const setCode = (code: Code) => { _setCode(code); @@ -58,8 +57,8 @@ export default function Home() { async () => await initLogin(code), 'Could not login', dispatch - ) - if(response) { + ); + if (response) { router.push('../password/'); } else { setLoading(false); diff --git a/web_ui/frontend/app/(login)/initialization/password/page.tsx b/web_ui/frontend/app/(login)/initialization/password/page.tsx index ce4fe85b9..a35e18c3b 100644 --- a/web_ui/frontend/app/(login)/initialization/password/page.tsx +++ b/web_ui/frontend/app/(login)/initialization/password/page.tsx @@ -30,15 +30,13 @@ import { AlertDispatchContext } from '@/components/AlertProvider'; import { initLogin, resetLogin } from '@/helpers/api'; export default function Home() { - - const dispatch = useContext(AlertDispatchContext) + const dispatch = useContext(AlertDispatchContext); const router = useRouter(); let [password, _setPassword] = useState(''); let [confirmPassword, _setConfirmPassword] = useState(''); let [loading, setLoading] = useState(false); - async function submit(password: string) { setLoading(true); @@ -46,8 +44,8 @@ export default function Home() { async () => await resetLogin(password), 'Could not login', dispatch - ) - if(response) { + ); + if (response) { router.push('../password/'); } else { setLoading(false); @@ -64,12 +62,12 @@ export default function Home() { type: 'openAlert', payload: { alertProps: { - severity: 'warning' + severity: 'warning', }, message: 'Passwords do not match', - onClose: () => dispatch({ type: 'closeAlert' }) - } - }) + onClose: () => dispatch({ type: 'closeAlert' }), + }, + }); } } diff --git a/web_ui/frontend/app/(login)/login/page.tsx b/web_ui/frontend/app/(login)/login/page.tsx index 4642364b0..a597ca523 100644 --- a/web_ui/frontend/app/(login)/login/page.tsx +++ b/web_ui/frontend/app/(login)/login/page.tsx @@ -44,7 +44,6 @@ import { login } from '@/helpers/api'; import { AlertDispatchContext } from '@/components/AlertProvider'; const AdminLogin = () => { - const dispatch = useContext(AlertDispatchContext); const router = useRouter(); @@ -56,19 +55,21 @@ const AdminLogin = () => { const { data: enabledServers } = useSWR( 'getEnabledServers', - async () => await alertOnError( - getEnabledServers, - "Could not get enabled servers", - dispatch - ) + async () => + await alertOnError( + getEnabledServers, + 'Could not get enabled servers', + dispatch + ) ); const { data: oauthServers } = useSWR( 'getOauthEnabledServers', - async () => await alertOnError( - getOauthEnabledServers, - "Could not get oauth enabled servers", - dispatch - ), + async () => + await alertOnError( + getOauthEnabledServers, + 'Could not get oauth enabled servers', + dispatch + ), { fallbackData: [] } ); @@ -83,9 +84,9 @@ const AdminLogin = () => { const response = await alertOnError( async () => await login(password), - "Could not login", + 'Could not login', dispatch - ) + ); if (response) { await mutate(getUser); @@ -160,25 +161,26 @@ const AdminLogin = () => { }; export default function Home() { - const dispatch = useContext(AlertDispatchContext); const [returnUrl, setReturnUrl] = useState(undefined); const { data: enabledServers } = useSWR( 'getEnabledServers', - async () => await alertOnError( - getEnabledServers, - "Could not get enabled servers", - dispatch - ) + async () => + await alertOnError( + getEnabledServers, + 'Could not get enabled servers', + dispatch + ) ); const { data: oauthServers } = useSWR( 'getOauthEnabledServers', - async () => await alertOnError( - getOauthEnabledServers, - "Could not determine if the active server had OAuth enabled", - dispatch - ), + async () => + await alertOnError( + getOauthEnabledServers, + 'Could not determine if the active server had OAuth enabled', + dispatch + ), { fallbackData: [] } ); diff --git a/web_ui/frontend/app/config/Config.tsx b/web_ui/frontend/app/config/Config.tsx index 97a337b25..bfd42fbde 100644 --- a/web_ui/frontend/app/config/Config.tsx +++ b/web_ui/frontend/app/config/Config.tsx @@ -27,7 +27,14 @@ import { IconButton, Alert, } from '@mui/material'; -import React, { memo, useCallback, useContext, useEffect, useMemo, useState } from 'react'; +import React, { + memo, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from 'react'; import { AppRegistration, AssistantDirection, @@ -60,7 +67,6 @@ import { getConfig } from '@/helpers/api'; import { AlertDispatchContext } from '@/components/AlertProvider'; function Config({ metadata }: { metadata: ParameterMetadataRecord }) { - const dispatch = useContext(AlertDispatchContext); const [status, setStatus] = useState( @@ -70,11 +76,8 @@ function Config({ metadata }: { metadata: ParameterMetadataRecord }) { const { data, mutate, error } = useSWR( 'getConfig', - async () => await alertOnError( - getConfigJson, - "Could not get config", - dispatch - ) + async () => + await alertOnError(getConfigJson, 'Could not get config', dispatch) ); const { data: enabledServers } = useSWR( 'getEnabledServers', @@ -223,8 +226,8 @@ function Config({ metadata }: { metadata: ParameterMetadataRecord }) { } const getConfigJson = async (): Promise => { - const response = await getConfig() - if(response){ + const response = await getConfig(); + if (response) { return await response.json(); } }; diff --git a/web_ui/frontend/app/director/components/DirectorCard.tsx b/web_ui/frontend/app/director/components/DirectorCard.tsx index 305ee0950..07b496bee 100644 --- a/web_ui/frontend/app/director/components/DirectorCard.tsx +++ b/web_ui/frontend/app/director/components/DirectorCard.tsx @@ -96,7 +96,7 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { }, 'Failed to toggle server status', dispatch - ) + ); mutate(); diff --git a/web_ui/frontend/app/director/components/DirectorDropdown.tsx b/web_ui/frontend/app/director/components/DirectorDropdown.tsx index 60d1c433d..6a20d97b3 100644 --- a/web_ui/frontend/app/director/components/DirectorDropdown.tsx +++ b/web_ui/frontend/app/director/components/DirectorDropdown.tsx @@ -107,7 +107,10 @@ const directoryListToTreeHelper = ( tree[path[0]] = {}; } - tree[path[0]] = directoryListToTreeHelper(path.slice(1), tree[path[0]] as StringTree); + tree[path[0]] = directoryListToTreeHelper( + path.slice(1), + tree[path[0]] as StringTree + ); return tree; }; diff --git a/web_ui/frontend/app/director/page.tsx b/web_ui/frontend/app/director/page.tsx index ef7bfde10..a07fb437c 100644 --- a/web_ui/frontend/app/director/page.tsx +++ b/web_ui/frontend/app/director/page.tsx @@ -22,9 +22,7 @@ import { Box, Grid, Skeleton, Typography } from '@mui/material'; import { useContext, useMemo } from 'react'; import useSWR from 'swr'; import { Server } from '@/index'; -import { - DirectorCardList, -} from './components'; +import { DirectorCardList } from './components'; import { getUser } from '@/helpers/login'; import FederationOverview from '@/components/FederationOverview'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; @@ -34,17 +32,16 @@ import { alertOnError } from '@/helpers/util'; import { AlertDispatchContext } from '@/components/AlertProvider'; export default function Page() { - const dispatch = useContext(AlertDispatchContext); const { data } = useSWR( 'getServers', - async () => await alertOnError(getServers, 'Failed to fetch servers', dispatch) + async () => + await alertOnError(getServers, 'Failed to fetch servers', dispatch) ); - const { data: user, error } = useSWR( - 'getUser', - () => alertOnError(getUser, "Failed to fetch user", dispatch) + const { data: user, error } = useSWR('getUser', () => + alertOnError(getUser, 'Failed to fetch user', dispatch) ); const cacheData = useMemo(() => { diff --git a/web_ui/frontend/app/registry/cache/edit/page.tsx b/web_ui/frontend/app/registry/cache/edit/page.tsx index 6c06ab12b..433d81f16 100644 --- a/web_ui/frontend/app/registry/cache/edit/page.tsx +++ b/web_ui/frontend/app/registry/cache/edit/page.tsx @@ -19,9 +19,7 @@ 'use client'; import { PutPage } from '@/app/registry/components/PutPage'; -import { - namespaceToCache, -} from '@/app/registry/components/util'; +import { namespaceToCache } from '@/app/registry/components/util'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; diff --git a/web_ui/frontend/app/registry/cache/register/page.tsx b/web_ui/frontend/app/registry/cache/register/page.tsx index 70386a7db..61d885670 100644 --- a/web_ui/frontend/app/registry/cache/register/page.tsx +++ b/web_ui/frontend/app/registry/cache/register/page.tsx @@ -18,15 +18,12 @@ 'use client'; -import { - namespaceToCache -} from '@/app/registry/components/util'; +import { namespaceToCache } from '@/app/registry/components/util'; import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; import { postGeneralNamespace } from '@/helpers/api'; - export default function Page() { const postCache = async (data: any) => { const cache = namespaceToCache(structuredClone(data)); diff --git a/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx b/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx index 6d7d5135b..a69e3f107 100644 --- a/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx +++ b/web_ui/frontend/app/registry/components/CustomRegistrationField/PubkeyField.tsx @@ -25,7 +25,9 @@ const pubkeyValidator = (value: string) => { } }; -const PubkeyField = ({ ...props }: BaseCustomRegistrationFieldProps) => { +const PubkeyField = ({ + ...props +}: BaseCustomRegistrationFieldProps) => { return ( { - const dispatch = useContext(AlertDispatchContext); const [data, setData] = useState | undefined>( namespace || {} ); - const { data: fields, error } = useSWR[] | undefined>( + const { data: fields, error } = useSWR< + Omit[] | undefined + >( 'optionsNamespaceRegistrationFields', - async () => await alertOnError( - optionsNamespaceRegistrationFields, - "Couldn't fetch registration fields", - dispatch - ), + async () => + await alertOnError( + optionsNamespaceRegistrationFields, + "Couldn't fetch registration fields", + dispatch + ), { fallbackData: [] } ); diff --git a/web_ui/frontend/app/registry/components/PostPage.tsx b/web_ui/frontend/app/registry/components/PostPage.tsx index 91ecf2478..3e22a1631 100644 --- a/web_ui/frontend/app/registry/components/PostPage.tsx +++ b/web_ui/frontend/app/registry/components/PostPage.tsx @@ -28,7 +28,6 @@ import { alertOnError } from '@/helpers/util'; import { AlertDispatchContext } from '@/components/AlertProvider'; const PostPage = ({ update }: NamespaceFormPage) => { - const dispatch = useContext(AlertDispatchContext); const [fromUrl, setFromUrl] = useState(undefined); @@ -41,14 +40,14 @@ const PostPage = ({ update }: NamespaceFormPage) => { if (fromUrl != undefined) { const parsedUrl = await alertOnError( () => new URL(fromUrl), - "Failed to parse URL", + 'Failed to parse URL', dispatch ); if (parsedUrl) { setFromUrl(parsedUrl); } } - })() + })(); }, []); return ( @@ -58,10 +57,11 @@ const PostPage = ({ update }: NamespaceFormPage) => { { await alertOnError( - async () => await submitNamespaceForm(namespace, fromUrl, update), - "Failed to update namespace", + async () => + await submitNamespaceForm(namespace, fromUrl, update), + 'Failed to update namespace', dispatch - ) + ); }} /> diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index 4b3a8c820..1fe498263 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -28,7 +28,8 @@ import { } from '@mui/material'; import React, { ReactNode, - Suspense, useContext, + Suspense, + useContext, useEffect, useMemo, useState, @@ -37,11 +38,9 @@ import React, { import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { Namespace, Alert as AlertType } from '@/index'; import Form from '@/app/registry/components/Form'; -import { - submitNamespaceForm, -} from '@/app/registry/components/util'; +import { submitNamespaceForm } from '@/app/registry/components/util'; import { getNamespace } from '@/helpers/api'; -import { NamespaceFormPage } from '@/app/registry/components' +import { NamespaceFormPage } from '@/app/registry/components'; import { AlertDispatchContext } from '@/components/AlertProvider'; import { alertOnError } from '@/helpers/util'; @@ -59,13 +58,14 @@ const PutPage = ({ update }: NamespaceFormPage) => { if (id === null) { dispatch({ - type: "openAlert", + type: 'openAlert', payload: { - title: "No Namespace ID Provided", - message: "Your URL should contain a query parameter 'id' with the ID of the namespace you want to edit", - onClose: () => dispatch({ type: "closeAlert" }) - } - }) + title: 'No Namespace ID Provided', + message: + "Your URL should contain a query parameter 'id' with the ID of the namespace you want to edit", + onClose: () => dispatch({ type: 'closeAlert' }), + }, + }); return; } @@ -76,32 +76,34 @@ const PutPage = ({ update }: NamespaceFormPage) => { } } catch (e) { dispatch({ - type: "openAlert", + type: 'openAlert', payload: { - title: "Invalid fromUrl provided", - message: "The `fromUrl` parameter provided is not a valid URL, this will only impact your redirection on completion of this form", + title: 'Invalid fromUrl provided', + message: + 'The `fromUrl` parameter provided is not a valid URL, this will only impact your redirection on completion of this form', alertProps: { - severity: "warning" + severity: 'warning', }, - onClose: () => dispatch({ type: "closeAlert" }) - } - }) + onClose: () => dispatch({ type: 'closeAlert' }), + }, + }); } try { setId(parseInt(id)); } catch (e) { dispatch({ - type: "openAlert", + type: 'openAlert', payload: { - title: "Invalid Namespace ID provided", - message: "The Namespace Id provided is not a valid number. Please report this issue, as well as what link directed you here.", + title: 'Invalid Namespace ID provided', + message: + 'The Namespace Id provided is not a valid number. Please report this issue, as well as what link directed you here.', alertProps: { - severity: "error" + severity: 'error', }, - onClose: () => dispatch({ type: "closeAlert" }) - } - }) + onClose: () => dispatch({ type: 'closeAlert' }), + }, + }); } }, []); @@ -112,8 +114,8 @@ const PutPage = ({ update }: NamespaceFormPage) => { async () => await getNamespace(id), "Couldn't get namespace", dispatch - ) - if(response){ + ); + if (response) { setNamespace(response); } } @@ -130,10 +132,11 @@ const PutPage = ({ update }: NamespaceFormPage) => { onSubmit={async (data) => { let namespace = { ...data, id: id }; await alertOnError( - async () => await submitNamespaceForm(namespace, fromUrl, update), - "Failed to update namespace", + async () => + await submitNamespaceForm(namespace, fromUrl, update), + 'Failed to update namespace', dispatch - ) + ); }} /> ) : ( diff --git a/web_ui/frontend/app/registry/components/util.tsx b/web_ui/frontend/app/registry/components/util.tsx index e6c595942..6ddbc0cbf 100644 --- a/web_ui/frontend/app/registry/components/util.tsx +++ b/web_ui/frontend/app/registry/components/util.tsx @@ -31,7 +31,10 @@ export const calculateKeys = (key: string) => { * @param o Object to get the value from * @param key List of keys to traverse */ -export const getValue = (o: Record | undefined, key: string[]): any => { +export const getValue = ( + o: Record | undefined, + key: string[] +): any => { if (o === undefined) { return undefined; } diff --git a/web_ui/frontend/app/registry/denied/page.tsx b/web_ui/frontend/app/registry/denied/page.tsx index 00c792651..7d1775521 100644 --- a/web_ui/frontend/app/registry/denied/page.tsx +++ b/web_ui/frontend/app/registry/denied/page.tsx @@ -18,18 +18,10 @@ 'use client'; -import { - Box, - Grid, - Typography, - Alert, - Collapse -} from '@mui/material'; +import { Box, Grid, Typography, Alert, Collapse } from '@mui/material'; import React, { useContext, useMemo } from 'react'; -import { - CardSkeleton -} from '@/components/Namespace'; +import { CardSkeleton } from '@/components/Namespace'; import { getUser } from '@/helpers/login'; import NamespaceCardList from '@/components/Namespace/NamespaceCardList'; import useSWR from 'swr'; @@ -41,24 +33,13 @@ import { AlertDispatchContext } from '@/components/AlertProvider'; import { alertOnError } from '@/helpers/util'; export default function Home() { + const dispatch = useContext(AlertDispatchContext); - const dispatch = useContext(AlertDispatchContext) - - const { data } = useSWR( - 'getNamespaces', - async () => alertOnError( - getExtendedNamespaces, - "Couldn't fetch namespaces", - dispatch - ) - ) - const { data: user, error } = useSWR( - 'getUser', - async () => alertOnError( - getUser, - "Couldn't fetch user", - dispatch - ) + const { data } = useSWR('getNamespaces', async () => + alertOnError(getExtendedNamespaces, "Couldn't fetch namespaces", dispatch) + ); + const { data: user, error } = useSWR('getUser', async () => + alertOnError(getUser, "Couldn't fetch user", dispatch) ); const deniedNamespaces = useMemo( diff --git a/web_ui/frontend/app/registry/origin/edit/page.tsx b/web_ui/frontend/app/registry/origin/edit/page.tsx index 25629eb54..4fe766aff 100644 --- a/web_ui/frontend/app/registry/origin/edit/page.tsx +++ b/web_ui/frontend/app/registry/origin/edit/page.tsx @@ -19,9 +19,7 @@ 'use client'; import { PutPage } from '@/app/registry/components/PutPage'; -import { - namespaceToOrigin -} from '@/app/registry/components/util'; +import { namespaceToOrigin } from '@/app/registry/components/util'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; import { putGeneralNamespace } from '@/helpers/api'; diff --git a/web_ui/frontend/app/registry/origin/register/page.tsx b/web_ui/frontend/app/registry/origin/register/page.tsx index 59e52d4a7..f52464fef 100644 --- a/web_ui/frontend/app/registry/origin/register/page.tsx +++ b/web_ui/frontend/app/registry/origin/register/page.tsx @@ -18,9 +18,7 @@ 'use client'; -import { - namespaceToOrigin -} from '@/app/registry/components/util'; +import { namespaceToOrigin } from '@/app/registry/components/util'; import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; diff --git a/web_ui/frontend/app/registry/page.tsx b/web_ui/frontend/app/registry/page.tsx index 746a45d6c..ab5204b69 100644 --- a/web_ui/frontend/app/registry/page.tsx +++ b/web_ui/frontend/app/registry/page.tsx @@ -49,16 +49,18 @@ import { alertOnError } from '@/helpers/util'; import { getExtendedNamespaces } from '@/helpers/api'; export default function Home() { - const dispatch = useContext(AlertDispatchContext); - const { data, mutate: mutateNamespaces } = useSWR<{ namespace: Namespace }[] | undefined>( + const { data, mutate: mutateNamespaces } = useSWR< + { namespace: Namespace }[] | undefined + >( 'getNamespaces', - () => alertOnError( - getExtendedNamespaces, - 'Failed to fetch namespaces', - dispatch - ), + () => + alertOnError( + getExtendedNamespaces, + 'Failed to fetch namespaces', + dispatch + ), { fallbackData: [], } @@ -66,7 +68,7 @@ export default function Home() { const { data: user, error } = useSWR( 'getUser', - async () => await alertOnError(getUser, "Error Getting User", dispatch) + async () => await alertOnError(getUser, 'Error Getting User', dispatch) ); const pendingData = useMemo(() => { diff --git a/web_ui/frontend/app/test/page.tsx b/web_ui/frontend/app/test/page.tsx index 96176f517..5f6e11e7e 100644 --- a/web_ui/frontend/app/test/page.tsx +++ b/web_ui/frontend/app/test/page.tsx @@ -5,7 +5,6 @@ import React, { useContext } from 'react'; import { Box } from '@mui/material'; import CodeBlock from '@/components/CodeBlock'; - const Page = () => { const dispatch = useContext(AlertDispatchContext); @@ -16,9 +15,9 @@ const Page = () => { dispatch({ type: 'openAlert', payload: { - title: "Response Time Slow", + title: 'Response Time Slow', alertProps: { - severity: 'error' + severity: 'error', }, message: ( @@ -34,8 +33,7 @@ const Page = () => { ), onClose: () => dispatch({ type: 'closeAlert' }), - - } + }, }); }} > @@ -50,6 +48,6 @@ const Page = () => { ); -} +}; export default Page; diff --git a/web_ui/frontend/components/AlertPortal.tsx b/web_ui/frontend/components/AlertPortal.tsx index 4c482e3a4..4c75a0574 100644 --- a/web_ui/frontend/components/AlertPortal.tsx +++ b/web_ui/frontend/components/AlertPortal.tsx @@ -1,6 +1,12 @@ import { Portal } from '@mui/base'; import React, { ReactNode } from 'react'; -import { Alert, AlertProps, Snackbar, SnackbarProps, AlertTitle } from '@mui/material'; +import { + Alert, + AlertProps, + Snackbar, + SnackbarProps, + AlertTitle, +} from '@mui/material'; export interface AlertPortalProps { onClose: () => void; @@ -19,9 +25,8 @@ export const AlertPortal = ({ alertProps, snackBarProps, }: AlertPortalProps) => { - if (autoHideDuration) { - setTimeout(() => onClose(), autoHideDuration) + setTimeout(() => onClose(), autoHideDuration); } return ( @@ -36,7 +41,7 @@ export const AlertPortal = ({ severity={alertProps?.severity} sx={{ width: '100%' }} > - { title && {title} } + {title && {title}} {message} diff --git a/web_ui/frontend/components/AlertProvider.tsx b/web_ui/frontend/components/AlertProvider.tsx index 91177efa9..11e356488 100644 --- a/web_ui/frontend/components/AlertProvider.tsx +++ b/web_ui/frontend/components/AlertProvider.tsx @@ -4,10 +4,11 @@ import { createContext, Dispatch, useReducer } from 'react'; import { AlertPortal, AlertPortalProps } from '@/components/AlertPortal'; import CodeBlock from '@/components/CodeBlock'; -const defaultAlertContext: AlertPortalProps | undefined = undefined +const defaultAlertContext: AlertPortalProps | undefined = undefined; -export const AlertContext = - createContext(defaultAlertContext); +export const AlertContext = createContext( + defaultAlertContext +); export const AlertDispatchContext = createContext>( () => {} @@ -20,7 +21,7 @@ export const AlertProvider = ({ children }: { children: React.ReactNode }) => { {children} - { state && } + {state && } ); @@ -34,28 +35,31 @@ const alertReducer = ( case 'closeAlert': return undefined; case 'openErrorAlert': - const {title, error, onClose} = action.payload; + const { title, error, onClose } = action.payload; return { title, onClose, message: {error}, alertProps: { - severity: 'error' - } - } + severity: 'error', + }, + }; case 'openAlert': return action.payload; default: return state; } -} +}; -export type AlertReducerAction = closeAlertAction | openErrorAlertAction | openAlertAction; +export type AlertReducerAction = + | closeAlertAction + | openErrorAlertAction + | openAlertAction; type closeAlertAction = { - type: 'closeAlert' -} + type: 'closeAlert'; +}; type openErrorAlertAction = { type: 'openErrorAlert'; @@ -63,10 +67,10 @@ type openErrorAlertAction = { title: string; error: string; onClose: () => void; - } -} + }; +}; type openAlertAction = { - type: 'openAlert', - payload: AlertPortalProps -} + type: 'openAlert'; + payload: AlertPortalProps; +}; diff --git a/web_ui/frontend/components/CodeBlock.tsx b/web_ui/frontend/components/CodeBlock.tsx index 900dc58e6..f742793e7 100644 --- a/web_ui/frontend/components/CodeBlock.tsx +++ b/web_ui/frontend/components/CodeBlock.tsx @@ -8,18 +8,20 @@ import { Box } from '@mui/material'; * @param children * @constructor */ -export const CodeBlock = ({children}: {children: string | string[]}) => { - return - { - navigator.clipboard.writeText(children.toString()); - }} - > - {children} - - -} +export const CodeBlock = ({ children }: { children: string | string[] }) => { + return ( + + { + navigator.clipboard.writeText(children.toString()); + }} + > + {children} + + + ); +}; export default CodeBlock; diff --git a/web_ui/frontend/components/FederationOverview.tsx b/web_ui/frontend/components/FederationOverview.tsx index a47f1d1a5..575e3e8ca 100644 --- a/web_ui/frontend/components/FederationOverview.tsx +++ b/web_ui/frontend/components/FederationOverview.tsx @@ -48,16 +48,12 @@ const FederationOverview = () => { >([]); let getConfigJson = async () => { - const response = await getConfig() + const response = await getConfig(); const responseData = (await response.json()) as Config; const federationUrls = UrlData.map(({ key, text }) => { let url = getObjectValue(responseData, key); - if ( - url && - !url?.startsWith('http://') && - !url?.startsWith('https://') - ) { + if (url && !url?.startsWith('http://') && !url?.startsWith('https://')) { url = 'https://' + url; } @@ -68,7 +64,6 @@ const FederationOverview = () => { }); setConfig(federationUrls); - }; useEffect(() => { diff --git a/web_ui/frontend/components/Namespace/Card.tsx b/web_ui/frontend/components/Namespace/Card.tsx index b2e9d0a30..fcbb80836 100644 --- a/web_ui/frontend/components/Namespace/Card.tsx +++ b/web_ui/frontend/components/Namespace/Card.tsx @@ -108,7 +108,7 @@ export const Card = ({ namespace, authenticated, onUpdate }: CardProps) => { async () => await deleteNamespace(namespace.id), 'Could Not Delete Registration', dispatch - ) + ); setTimeout(() => mutate('getNamespaces'), 600); if (onUpdate) { onUpdate(); diff --git a/web_ui/frontend/components/Namespace/DeniedCard.tsx b/web_ui/frontend/components/Namespace/DeniedCard.tsx index 2db34baaa..6ef6999cb 100644 --- a/web_ui/frontend/components/Namespace/DeniedCard.tsx +++ b/web_ui/frontend/components/Namespace/DeniedCard.tsx @@ -24,7 +24,7 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { const ref = useRef(null); const [transition, setTransition] = useState(false); const dispatch = useContext(AlertDispatchContext); - const alert = useContext(AlertContext) + const alert = useContext(AlertContext); const { mutate } = useSWRConfig(); return ( @@ -88,7 +88,7 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { () => deleteNamespace(namespace.id), 'Could Not Delete Registration', dispatch - ) + ); }} > @@ -104,7 +104,7 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { () => approveNamespace(namespace.id), 'Could Not Approve Registration', dispatch - ) + ); setTimeout(() => mutate('getNamespaces'), 600); }} > diff --git a/web_ui/frontend/components/Namespace/InformationDropdown.tsx b/web_ui/frontend/components/Namespace/InformationDropdown.tsx index 752fe37b6..b83b5df00 100644 --- a/web_ui/frontend/components/Namespace/InformationDropdown.tsx +++ b/web_ui/frontend/components/Namespace/InformationDropdown.tsx @@ -1,6 +1,10 @@ import { Box, Tooltip, Collapse, Grid, Typography } from '@mui/material'; import React from 'react'; -import { Dropdown, InformationSpan, NamespaceAdminMetadata } from '@/components'; +import { + Dropdown, + InformationSpan, + NamespaceAdminMetadata, +} from '@/components'; interface InformationDropdownProps { adminMetadata: NamespaceAdminMetadata; diff --git a/web_ui/frontend/components/Namespace/PendingCard.tsx b/web_ui/frontend/components/Namespace/PendingCard.tsx index f138ab3a4..b661fe284 100644 --- a/web_ui/frontend/components/Namespace/PendingCard.tsx +++ b/web_ui/frontend/components/Namespace/PendingCard.tsx @@ -28,7 +28,7 @@ export const PendingCard = ({ const ref = useRef(null); const [transition, setTransition] = useState(false); - const dispatch = useContext(AlertDispatchContext) + const dispatch = useContext(AlertDispatchContext); return ( @@ -73,13 +73,13 @@ export const PendingCard = ({ sx={{ bgcolor: '#ff00001a', mx: 1 }} color={'error'} onClick={async (e) => { - e.stopPropagation() + e.stopPropagation(); await alertOnError( () => denyNamespace(namespace.id), "Couldn't deny namespace", dispatch - ) - onUpdate() + ); + onUpdate(); }} > @@ -90,13 +90,13 @@ export const PendingCard = ({ sx={{ bgcolor: '#2e7d3224', mx: 1 }} color={'success'} onClick={async (e) => { - e.stopPropagation() + e.stopPropagation(); await alertOnError( () => approveNamespace(namespace.id), "Couldn't approve namespace", dispatch - ) - onUpdate() + ); + onUpdate(); }} > diff --git a/web_ui/frontend/components/Namespace/index.tsx b/web_ui/frontend/components/Namespace/index.tsx index 7f7e4ac4b..3a43facd7 100644 --- a/web_ui/frontend/components/Namespace/index.tsx +++ b/web_ui/frontend/components/Namespace/index.tsx @@ -37,7 +37,6 @@ export interface FlatObject { export type NamespaceCardProps = CardProps & PendingCardProps; - export const getServerType = (namespace: Namespace) => { // If the namespace is empty the value is undefined if (namespace?.prefix == null || namespace.prefix == '') { diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts index dbbfe8b9e..acd04ce28 100644 --- a/web_ui/frontend/helpers/api.ts +++ b/web_ui/frontend/helpers/api.ts @@ -26,26 +26,22 @@ export async function fetchApi( } throw new Error(alertMessage); } - return response + return response; } catch (e) { if (e instanceof Error) { - throw Error("Fetch to API Failed", { cause: e }) + throw Error('Fetch to API Failed', { cause: e }); } else { - throw Error("Fetch to API Failed", { cause: e}) + throw Error('Fetch to API Failed', { cause: e }); } } } - /** * Get config */ export const getConfig = async (): Promise => { - return fetchApi( - async () => await secureFetch('/api/v1.0/config') - ) -} - + return fetchApi(async () => await secureFetch('/api/v1.0/config')); +}; /** * Deletes a namespace @@ -53,27 +49,25 @@ export const getConfig = async (): Promise => { */ export const deleteNamespace = async (id: number) => { return fetchApi( - async () => await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}`, { - method: 'DELETE', - }) - ) + async () => + await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}`, { + method: 'DELETE', + }) + ); }; - /** * Approves a namespace * @param id Namespace ID */ export const approveNamespace = async (id: number) => { return fetchApi( - async () => await secureFetch( - `/api/v1.0/registry_ui/namespaces/${id}/approve`, - { + async () => + await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}/approve`, { method: 'PATCH', - } - ) - ) -} + }) + ); +}; /** * Denies a namespace @@ -81,14 +75,12 @@ export const approveNamespace = async (id: number) => { */ export const denyNamespace = async (id: number) => { return fetchApi( - async () => await secureFetch( - `/api/v1.0/registry_ui/namespaces/${id}/deny`, - { + async () => + await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}/deny`, { method: 'PATCH', - } - ) - ) -} + }) + ); +}; /** * Enables a server on the director @@ -96,14 +88,12 @@ export const denyNamespace = async (id: number) => { */ export const allowServer = async (name: string) => { return fetchApi( - async () => await secureFetch( - `/api/v1.0/director_ui/servers/allow/${name}`, - { + async () => + await secureFetch(`/api/v1.0/director_ui/servers/allow/${name}`, { method: 'PATCH', - } - ) - ) -} + }) + ); +}; /** * Filters ( Disables ) a server on the director @@ -111,20 +101,20 @@ export const allowServer = async (name: string) => { */ export const filterServer = async (name: string) => { return fetchApi( - async () => await secureFetch( - `/api/v1.0/director_ui/servers/filter/${name}`, - { + async () => + await secureFetch(`/api/v1.0/director_ui/servers/filter/${name}`, { method: 'PATCH', - } - ) - ) -} + }) + ); +}; /** * Get extended namespaces */ -export const getExtendedNamespaces = async (): Promise<{ namespace: Namespace }[]> => { - const response = await getNamespaces() +export const getExtendedNamespaces = async (): Promise< + { namespace: Namespace }[] +> => { + const response = await getNamespaces(); const data: Namespace[] = await response.json(); data.sort((a, b) => (a.id > b.id ? 1 : -1)); data.forEach((namespace) => { @@ -143,7 +133,7 @@ export const getExtendedNamespaces = async (): Promise<{ namespace: Namespace }[ return data.map((d) => { return { namespace: d }; }); -} +}; /** * Get namespaces @@ -156,7 +146,7 @@ export const getNamespaces = async (): Promise => { const response = await fetchApi(async () => await fetch(url)); return await response.json(); -} +}; /** * Gets a namespace by ID @@ -177,35 +167,31 @@ export const postGeneralNamespace = async ( data: Namespace ): Promise => { return await fetchApi( - async () => - await fetch('/api/v1.0/registry_ui/namespaces', { - body: JSON.stringify(data), - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - }) - ) + async () => + await fetch('/api/v1.0/registry_ui/namespaces', { + body: JSON.stringify(data), + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + }) + ); }; export const putGeneralNamespace = async ( data: Namespace ): Promise => { - return await fetchApi( - async () => { - return fetch(`/api/v1.0/registry_ui/namespaces/${data.id}`, - { - body: JSON.stringify(data), - method: 'PUT', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - } - ) - } - ) + return await fetchApi(async () => { + return fetch(`/api/v1.0/registry_ui/namespaces/${data.id}`, { + body: JSON.stringify(data), + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + }); + }); }; /** @@ -215,10 +201,13 @@ export const putGeneralNamespace = async ( export const optionsNamespaceRegistrationFields = async (): Promise< Omit[] > => { - const response = await fetchApi(async () => await fetch('/api/v1.0/registry_ui/namespaces', { - method: 'OPTIONS', - })) - return response.json() + const response = await fetchApi( + async () => + await fetch('/api/v1.0/registry_ui/namespaces', { + method: 'OPTIONS', + }) + ); + return response.json(); }; /** @@ -236,8 +225,8 @@ export const initLogin = async (code: string): Promise => { code: code, }), }) - ) -} + ); +}; /** * Reset ( Do initial ) Login @@ -254,13 +243,16 @@ export const resetLogin = async (password: string): Promise => { password: password, }), }) - ) -} + ); +}; /** * Login */ -export const login = async (password: string, user: string = "admin"): Promise => { +export const login = async ( + password: string, + user: string = 'admin' +): Promise => { return await fetchApi( async () => await fetch('/api/v1.0/auth/login', { @@ -273,5 +265,5 @@ export const login = async (password: string, user: string = "admin"): Promise => { diff --git a/web_ui/frontend/helpers/util.ts b/web_ui/frontend/helpers/util.ts index 42bb56cba..a79ad2196 100644 --- a/web_ui/frontend/helpers/util.ts +++ b/web_ui/frontend/helpers/util.ts @@ -84,7 +84,6 @@ export function evaluateOrReturn( return o as T; } - /** * Get the average of an array of numbers * @param arr Array of numbers @@ -95,28 +94,27 @@ export const average = (arr: number[]) => { type ErrorWithCause = Error & { cause?: Error }; - /** * If an error is caught from f then display the error via an alert UI */ -export async function alertOnError( +export async function alertOnError( f: () => Promise | T | undefined, title: string = 'Error', dispatch: Dispatch -){ +) { try { return await f(); } catch (error) { console.error(error); - if(error instanceof Error) { + if (error instanceof Error) { dispatch({ - type: "openErrorAlert", + type: 'openErrorAlert', payload: { title, error: errorToString(error as ErrorWithCause), - onClose: () => dispatch({ type: "closeAlert" }) - } - }) + onClose: () => dispatch({ type: 'closeAlert' }), + }, + }); } } } @@ -125,17 +123,18 @@ export async function alertOnError( * Convert a error into a string * @param error */ -export const errorToString = (error: ErrorWithCause) : string => { - - if(error?.cause){ - +export const errorToString = (error: ErrorWithCause): string => { + if (error?.cause) { // Check that error is instance of Error - if(!(error?.cause instanceof Error)) { - console.error("Malformed error, cause is not an instance of Error", error) + if (!(error?.cause instanceof Error)) { + console.error( + 'Malformed error, cause is not an instance of Error', + error + ); } - return `${error.message}\n↳ ${errorToString(error.cause as ErrorWithCause)}` + return `${error.message}\n↳ ${errorToString(error.cause as ErrorWithCause)}`; } - return `${error.message}` -} + return `${error.message}`; +}; From 9c080478289a27b82e7420815cf744dffcb6b25e Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 12 Nov 2024 12:29:43 -0600 Subject: [PATCH 07/86] Update Error Handling - Move registry pages behind auth check - Create get.ts which holds api endpoints that get updated - Fix AuthenticatedContent.tsx to check it auth value is true --- .../frontend/app/registry/cache/edit/page.tsx | 5 +- .../app/registry/cache/register/page.tsx | 5 +- .../frontend/app/registry/components/Form.tsx | 10 ++- .../app/registry/components/PutPage.tsx | 2 +- web_ui/frontend/app/registry/denied/page.tsx | 4 +- .../app/registry/namespace/edit/page.tsx | 5 +- .../app/registry/namespace/register/page.tsx | 5 +- .../app/registry/origin/edit/page.tsx | 5 +- .../app/registry/origin/register/page.tsx | 5 +- web_ui/frontend/app/registry/page.tsx | 4 +- .../components/FederationOverview.tsx | 58 +++----------- web_ui/frontend/components/Namespace/Card.tsx | 2 +- .../components/Namespace/DeniedCard.tsx | 2 +- .../layout/AuthenticatedContent.tsx | 6 +- web_ui/frontend/dev/image/nginx.conf | 2 + web_ui/frontend/helpers/api.ts | 66 +++++----------- web_ui/frontend/helpers/get.ts | 78 ++++++++++++++++++- 17 files changed, 148 insertions(+), 116 deletions(-) diff --git a/web_ui/frontend/app/registry/cache/edit/page.tsx b/web_ui/frontend/app/registry/cache/edit/page.tsx index 433d81f16..c9f9eaec3 100644 --- a/web_ui/frontend/app/registry/cache/edit/page.tsx +++ b/web_ui/frontend/app/registry/cache/edit/page.tsx @@ -24,6 +24,7 @@ import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; import { putGeneralNamespace } from '@/helpers/api'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export default function Page() { const putCache = async (data: any) => { @@ -43,7 +44,9 @@ export default function Page() { - + + + diff --git a/web_ui/frontend/app/registry/cache/register/page.tsx b/web_ui/frontend/app/registry/cache/register/page.tsx index 61d885670..0364bc5f5 100644 --- a/web_ui/frontend/app/registry/cache/register/page.tsx +++ b/web_ui/frontend/app/registry/cache/register/page.tsx @@ -23,6 +23,7 @@ import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; import { postGeneralNamespace } from '@/helpers/api'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export default function Page() { const postCache = async (data: any) => { @@ -42,7 +43,9 @@ export default function Page() { - + + + diff --git a/web_ui/frontend/app/registry/components/Form.tsx b/web_ui/frontend/app/registry/components/Form.tsx index f235c85f7..230999708 100644 --- a/web_ui/frontend/app/registry/components/Form.tsx +++ b/web_ui/frontend/app/registry/components/Form.tsx @@ -58,12 +58,16 @@ const Form = ({ namespace, onSubmit }: FormProps) => { Omit[] | undefined >( 'optionsNamespaceRegistrationFields', - async () => - await alertOnError( + async () => { + const response = await alertOnError( optionsNamespaceRegistrationFields, "Couldn't fetch registration fields", dispatch - ), + ); + if(response){ + return await response.json(); + } + }, { fallbackData: [] } ); diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index 1fe498263..ee2e1b850 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -116,7 +116,7 @@ const PutPage = ({ update }: NamespaceFormPage) => { dispatch ); if (response) { - setNamespace(response); + setNamespace(await response.json()); } } })(); diff --git a/web_ui/frontend/app/registry/denied/page.tsx b/web_ui/frontend/app/registry/denied/page.tsx index 7d1775521..851fb6d03 100644 --- a/web_ui/frontend/app/registry/denied/page.tsx +++ b/web_ui/frontend/app/registry/denied/page.tsx @@ -28,14 +28,14 @@ import useSWR from 'swr'; import { CardProps } from '@/components/Namespace/Card'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import DeniedCard from '@/components/Namespace/DeniedCard'; -import { getExtendedNamespaces } from '@/helpers/api'; +import { getExtendedNamespaces } from '@/helpers/get'; import { AlertDispatchContext } from '@/components/AlertProvider'; import { alertOnError } from '@/helpers/util'; export default function Home() { const dispatch = useContext(AlertDispatchContext); - const { data } = useSWR('getNamespaces', async () => + const { data } = useSWR('getExtendedNamespaces', async () => alertOnError(getExtendedNamespaces, "Couldn't fetch namespaces", dispatch) ); const { data: user, error } = useSWR('getUser', async () => diff --git a/web_ui/frontend/app/registry/namespace/edit/page.tsx b/web_ui/frontend/app/registry/namespace/edit/page.tsx index 757897925..de07c32fd 100644 --- a/web_ui/frontend/app/registry/namespace/edit/page.tsx +++ b/web_ui/frontend/app/registry/namespace/edit/page.tsx @@ -22,6 +22,7 @@ import { PutPage } from '@/app/registry/components/PutPage'; import { putGeneralNamespace } from '@/helpers/api'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export default function Page() { const putCache = async (data: any) => { @@ -40,7 +41,9 @@ export default function Page() { - + + + diff --git a/web_ui/frontend/app/registry/namespace/register/page.tsx b/web_ui/frontend/app/registry/namespace/register/page.tsx index 6d51df66a..48d727f74 100644 --- a/web_ui/frontend/app/registry/namespace/register/page.tsx +++ b/web_ui/frontend/app/registry/namespace/register/page.tsx @@ -22,6 +22,7 @@ import { postGeneralNamespace } from '@/helpers/api'; import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export default function Page() { const postCache = async (data: any) => { @@ -40,7 +41,9 @@ export default function Page() { - + + + diff --git a/web_ui/frontend/app/registry/origin/edit/page.tsx b/web_ui/frontend/app/registry/origin/edit/page.tsx index 4fe766aff..a40a7b81b 100644 --- a/web_ui/frontend/app/registry/origin/edit/page.tsx +++ b/web_ui/frontend/app/registry/origin/edit/page.tsx @@ -23,6 +23,7 @@ import { namespaceToOrigin } from '@/app/registry/components/util'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; import { putGeneralNamespace } from '@/helpers/api'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export default function Page() { const putCache = async (data: any) => { @@ -42,7 +43,9 @@ export default function Page() { - + + + diff --git a/web_ui/frontend/app/registry/origin/register/page.tsx b/web_ui/frontend/app/registry/origin/register/page.tsx index f52464fef..dc61fc18b 100644 --- a/web_ui/frontend/app/registry/origin/register/page.tsx +++ b/web_ui/frontend/app/registry/origin/register/page.tsx @@ -23,6 +23,7 @@ import { PostPage } from '@/app/registry/components/PostPage'; import { Box, Grid, Typography } from '@mui/material'; import React from 'react'; import { postGeneralNamespace } from '@/helpers/api'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export default function Page() { const postCache = async (data: any) => { @@ -42,7 +43,9 @@ export default function Page() { - + + + diff --git a/web_ui/frontend/app/registry/page.tsx b/web_ui/frontend/app/registry/page.tsx index ab5204b69..0835c9fc9 100644 --- a/web_ui/frontend/app/registry/page.tsx +++ b/web_ui/frontend/app/registry/page.tsx @@ -46,7 +46,7 @@ import { CardProps } from '@/components/Namespace/Card'; import { PendingCardProps } from '@/components/Namespace/PendingCard'; import { AlertDispatchContext } from '@/components/AlertProvider'; import { alertOnError } from '@/helpers/util'; -import { getExtendedNamespaces } from '@/helpers/api'; +import { getExtendedNamespaces } from '@/helpers/get'; export default function Home() { const dispatch = useContext(AlertDispatchContext); @@ -54,7 +54,7 @@ export default function Home() { const { data, mutate: mutateNamespaces } = useSWR< { namespace: Namespace }[] | undefined >( - 'getNamespaces', + 'getExtendedNamespaces', () => alertOnError( getExtendedNamespaces, diff --git a/web_ui/frontend/components/FederationOverview.tsx b/web_ui/frontend/components/FederationOverview.tsx index 575e3e8ca..d386c62f7 100644 --- a/web_ui/frontend/components/FederationOverview.tsx +++ b/web_ui/frontend/components/FederationOverview.tsx @@ -8,6 +8,8 @@ import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import Link from 'next/link'; import { getErrorMessage, getObjectValue } from '@/helpers/util'; import { getConfig } from '@/helpers/api'; +import { getFederationUrls } from '@/helpers/get'; +import useSWR from 'swr'; const LinkBox = ({ href, text }: { href: string; text: string }) => { return ( @@ -30,66 +32,28 @@ const LinkBox = ({ href, text }: { href: string; text: string }) => { ); }; -const UrlData = [ - { key: ['Federation', 'NamespaceUrl'], text: 'Namespace Registry' }, - { key: ['Federation', 'DirectorUrl'], text: 'Director' }, - { key: ['Federation', 'RegistryUrl'], text: 'Registry' }, - { - key: ['Federation', 'TopologyNamespaceUrl'], - text: 'Topology Namespace', - }, - { key: ['Federation', 'DiscoveryUrl'], text: 'Discovery' }, - { key: ['Federation', 'JwkUrl'], text: 'JWK' }, -]; const FederationOverview = () => { - const [config, setConfig] = useState< - { text: string; url: string | undefined }[] - >([]); - let getConfigJson = async () => { - const response = await getConfig(); - const responseData = (await response.json()) as Config; - - const federationUrls = UrlData.map(({ key, text }) => { - let url = getObjectValue(responseData, key); - if (url && !url?.startsWith('http://') && !url?.startsWith('https://')) { - url = 'https://' + url; - } - - return { - text, - url, - }; - }); - - setConfig(federationUrls); - }; - - useEffect(() => { - getConfigJson(); - }, []); - - if (config === undefined) { - return; - } + const {data : federationUrls , error} = useSWR( + 'getFederationUrls', + getFederationUrls, + {fallbackData: []} + ); return ( - u?.role == 'admin'} - > - {!Object.values(config).every((x) => x == undefined) ? ( + <> + {!Object.values(federationUrls).every((x) => x == undefined) ? ( Federation Overview ) : null} - {config.map(({ text, url }) => { + {federationUrls.map(({ text, url }) => { if (url) { return ; } })} - + ); }; diff --git a/web_ui/frontend/components/Namespace/Card.tsx b/web_ui/frontend/components/Namespace/Card.tsx index fcbb80836..01d443243 100644 --- a/web_ui/frontend/components/Namespace/Card.tsx +++ b/web_ui/frontend/components/Namespace/Card.tsx @@ -109,7 +109,7 @@ export const Card = ({ namespace, authenticated, onUpdate }: CardProps) => { 'Could Not Delete Registration', dispatch ); - setTimeout(() => mutate('getNamespaces'), 600); + setTimeout(() => mutate('getExtendedNamespaces'), 600); if (onUpdate) { onUpdate(); } diff --git a/web_ui/frontend/components/Namespace/DeniedCard.tsx b/web_ui/frontend/components/Namespace/DeniedCard.tsx index 6ef6999cb..93617c531 100644 --- a/web_ui/frontend/components/Namespace/DeniedCard.tsx +++ b/web_ui/frontend/components/Namespace/DeniedCard.tsx @@ -105,7 +105,7 @@ export const DeniedCard = ({ namespace, authenticated }: DeniedCardProps) => { 'Could Not Approve Registration', dispatch ); - setTimeout(() => mutate('getNamespaces'), 600); + setTimeout(() => mutate('getExtendedNamespaces'), 600); }} > diff --git a/web_ui/frontend/components/layout/AuthenticatedContent.tsx b/web_ui/frontend/components/layout/AuthenticatedContent.tsx index 5eef49bcc..966a98ab8 100644 --- a/web_ui/frontend/components/layout/AuthenticatedContent.tsx +++ b/web_ui/frontend/components/layout/AuthenticatedContent.tsx @@ -69,7 +69,7 @@ const AuthenticatedContent = ({ if (data && checkAuthentication) { return checkAuthentication(data); } else { - return data?.authenticated !== undefined; + return data?.authenticated === undefined ? false : data.authenticated; } }, [data, checkAuthentication]); @@ -84,9 +84,9 @@ const AuthenticatedContent = ({ // Redirect to login page if not authenticated and redirect is true useEffect(() => { if (!isValidating && !authenticated && redirect) { - router.push('/login/?returnURL=' + pageUrl); + router.replace('/login/?returnURL=' + pageUrl); } - }, [data, isValidating]); + }, [data, isValidating, authenticated]); // If there was a error then print it to the screen if (error) { diff --git a/web_ui/frontend/dev/image/nginx.conf b/web_ui/frontend/dev/image/nginx.conf index 97fa6bc7e..4c5f83fda 100644 --- a/web_ui/frontend/dev/image/nginx.conf +++ b/web_ui/frontend/dev/image/nginx.conf @@ -56,6 +56,8 @@ http { proxy_connect_timeout 10s; proxy_set_header X-Real-IP $remote_addr; proxy_pass http://host.docker.internal:3000; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; } gzip on; diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts index acd04ce28..00b9921a8 100644 --- a/web_ui/frontend/helpers/api.ts +++ b/web_ui/frontend/helpers/api.ts @@ -1,9 +1,12 @@ +/** + * API Helper Functions + * + * Strictly return the response from the API, throwing an error if the response is not ok + */ + import { secureFetch } from '@/helpers/login'; import { getErrorMessage } from '@/helpers/util'; -import { Alert, Namespace } from '@/index'; -import { CustomRegistrationFieldProps } from '@/app/registry/components/CustomRegistrationField'; - -// TODO: Decide if we should standardize the output in all of these functions. Should they all be responses? +import { Namespace } from '@/index'; /** * Wraps an api request with error handling for both the request and the response if error @@ -60,7 +63,7 @@ export const deleteNamespace = async (id: number) => { * Approves a namespace * @param id Namespace ID */ -export const approveNamespace = async (id: number) => { +export const approveNamespace = async (id: number): Promise => { return fetchApi( async () => await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}/approve`, { @@ -73,7 +76,7 @@ export const approveNamespace = async (id: number) => { * Denies a namespace * @param id Namespace ID */ -export const denyNamespace = async (id: number) => { +export const denyNamespace = async (id: number): Promise => { return fetchApi( async () => await secureFetch(`/api/v1.0/registry_ui/namespaces/${id}/deny`, { @@ -86,7 +89,7 @@ export const denyNamespace = async (id: number) => { * Enables a server on the director * @param name Server name */ -export const allowServer = async (name: string) => { +export const allowServer = async (name: string): Promise => { return fetchApi( async () => await secureFetch(`/api/v1.0/director_ui/servers/allow/${name}`, { @@ -99,7 +102,7 @@ export const allowServer = async (name: string) => { * Filters ( Disables ) a server on the director * @param name Server name */ -export const filterServer = async (name: string) => { +export const filterServer = async (name: string): Promise => { return fetchApi( async () => await secureFetch(`/api/v1.0/director_ui/servers/filter/${name}`, { @@ -108,33 +111,6 @@ export const filterServer = async (name: string) => { ); }; -/** - * Get extended namespaces - */ -export const getExtendedNamespaces = async (): Promise< - { namespace: Namespace }[] -> => { - const response = await getNamespaces(); - const data: Namespace[] = await response.json(); - data.sort((a, b) => (a.id > b.id ? 1 : -1)); - data.forEach((namespace) => { - if (namespace.prefix.startsWith('/caches/')) { - namespace.type = 'cache'; - namespace.prefix = namespace.prefix.replace('/caches/', ''); - } else if (namespace.prefix.startsWith('/origins/')) { - namespace.type = 'origin'; - namespace.prefix = namespace.prefix.replace('/origins/', ''); - } else { - namespace.type = 'namespace'; - } - }); - - // TODO: This extra should be done somewhere else, why is it done? - return data.map((d) => { - return { namespace: d }; - }); -}; - /** * Get namespaces */ @@ -144,8 +120,7 @@ export const getNamespaces = async (): Promise => { window.location.origin ); - const response = await fetchApi(async () => await fetch(url)); - return await response.json(); + return await fetchApi(async () => await fetch(url)); }; /** @@ -154,13 +129,12 @@ export const getNamespaces = async (): Promise => { */ export const getNamespace = async ( id: string | number -): Promise => { +): Promise => { const url = new URL( `/api/v1.0/registry_ui/namespaces/${id}`, window.location.origin ); - const response = await fetchApi(async () => await fetch(url)); - return await response.json(); + return await fetchApi(async () => await fetch(url)); }; export const postGeneralNamespace = async ( @@ -168,7 +142,7 @@ export const postGeneralNamespace = async ( ): Promise => { return await fetchApi( async () => - await fetch('/api/v1.0/registry_ui/namespaces', { + await secureFetch('/api/v1.0/registry_ui/namespaces', { body: JSON.stringify(data), method: 'POST', headers: { @@ -183,7 +157,7 @@ export const putGeneralNamespace = async ( data: Namespace ): Promise => { return await fetchApi(async () => { - return fetch(`/api/v1.0/registry_ui/namespaces/${data.id}`, { + return secureFetch(`/api/v1.0/registry_ui/namespaces/${data.id}`, { body: JSON.stringify(data), method: 'PUT', headers: { @@ -196,18 +170,14 @@ export const putGeneralNamespace = async ( /** * Get registration fields from options for namespace - * // TODO: Complain about the misuse of options */ -export const optionsNamespaceRegistrationFields = async (): Promise< - Omit[] -> => { - const response = await fetchApi( +export const optionsNamespaceRegistrationFields = async (): Promise => { + return await fetchApi( async () => await fetch('/api/v1.0/registry_ui/namespaces', { method: 'OPTIONS', }) ); - return response.json(); }; /** diff --git a/web_ui/frontend/helpers/get.ts b/web_ui/frontend/helpers/get.ts index adb772466..a32d8ecbf 100644 --- a/web_ui/frontend/helpers/get.ts +++ b/web_ui/frontend/helpers/get.ts @@ -1,6 +1,15 @@ -import { ParameterValueRecord } from '@/components/configuration'; -import { getConfig as getConfigResponse } from '@/helpers/api'; +/** + * API wrappers for manipulating fetched data + * + * @module helpers/get + */ + + +import { Config, ParameterValueRecord } from '@/components/configuration'; +import { getConfig as getConfigResponse, getNamespaces } from '@/helpers/api'; import { flattenObject } from '@/app/config/util'; +import { Namespace } from '@/index'; +import { getObjectValue } from '@/helpers/util'; export const getConfig = async (): Promise => { let response = await getConfigResponse(); @@ -8,3 +17,68 @@ export const getConfig = async (): Promise => { let flatData = flattenObject(data); return flatData; }; + +/** + * Get extended namespaces + */ +export const getExtendedNamespaces = async (): Promise< + { namespace: Namespace }[] +> => { + const response = await getNamespaces(); + const data: Namespace[] = await response.json(); + data.sort((a, b) => (a.id > b.id ? 1 : -1)); + data.forEach((namespace) => { + if (namespace.prefix.startsWith('/caches/')) { + namespace.type = 'cache'; + namespace.prefix = namespace.prefix.replace('/caches/', ''); + } else if (namespace.prefix.startsWith('/origins/')) { + namespace.type = 'origin'; + namespace.prefix = namespace.prefix.replace('/origins/', ''); + } else { + namespace.type = 'namespace'; + } + }); + + return data.map((d) => { + return { namespace: d }; + }); +}; + +/** + * Get federation URLs + */ +export const getFederationUrls = async () => { + try { + const response = await getConfigResponse(); + const responseData = (await response.json()) as Config; + + const federationUrls = UrlData.map(({ key, text }) => { + let url = getObjectValue(responseData, key); + if (url && !url?.startsWith('http://') && !url?.startsWith('https://')) { + url = 'https://' + url; + } + + return { + text, + url, + }; + }); + + return federationUrls + } catch (e) { + console.error(e); + return []; + } + +} +const UrlData = [ + { key: ['Federation', 'NamespaceUrl'], text: 'Namespace Registry' }, + { key: ['Federation', 'DirectorUrl'], text: 'Director' }, + { key: ['Federation', 'RegistryUrl'], text: 'Registry' }, + { + key: ['Federation', 'TopologyNamespaceUrl'], + text: 'Topology Namespace', + }, + { key: ['Federation', 'DiscoveryUrl'], text: 'Discovery' }, + { key: ['Federation', 'JwkUrl'], text: 'JWK' }, +]; From 54739f39baac62b66b9ff7f4eeca593aed38e25d Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 12 Nov 2024 14:08:34 -0600 Subject: [PATCH 08/86] Format --- .../frontend/app/registry/components/Form.tsx | 2 +- .../app/registry/namespace/register/page.tsx | 2 +- .../components/FederationOverview.tsx | 6 ++---- web_ui/frontend/helpers/api.ts | 21 +++++++++---------- web_ui/frontend/helpers/get.ts | 6 ++---- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/web_ui/frontend/app/registry/components/Form.tsx b/web_ui/frontend/app/registry/components/Form.tsx index 230999708..971cf1e04 100644 --- a/web_ui/frontend/app/registry/components/Form.tsx +++ b/web_ui/frontend/app/registry/components/Form.tsx @@ -64,7 +64,7 @@ const Form = ({ namespace, onSubmit }: FormProps) => { "Couldn't fetch registration fields", dispatch ); - if(response){ + if (response) { return await response.json(); } }, diff --git a/web_ui/frontend/app/registry/namespace/register/page.tsx b/web_ui/frontend/app/registry/namespace/register/page.tsx index 48d727f74..be3aaeae2 100644 --- a/web_ui/frontend/app/registry/namespace/register/page.tsx +++ b/web_ui/frontend/app/registry/namespace/register/page.tsx @@ -42,7 +42,7 @@ export default function Page() { - + diff --git a/web_ui/frontend/components/FederationOverview.tsx b/web_ui/frontend/components/FederationOverview.tsx index d386c62f7..f47d36841 100644 --- a/web_ui/frontend/components/FederationOverview.tsx +++ b/web_ui/frontend/components/FederationOverview.tsx @@ -32,13 +32,11 @@ const LinkBox = ({ href, text }: { href: string; text: string }) => { ); }; - const FederationOverview = () => { - - const {data : federationUrls , error} = useSWR( + const { data: federationUrls, error } = useSWR( 'getFederationUrls', getFederationUrls, - {fallbackData: []} + { fallbackData: [] } ); return ( diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts index 00b9921a8..9c083e351 100644 --- a/web_ui/frontend/helpers/api.ts +++ b/web_ui/frontend/helpers/api.ts @@ -127,9 +127,7 @@ export const getNamespaces = async (): Promise => { * Gets a namespace by ID * @param id Namespace ID */ -export const getNamespace = async ( - id: string | number -): Promise => { +export const getNamespace = async (id: string | number): Promise => { const url = new URL( `/api/v1.0/registry_ui/namespaces/${id}`, window.location.origin @@ -171,14 +169,15 @@ export const putGeneralNamespace = async ( /** * Get registration fields from options for namespace */ -export const optionsNamespaceRegistrationFields = async (): Promise => { - return await fetchApi( - async () => - await fetch('/api/v1.0/registry_ui/namespaces', { - method: 'OPTIONS', - }) - ); -}; +export const optionsNamespaceRegistrationFields = + async (): Promise => { + return await fetchApi( + async () => + await fetch('/api/v1.0/registry_ui/namespaces', { + method: 'OPTIONS', + }) + ); + }; /** * Initializes a login via terminal code diff --git a/web_ui/frontend/helpers/get.ts b/web_ui/frontend/helpers/get.ts index a32d8ecbf..629554330 100644 --- a/web_ui/frontend/helpers/get.ts +++ b/web_ui/frontend/helpers/get.ts @@ -4,7 +4,6 @@ * @module helpers/get */ - import { Config, ParameterValueRecord } from '@/components/configuration'; import { getConfig as getConfigResponse, getNamespaces } from '@/helpers/api'; import { flattenObject } from '@/app/config/util'; @@ -64,13 +63,12 @@ export const getFederationUrls = async () => { }; }); - return federationUrls + return federationUrls; } catch (e) { console.error(e); return []; } - -} +}; const UrlData = [ { key: ['Federation', 'NamespaceUrl'], text: 'Namespace Registry' }, { key: ['Federation', 'DirectorUrl'], text: 'Director' }, From ca52f27475edcbc7055fdd005309816c67250b7c Mon Sep 17 00:00:00 2001 From: alexandertuna Date: Tue, 12 Nov 2024 16:49:34 -0600 Subject: [PATCH 09/86] Update macos.mdx to make architectures match --- docs/pages/install/macos.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/install/macos.mdx b/docs/pages/install/macos.mdx index 23e51d5c9..b6f7e5185 100644 --- a/docs/pages/install/macos.mdx +++ b/docs/pages/install/macos.mdx @@ -20,7 +20,7 @@ Pelican provides a binary executable file instead of a `DMG` installer for MacOS Example to install Pelican executable for an Apple Silicon Mac: ```bash - curl -LO https://github.com/PelicanPlatform/pelican/releases/download/v7.10.5/pelican_Darwin_x86_64.tar.gz + curl -LO https://github.com/PelicanPlatform/pelican/releases/download/v7.10.5/pelican_Darwin_arm64.tar.gz tar -zxvf pelican_Darwin_arm64.tar.gz ``` From 7c2da3d5addec9078ba69665228a34ba2e2d0aa2 Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Wed, 13 Nov 2024 23:41:05 +0000 Subject: [PATCH 10/86] Add pelican_plugin as acceptable plugin name --- cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index b6d4f0cea..295915339 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,7 +45,7 @@ func handleCLI(args []string) error { // Being case-insensitive execName = strings.ToLower(execName) - if strings.HasPrefix(execName, "stash_plugin") || strings.HasPrefix(execName, "osdf_plugin") || strings.HasPrefix(execName, "pelican_xfer_plugin") { + if strings.HasPrefix(execName, "stash_plugin") || strings.HasPrefix(execName, "osdf_plugin") || strings.HasPrefix(execName, "pelican_xfer_plugin") || strings.HasPrefix(execName, "pelican_plugin") { stashPluginMain(args[1:]) } else if strings.HasPrefix(execName, "stashcp") { err := copyCmd.Execute() From 093cf49871ebe2eada8a03f88b8bd40bedefba16 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Thu, 14 Nov 2024 16:19:24 -0600 Subject: [PATCH 11/86] Add Namespace capabilities view to Director - Show namespace capabilities - Show interaction between namespace and server capabilities --- .../app/director/components/DirectorCard.tsx | 40 ++++- .../director/components/DirectorCardList.tsx | 3 +- .../director/components/DirectorDropdown.tsx | 138 +++++++----------- .../app/director/components/index.tsx | 32 ++++ web_ui/frontend/app/director/page.tsx | 12 +- .../components/CapabilitiesDisplay.tsx | 28 +++- .../frontend/components/DataExportTable.tsx | 2 +- web_ui/frontend/components/DirectoryTree.tsx | 3 +- .../components/NamespaceCapabilitiesTable.tsx | 3 + .../components/ServerCapabilitiesTable.tsx | 74 ++++++++++ web_ui/frontend/index.d.ts | 9 +- web_ui/frontend/types.ts | 55 +++++++ 12 files changed, 286 insertions(+), 113 deletions(-) create mode 100644 web_ui/frontend/components/NamespaceCapabilitiesTable.tsx create mode 100644 web_ui/frontend/components/ServerCapabilitiesTable.tsx create mode 100644 web_ui/frontend/types.ts diff --git a/web_ui/frontend/app/director/components/DirectorCard.tsx b/web_ui/frontend/app/director/components/DirectorCard.tsx index 72419fd18..36de1b8d3 100644 --- a/web_ui/frontend/app/director/components/DirectorCard.tsx +++ b/web_ui/frontend/app/director/components/DirectorCard.tsx @@ -1,5 +1,5 @@ import { Authenticated, secureFetch } from '@/helpers/login'; -import React, { useRef, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { Avatar, Box, @@ -23,9 +23,10 @@ import Link from 'next/link'; import { User } from '@/index'; import { getErrorMessage } from '@/helpers/util'; import { DirectorDropdown } from '@/app/director/components/DirectorDropdown'; +import { ServerDetailed, ServerGeneral } from '@/types'; export interface DirectorCardProps { - server: Server; + server: ServerGeneral; authenticated?: User; } @@ -34,9 +35,19 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { const [error, setError] = useState(undefined); const [disabled, setDisabled] = useState(false); const [dropdownOpen, setDropdownOpen] = useState(false); + const [detailedServer, setDetailedServer] = useState< + ServerDetailed | undefined + >(); const { mutate } = useSWR('getServers'); + // TODO: REMOVE + useEffect(() => { + (async () => { + setDetailedServer(await getServer(server.name)); + })(); + }, []); + return ( <> @@ -56,7 +67,12 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { server.healthStatus === 'Error' ? red[100] : 'secondary.main', p: 1, }} - onClick={() => setDropdownOpen(!dropdownOpen)} + onClick={async () => { + setDropdownOpen(!dropdownOpen); + if (detailedServer === undefined) { + setDetailedServer(await getServer(server.name)); + } + }} > { - + => { } }; +const getServer = async (name: string): Promise => { + try { + const response = await secureFetch(`/api/v1.0/director_ui/servers/${name}`); + if (response.ok) { + return await response.json(); + } else { + return undefined; + } + } catch (e) { + return undefined; + } +}; + export default DirectorCard; diff --git a/web_ui/frontend/app/director/components/DirectorCardList.tsx b/web_ui/frontend/app/director/components/DirectorCardList.tsx index c461f5634..c5600a9ef 100644 --- a/web_ui/frontend/app/director/components/DirectorCardList.tsx +++ b/web_ui/frontend/app/director/components/DirectorCardList.tsx @@ -10,6 +10,7 @@ import { DirectorCard, DirectorCardProps } from './'; import { Server } from '@/index'; import { BooleanToggleButton, CardList } from '@/components'; import useFuse from '@/helpers/useFuse'; +import { ServerGeneral } from '@/types'; interface DirectorCardListProps { data: Partial[]; @@ -88,7 +89,7 @@ export function DirectorCardList({ data, cardProps }: DirectorCardListProps) { ); } -const serverHasError = (server?: Server) => { +const serverHasError = (server?: ServerGeneral) => { return server?.healthStatus === 'Error'; }; diff --git a/web_ui/frontend/app/director/components/DirectorDropdown.tsx b/web_ui/frontend/app/director/components/DirectorDropdown.tsx index 15d7bcb03..8a5bc0e67 100644 --- a/web_ui/frontend/app/director/components/DirectorDropdown.tsx +++ b/web_ui/frontend/app/director/components/DirectorDropdown.tsx @@ -1,17 +1,15 @@ -import { Capabilities, Server, StringTree } from '@/index'; -import { - CapabilitiesChip, - CapabilitiesDisplay, - Dropdown, - InformationSpan, -} from '@/components'; +import { CapabilitiesChip, Dropdown, InformationSpan } from '@/components'; import { Box, Grid, Typography } from '@mui/material'; import DirectoryTree from '@/components/DirectoryTree'; import React from 'react'; import { SinglePointMap } from '@/components/Map'; +import { directoryListToTree } from '@/app/director/components/index'; +import { ServerCapabilitiesTable } from '@/components/ServerCapabilitiesTable'; +import { Capabilities, ServerDetailed, ServerGeneral } from '@/types'; +import { Capability } from '@/components/configuration'; interface DirectorDropdownProps { - server: Server; + server: ServerGeneral | ServerDetailed; transition: boolean; } @@ -20,94 +18,68 @@ export const DirectorDropdown = ({ transition, }: DirectorDropdownProps) => { return ( - - - - - - - - - - - - {transition && ( - - )} - + <> + + + + + + + + + + + + {transition && ( + + )} + + - - {server.capabilities && ( - - + + - )} - - - Namespace Prefixes - - - - + + ); }; -const CapabilitiesRow = ({ capabilities }: { capabilities: Capabilities }) => { +export const CapabilitiesRow = ({ + capabilities, + parentCapabilities, +}: { + capabilities: Capabilities; + parentCapabilities?: Capabilities; +}) => { return ( {Object.entries(capabilities).map(([key, value]) => { + const castKey = key as keyof Capabilities; return ( - + ); })} ); }; - -const directoryListToTree = (directoryList: string[]): StringTree => { - let tree = {}; - directoryList.forEach((directory) => { - const path = directory - .split('/') - .filter((x) => x != '') - .map((x) => '/' + x); - tree = directoryListToTreeHelper(path, tree); - }); - - return tree; -}; - -const directoryListToTreeHelper = ( - path: string[], - tree: StringTree -): true | StringTree => { - if (path.length == 0) { - return true; - } - - if (!tree[path[0]] || tree[path[0]] === true) { - tree[path[0]] = {}; - } - - tree[path[0]] = directoryListToTreeHelper(path.slice(1), tree[path[0]]); - - return tree; -}; diff --git a/web_ui/frontend/app/director/components/index.tsx b/web_ui/frontend/app/director/components/index.tsx index 95b293cba..f01da91a6 100644 --- a/web_ui/frontend/app/director/components/index.tsx +++ b/web_ui/frontend/app/director/components/index.tsx @@ -1,2 +1,34 @@ +import { StringTree } from '@/index'; + export * from './DirectorCard'; export * from './DirectorCardList'; + +export const directoryListToTree = (directoryList: string[]): StringTree => { + let tree = {}; + directoryList.forEach((directory) => { + const path = directory + .split('/') + .filter((x) => x != '') + .map((x) => '/' + x); + tree = directoryListToTreeHelper(path, tree); + }); + + return tree; +}; + +export const directoryListToTreeHelper = ( + path: string[], + tree: StringTree +): true | StringTree => { + if (path.length == 0) { + return true; + } + + if (!tree[path[0]] || tree[path[0]] === true) { + tree[path[0]] = {}; + } + + tree[path[0]] = directoryListToTreeHelper(path.slice(1), tree[path[0]]); + + return tree; +}; diff --git a/web_ui/frontend/app/director/page.tsx b/web_ui/frontend/app/director/page.tsx index 3b1f64c1a..4cc1e3cfb 100644 --- a/web_ui/frontend/app/director/page.tsx +++ b/web_ui/frontend/app/director/page.tsx @@ -21,19 +21,15 @@ import { Box, Grid, Skeleton, Typography } from '@mui/material'; import { useMemo } from 'react'; import useSWR from 'swr'; -import { Server } from '@/index'; -import { - DirectorCardList, - DirectorCard, - DirectorCardProps, -} from './components'; +import { DirectorCardList } from './components'; import { getUser } from '@/helpers/login'; import FederationOverview from '@/components/FederationOverview'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { PaddedContent } from '@/components/layout'; +import { ServerGeneral } from '@/types'; export default function Page() { - const { data } = useSWR('getServers', getServers); + const { data } = useSWR('getServers', getServers); const { data: user, error } = useSWR('getUser', getUser); @@ -99,7 +95,7 @@ const getServers = async () => { let response = await fetch(url); if (response.ok) { - const responseData: Server[] = await response.json(); + const responseData: ServerGeneral[] = await response.json(); responseData.sort((a, b) => a.name.localeCompare(b.name)); return responseData; } diff --git a/web_ui/frontend/components/CapabilitiesDisplay.tsx b/web_ui/frontend/components/CapabilitiesDisplay.tsx index 4907b3e83..d3d82c7f8 100644 --- a/web_ui/frontend/components/CapabilitiesDisplay.tsx +++ b/web_ui/frontend/components/CapabilitiesDisplay.tsx @@ -1,8 +1,8 @@ -import { Capabilities } from '@/index'; +import { Capabilities } from '@/types'; import { Box, Tooltip, Typography } from '@mui/material'; -import { grey } from '@mui/material/colors'; +import { green, grey } from '@mui/material/colors'; import { Check, Clear } from '@mui/icons-material'; -import React from 'react'; +import React, { useMemo } from 'react'; export const CapabilitiesDisplay = ({ capabilities, @@ -14,7 +14,7 @@ export const CapabilitiesDisplay = ({ {Object.entries(capabilities).map(([key, value]) => { return ( - + ); })} @@ -22,13 +22,29 @@ export const CapabilitiesDisplay = ({ ); }; +/** + * Capabilities chip used to convey the capabilities of a server or namespace + * There are two levels of activity to help represent the relationship between + * activity and the server or namespace. + * @param name + * @param value + * @param active + * @constructor + */ export const CapabilitiesChip = ({ name, value, + parentValue, }: { name: string; value: boolean; + parentValue?: boolean; }) => { + // Switch statement to determine the color of the chip + const isActive = useMemo(() => { + return parentValue !== undefined ? value && parentValue : value; + }, [value, parentValue]); + return ( diff --git a/web_ui/frontend/components/DataExportTable.tsx b/web_ui/frontend/components/DataExportTable.tsx index 9b43da0c8..a9a46748c 100644 --- a/web_ui/frontend/components/DataExportTable.tsx +++ b/web_ui/frontend/components/DataExportTable.tsx @@ -20,7 +20,7 @@ import { Skeleton } from '@mui/material'; import { Edit, Settings, Check, Clear } from '@mui/icons-material'; import useSWR from 'swr'; import { getErrorMessage } from '@/helpers/util'; -import type { Capabilities } from '@/index'; +import { Capabilities } from '@/types'; import { CapabilitiesDisplay } from '@/components'; type RegistrationStatus = diff --git a/web_ui/frontend/components/DirectoryTree.tsx b/web_ui/frontend/components/DirectoryTree.tsx index e25c2f36a..a77d4e11a 100644 --- a/web_ui/frontend/components/DirectoryTree.tsx +++ b/web_ui/frontend/components/DirectoryTree.tsx @@ -10,8 +10,6 @@ export const DirectoryTree = ({ data }: { data: StringTree }) => { const [selectedItems, setSelectedItems] = useState([]); const handleSelect = (ids: string[]) => { - console.log(ids, calculateSelectedItems(ids[0])); - setSelectedItems(calculateSelectedItems(ids[0])); }; @@ -30,6 +28,7 @@ const CustomTreeItemSmall = ({ ...props }: TreeItemProps) => { return ( {props.label}} /> ); diff --git a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx new file mode 100644 index 000000000..6c5a2776a --- /dev/null +++ b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx @@ -0,0 +1,3 @@ +/** + * A table to display the capabilities of a namespace + */ diff --git a/web_ui/frontend/components/ServerCapabilitiesTable.tsx b/web_ui/frontend/components/ServerCapabilitiesTable.tsx new file mode 100644 index 000000000..1800406cd --- /dev/null +++ b/web_ui/frontend/components/ServerCapabilitiesTable.tsx @@ -0,0 +1,74 @@ +/** + * Table to display the server capabilities with its namespaces + */ + +import { ServerDetailed, ServerGeneral } from '@/types'; +import { Box, Grid, Typography } from '@mui/material'; +import { CapabilitiesRow } from '@/app/director/components/DirectorDropdown'; +import { grey } from '@mui/material/colors'; + +interface ServerCapabilitiesTableProps { + server: ServerGeneral | ServerDetailed; +} + +/** + * Create a grid table that displays the server capabilities with the namespaces + * listed below indicating their individual capabilities and how they interact + * with the servers own capabilities. + * @param server + * @constructor + */ +export const ServerCapabilitiesTable = ({ + server, +}: ServerCapabilitiesTableProps) => { + return ( + + + + + + + + {server.type}'s Namespace Capabilities + + + + + + + + + + {'namespaces' in server && + server?.namespaces + ?.sort((a, b) => a.path.localeCompare(b.path)) + ?.map((namespace) => ( + + + + + + + {namespace.path} + + + + + + + + + + ))} + + ); +}; diff --git a/web_ui/frontend/index.d.ts b/web_ui/frontend/index.d.ts index 7be168325..5ddf21d9a 100644 --- a/web_ui/frontend/index.d.ts +++ b/web_ui/frontend/index.d.ts @@ -1,4 +1,5 @@ import { NamespaceAdminMetadata } from '@/components/Namespace'; +import { Capabilities } from '@/types'; export interface User { authenticated: boolean; @@ -26,14 +27,6 @@ export interface Server { namespacePrefixes: string[]; } -export interface Capabilities { - PublicReads: boolean; - Reads: boolean; - Writes: boolean; - Listings: boolean; - DirectReads: boolean; -} - export type StringTree = Record; interface Alert { diff --git a/web_ui/frontend/types.ts b/web_ui/frontend/types.ts new file mode 100644 index 000000000..cfcf71d66 --- /dev/null +++ b/web_ui/frontend/types.ts @@ -0,0 +1,55 @@ +export interface Capabilities { + PublicRead: boolean; + Read: boolean; + Write: boolean; + Listing: boolean; + FallBackRead: boolean; +} + +export interface TokenGeneration { + strategy: string; + vaultServer: string; + maxScopeDepth: number; + issuer: string; +} + +export interface TokenIssuer { + basePaths: string[]; + restrictedPaths: string[] | null; + issuer: string; +} + +export interface Namespace { + path: string; + capabilities: Capabilities; + tokenGeneration: TokenGeneration[]; + tokenIssuer: TokenIssuer[]; + fromTopology: boolean; +} + +interface ServerBase { + name: string; + storageType: string; + disableDirectorTest: boolean; + authUrl: string; + brokerUrl: string; + url: string; + webUrl: string; + type: string; + latitude: number; + longitude: number; + capabilities: Capabilities; + filtered: boolean; + filteredType: string; + fromTopology: boolean; + healthStatus: string; + ioLoad: number; +} + +export interface ServerDetailed extends ServerBase { + namespaces: Namespace[]; +} + +export interface ServerGeneral extends ServerBase { + namespacePrefixes: string[]; +} From 595b1483066a78e384456d51a11b3cd32cecbd8c Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Nov 2024 16:21:50 +0000 Subject: [PATCH 12/86] Remove unnecessary metric collection call --- director/director.go | 1 - 1 file changed, 1 deletion(-) diff --git a/director/director.go b/director/director.go index 07dd24cb9..f8f8f6344 100644 --- a/director/director.go +++ b/director/director.go @@ -344,7 +344,6 @@ func redirectToCache(ginCtx *gin.Context) { collectDirectorRedirectionMetric(ginCtx, "cache") } }() - defer collectDirectorRedirectionMetric(ginCtx, "cache") err := versionCompatCheck(reqVer, service) if err != nil { log.Warningf("A version incompatibility was encountered while redirecting to a cache and no response was served: %v", err) From 668cff464a444acf86156283730fd546ffc4faaf Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Fri, 15 Nov 2024 12:49:13 -0600 Subject: [PATCH 13/86] Pass access_token to namespace getter - Fixes register buttons for non admin users --- web_ui/frontend/app/registry/components/PutPage.tsx | 12 +++++++----- web_ui/frontend/app/registry/components/util.tsx | 6 +++++- .../components/layout/AuthenticatedContent.tsx | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index b5e1ebce1..772ad14a5 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -53,6 +53,7 @@ const PutPage = ({ update }: NamespaceFormPage) => { const urlParams = new URLSearchParams(window.location.search); const id = urlParams.get('id'); const fromUrl = urlParams.get('fromUrl'); + const accessToken = urlParams.get('access_token'); if (id === null) { setAlert({ severity: 'error', message: 'No Namespace ID Provided' }); @@ -73,19 +74,20 @@ const PutPage = ({ update }: NamespaceFormPage) => { } catch (e) { setAlert({ severity: 'error', message: 'Invalid Namespace ID Provided' }); } - }, []); - useEffect(() => { (async () => { if (id !== undefined) { try { - setNamespace(await getNamespace(id)); + setNamespace(await getNamespace(id, accessToken || undefined)); } catch (e) { - setAlert({ severity: 'error', message: e as string }); + if (e instanceof Error) { + setAlert({ severity: 'error', message: e.message }); + } + setAlert({ severity: 'error', message: "Could not fetch namespace" }); } } })(); - }, [id]); + }, []); return ( diff --git a/web_ui/frontend/app/registry/components/util.tsx b/web_ui/frontend/app/registry/components/util.tsx index 0eb3398ae..04cee3097 100644 --- a/web_ui/frontend/app/registry/components/util.tsx +++ b/web_ui/frontend/app/registry/components/util.tsx @@ -94,12 +94,16 @@ export const namespaceToOrigin = (data: Namespace) => { }; export const getNamespace = async ( - id: string | number + id: string | number, + accessToken?: string ): Promise => { const url = new URL( `/api/v1.0/registry_ui/namespaces/${id}`, window.location.origin ); + if (accessToken) { + url.searchParams.append('access_token', accessToken); + } const response = await fetch(url); if (response.ok) { return await response.json(); diff --git a/web_ui/frontend/components/layout/AuthenticatedContent.tsx b/web_ui/frontend/components/layout/AuthenticatedContent.tsx index 5eef49bcc..5465a7d57 100644 --- a/web_ui/frontend/components/layout/AuthenticatedContent.tsx +++ b/web_ui/frontend/components/layout/AuthenticatedContent.tsx @@ -69,7 +69,7 @@ const AuthenticatedContent = ({ if (data && checkAuthentication) { return checkAuthentication(data); } else { - return data?.authenticated !== undefined; + return !!data?.authenticated; } }, [data, checkAuthentication]); From 40e6678a247589eb3a9e4cbdae839d8305a7d594 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Fri, 15 Nov 2024 12:54:01 -0600 Subject: [PATCH 14/86] Pass access_token to namespace getter - Fixes register buttons for non admin users --- web_ui/frontend/app/registry/components/PutPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index 772ad14a5..06781796b 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -83,7 +83,7 @@ const PutPage = ({ update }: NamespaceFormPage) => { if (e instanceof Error) { setAlert({ severity: 'error', message: e.message }); } - setAlert({ severity: 'error', message: "Could not fetch namespace" }); + setAlert({ severity: 'error', message: 'Could not fetch namespace' }); } } })(); From a12a2f3ac6236b669b8386a8d8da3d7bed255d83 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Fri, 15 Nov 2024 14:18:24 -0600 Subject: [PATCH 15/86] Pass access_token to namespace getter - Fixes register buttons for non admin users --- web_ui/frontend/app/registry/components/util.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/web_ui/frontend/app/registry/components/util.tsx b/web_ui/frontend/app/registry/components/util.tsx index 04cee3097..4477f9c5c 100644 --- a/web_ui/frontend/app/registry/components/util.tsx +++ b/web_ui/frontend/app/registry/components/util.tsx @@ -128,8 +128,16 @@ export const postGeneralNamespace = async ( export const putGeneralNamespace = async ( data: Namespace ): Promise => { + + // If an access_token is in the URL, add it to the request + const url = new URL(`/api/v1.0/registry_ui/namespaces/${data.id}`, window.location.origin); + const accessToken = new URLSearchParams(window.location.search).get('access_token'); + if (accessToken) { + url.searchParams.append('access_token', accessToken); + } + return await handleRequestAlert( - `/api/v1.0/registry_ui/namespaces/${data.id}`, + url.toString(), { body: JSON.stringify(data), method: 'PUT', From f6fa24f7856ceddfa848bfe4e7a831d76b0d14e6 Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Thu, 14 Nov 2024 20:53:27 +0000 Subject: [PATCH 16/86] Set default value for empty stringSlice to [] in docs --- docs/parameters.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 47e263d9c..028a661d2 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -44,7 +44,7 @@ description: |+ Subdirectories of the provided directories are not read. Only the root config file's `ConfigLocations` key is used, and any redefinitions are ignored. type: stringSlice -default: none +default: [] components: ["*"] --- name: Debug @@ -566,7 +566,7 @@ description: |+ This configuration is meant mostly to be used by passing the -v flag from the command line. Paths exported with this configuration will inherit the origin's abilities, so individual export configurations are not possible. type: stringSlice -default: none +default: [] components: ["origin"] --- name: Origin.EnablePublicReads @@ -765,7 +765,7 @@ name: Origin.ScitokensRestrictedPaths description: |+ Enable the built-in issuer daemon for the origin. type: stringSlice -default: none +default: [] components: ["origin"] --- name: Origin.ScitokensMapSubject @@ -1210,7 +1210,7 @@ description: |+ the cache is allowed to access any namespace that's advertised to the director. Otherwise, it will only be allowed to access the listed namespaces. type: stringSlice -default: none +default: [] components: ["cache"] --- name: Cache.SelfTest @@ -1285,7 +1285,7 @@ description: |+ If present, the hostname is taken from the X-Forwarded-Host header in the request. Otherwise, Host is used. type: stringSlice -default: none +default: [] components: ["director"] --- name: Director.CacheSortMethod @@ -1314,7 +1314,7 @@ description: |+ If present, the hostname is taken from the X-Forwarded-Host header in the request. Otherwise, Host is used. type: stringSlice -default: none +default: [] components: ["director"] --- name: Director.MaxMindKeyFile @@ -1447,7 +1447,7 @@ description: |+ A list of server resource names that the Director should consider in downtime, preventing the Director from issuing redirects to them. Additional downtimes are aggregated from Topology (when the Director is served in OSDF mode), and the Web UI. type: stringSlice -default: none +default: [] components: ["director"] --- name: Director.SupportContactEmail @@ -1489,7 +1489,7 @@ description: |+ This setting allows for compatibility with specific legacy OSDF origins and is not needed for new origins. type: stringSlice -default: none +default: [] components: ["director"] hidden: true --- @@ -1852,7 +1852,7 @@ description: |+ The "subject" claim should be the "CILogon User Identifier" from CILogon user page: https://cilogon.org/ type: stringSlice -default: none +default: [] components: ["registry","origin","cache"] --- name: Server.StartupTimeout @@ -2477,7 +2477,7 @@ name: Shoveler.OutputDestinations description: |+ A list of destinations to forward XRootD monitoring packet to. type: stringSlice -default: none +default: [] components: ["origin", "cache"] --- name: Shoveler.VerifyHeader From 48e18b5a3271d4290830dd8b6133772fc53b7ffe Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Mon, 18 Nov 2024 16:18:21 +0000 Subject: [PATCH 17/86] Fixes an issue where a misplaced `defer ticker.Stop()` stopped the re-advertisement of caches/origins --- launcher_utils/advertise.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/launcher_utils/advertise.go b/launcher_utils/advertise.go index 6daddad55..4ca803ccd 100644 --- a/launcher_utils/advertise.go +++ b/launcher_utils/advertise.go @@ -68,9 +68,8 @@ func LaunchPeriodicAdvertise(ctx context.Context, egrp *errgroup.Group, servers doAdvertise(ctx, servers) ticker := time.NewTicker(1 * time.Minute) - defer ticker.Stop() egrp.Go(func() error { - + defer ticker.Stop() for { select { case <-ticker.C: From 1081270636ac56481740bd676f1a943efb7ddee8 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Mon, 18 Nov 2024 13:22:12 -0600 Subject: [PATCH 18/86] Redirect app to root on success password --- web_ui/frontend/app/(login)/initialization/password/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_ui/frontend/app/(login)/initialization/password/page.tsx b/web_ui/frontend/app/(login)/initialization/password/page.tsx index a35e18c3b..78027d89c 100644 --- a/web_ui/frontend/app/(login)/initialization/password/page.tsx +++ b/web_ui/frontend/app/(login)/initialization/password/page.tsx @@ -46,7 +46,7 @@ export default function Home() { dispatch ); if (response) { - router.push('../password/'); + router.push('/'); } else { setLoading(false); } From 78ae774325b103bab57ece29e8bed1ac58be1241 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Mon, 18 Nov 2024 15:40:21 -0600 Subject: [PATCH 19/86] Add Namespaces to the director UI - Add namespace card and capabilities table to director UI - Design InformationSpanHeader --- .../app/director/components/DirectorCard.tsx | 7 -- .../app/director/components/NamespaceCard.tsx | 84 +++++++++++++++++++ .../director/components/NamespaceCardList.tsx | 33 ++++++++ .../director/components/NamespaceDropdown.tsx | 60 +++++++++++++ .../app/director/components/index.tsx | 1 + web_ui/frontend/app/director/page.tsx | 33 +++++++- web_ui/frontend/app/origin/issuer/Issuer.tsx | 1 + web_ui/frontend/components/CardList.tsx | 2 +- .../frontend/components/InformationSpan.tsx | 38 ++++++++- .../components/NamespaceCapabilitiesTable.tsx | 77 +++++++++++++++++ web_ui/frontend/types.ts | 4 +- 11 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 web_ui/frontend/app/director/components/NamespaceCard.tsx create mode 100644 web_ui/frontend/app/director/components/NamespaceCardList.tsx create mode 100644 web_ui/frontend/app/director/components/NamespaceDropdown.tsx diff --git a/web_ui/frontend/app/director/components/DirectorCard.tsx b/web_ui/frontend/app/director/components/DirectorCard.tsx index 36de1b8d3..57329a6b5 100644 --- a/web_ui/frontend/app/director/components/DirectorCard.tsx +++ b/web_ui/frontend/app/director/components/DirectorCard.tsx @@ -41,13 +41,6 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { const { mutate } = useSWR('getServers'); - // TODO: REMOVE - useEffect(() => { - (async () => { - setDetailedServer(await getServer(server.name)); - })(); - }, []); - return ( <> diff --git a/web_ui/frontend/app/director/components/NamespaceCard.tsx b/web_ui/frontend/app/director/components/NamespaceCard.tsx new file mode 100644 index 000000000..b12706b83 --- /dev/null +++ b/web_ui/frontend/app/director/components/NamespaceCard.tsx @@ -0,0 +1,84 @@ +import { secureFetch } from '@/helpers/login'; +import React, { useState } from 'react'; +import { + Box, + Paper, + Typography, +} from '@mui/material'; +import { NamespaceIcon } from '@/components/Namespace/index'; +import { NamespaceDropdown } from './NamespaceDropdown'; +import { Namespace, ServerDetailed, ServerGeneral } from '@/types'; + +export interface NamespaceCardProps { + namespace: Namespace; +} + +export const NamespaceCard = ({ namespace }: NamespaceCardProps) => { + const [dropdownOpen, setDropdownOpen] = useState(false); + const [servers, setServers] = useState(undefined); + + return ( + <> + + { + setDropdownOpen(!dropdownOpen); + if (servers === undefined) { + setServers(await getAssociatedServers(namespace)); + } + }} + > + + + {namespace.path} + + + + + + ); +}; + +const getAssociatedServers = async (namespace: Namespace) => { + const servers = await Promise.all([...namespace.origins, ...namespace.caches].map(getServer)); + + // Alert the console if any servers are undefined, as this is unlikely to happen naturally + if(servers.some((s) => s === undefined)) { + console.error("Failed to fetch all servers, some are undefined"); + } + + return servers.filter((s) => s !== undefined) as ServerDetailed[]; + +} + +// TODO: Consolidate this when https://github.com/PelicanPlatform/pelican/pull/1687 is merged +const getServer = async (name: string): Promise => { + try { + const response = await secureFetch(`/api/v1.0/director_ui/servers/${name}`); + if (response.ok) { + return await response.json(); + } else { + return undefined; + } + } catch (e) { + return undefined; + } +}; + +export default NamespaceCard; diff --git a/web_ui/frontend/app/director/components/NamespaceCardList.tsx b/web_ui/frontend/app/director/components/NamespaceCardList.tsx new file mode 100644 index 000000000..e6580d112 --- /dev/null +++ b/web_ui/frontend/app/director/components/NamespaceCardList.tsx @@ -0,0 +1,33 @@ +import React, { + useState +} from 'react'; +import { Box, TextField } from '@mui/material'; +import { NamespaceCard, NamespaceCardProps } from './'; +import { CardList } from '@/components'; +import useFuse from '@/helpers/useFuse'; + +interface NamespaceCardListProps { + data?: Partial[]; +} + +export function NamespaceCardList({ data }: NamespaceCardListProps) { + const [search, setSearch] = useState(''); + + const searchedData = useFuse>(data || [], search); + + return ( + + + setSearch(e.target.value)} + label='Search' + /> + + + + ); +} + +export default NamespaceCardList; diff --git a/web_ui/frontend/app/director/components/NamespaceDropdown.tsx b/web_ui/frontend/app/director/components/NamespaceDropdown.tsx new file mode 100644 index 000000000..7400a2365 --- /dev/null +++ b/web_ui/frontend/app/director/components/NamespaceDropdown.tsx @@ -0,0 +1,60 @@ +import { Dropdown, InformationSpan, InformationSpanHeader } from '@/components'; +import { Box, Grid } from '@mui/material'; +import React, { Fragment } from 'react'; +import { Namespace, ServerDetailed } from '@/types'; +import { NamespaceCapabilitiesTable } from '@/components/NamespaceCapabilitiesTable'; + +interface NamespaceDropdownProps { + namespace: Namespace; + servers?: ServerDetailed[]; + transition: boolean; +} + +export const NamespaceDropdown = ({ + namespace, + servers, + transition +}: NamespaceDropdownProps) => { + return ( + <> + + + + + + {namespace.tokenGeneration.map((tg) => + + + + + + + )} + + {namespace.tokenIssuer.map((ti) => + + + + {ti.basePaths.map((bp) => + + )} + { ti.restrictedPaths && ( + <> + + {ti.restrictedPaths?.map((rp) => + + )} + + ) + } + + )} + + + + + + + + ); +}; diff --git a/web_ui/frontend/app/director/components/index.tsx b/web_ui/frontend/app/director/components/index.tsx index f01da91a6..8c4152f3f 100644 --- a/web_ui/frontend/app/director/components/index.tsx +++ b/web_ui/frontend/app/director/components/index.tsx @@ -2,6 +2,7 @@ import { StringTree } from '@/index'; export * from './DirectorCard'; export * from './DirectorCardList'; +export * from './NamespaceCard'; export const directoryListToTree = (directoryList: string[]): StringTree => { let tree = {}; diff --git a/web_ui/frontend/app/director/page.tsx b/web_ui/frontend/app/director/page.tsx index 4cc1e3cfb..78f8bec42 100644 --- a/web_ui/frontend/app/director/page.tsx +++ b/web_ui/frontend/app/director/page.tsx @@ -26,10 +26,12 @@ import { getUser } from '@/helpers/login'; import FederationOverview from '@/components/FederationOverview'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { PaddedContent } from '@/components/layout'; -import { ServerGeneral } from '@/types'; +import { Namespace, ServerGeneral } from '@/types'; +import { NamespaceCardList } from './components/NamespaceCardList'; export default function Page() { const { data } = useSWR('getServers', getServers); + const { data: namespaces } = useSWR('getNamespaces', getNamespaces); const { data: user, error } = useSWR('getUser', getUser); @@ -79,6 +81,22 @@ export default function Page() { )} + + + Namespaces + + {cacheData ? ( + { + return {namespace} + }) || []} + /> + ) : ( + + + + )} + @@ -102,3 +120,16 @@ const getServers = async () => { throw new Error('Failed to fetch servers'); }; + +const getNamespaces = async () => { + const url = new URL('/api/v1.0/director_ui/namespaces', window.location.origin); + + let response = await fetch(url); + if (response.ok) { + const responseData: Namespace[] = await response.json(); + responseData.sort((a, b) => a.path.localeCompare(b.path)); + return responseData; + } + + throw new Error('Failed to fetch servers'); +} diff --git a/web_ui/frontend/app/origin/issuer/Issuer.tsx b/web_ui/frontend/app/origin/issuer/Issuer.tsx index 51921c646..66bdbf6ad 100644 --- a/web_ui/frontend/app/origin/issuer/Issuer.tsx +++ b/web_ui/frontend/app/origin/issuer/Issuer.tsx @@ -75,6 +75,7 @@ export function Issuer({ metadata }: { metadata: ParameterMetadataRecord }) { const configView = useMemo(() => { return merge(structuredClone(serverConfig), structuredClone(patch)); }, [serverConfig, patch]); + const submitPatch = useCallback(async (patch: any) => { setStatus({ message: 'Submitting', severity: 'info' }); diff --git a/web_ui/frontend/components/CardList.tsx b/web_ui/frontend/components/CardList.tsx index c4be45577..5934e822a 100644 --- a/web_ui/frontend/components/CardList.tsx +++ b/web_ui/frontend/components/CardList.tsx @@ -17,7 +17,7 @@ import { interface CardListProps { data?: Partial[]; Card: ComponentType; - cardProps: Partial; + cardProps?: Partial; } export function CardList({ data, Card, cardProps }: CardListProps) { diff --git a/web_ui/frontend/components/InformationSpan.tsx b/web_ui/frontend/components/InformationSpan.tsx index 745f9602d..bf2d55ab7 100644 --- a/web_ui/frontend/components/InformationSpan.tsx +++ b/web_ui/frontend/components/InformationSpan.tsx @@ -1,13 +1,46 @@ import { Box, Tooltip, Typography } from '@mui/material'; import { grey } from '@mui/material/colors'; -import React from 'react'; +import React, { ReactNode } from 'react'; + +export const InformationSpanHeader = ({ + title, + indent = 0 +}: { + title: string, + indent?: number +}) => { + return ( + + + {"\u00A0\u00A0\u00A0\u00A0".repeat(Math.max(indent - 1, 0))}{indent > 0 ? "↳\u00A0" : ""}{title} + + + + + ); +} export const InformationSpan = ({ name, value, + indent = 0 }: { name: string; value: string; + indent?: number; }) => { return ( @@ -22,11 +55,10 @@ export const InformationSpan = ({ p: '4px 6px', }, display: 'flex', - justifyContent: 'space-between', }} > - {name} + {"\u00A0\u00A0\u00A0\u00A0".repeat(Math.max(indent - 1, 0))}{indent > 0 ? "↳\u00A0" : ""}{name}: {value} diff --git a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx index 6c5a2776a..263ea656a 100644 --- a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx +++ b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx @@ -1,3 +1,80 @@ /** * A table to display the capabilities of a namespace */ + +/** + * Table to display the server capabilities with its namespaces + */ + +import { Namespace, ServerDetailed, ServerGeneral } from '@/types'; +import { Box, Grid, Typography } from '@mui/material'; +import { CapabilitiesRow } from '@/app/director/components/DirectorDropdown'; +import { grey } from '@mui/material/colors'; + +interface NamespaceCapabilitiesTableProps { + namespace: Namespace + servers?: ServerDetailed[] +} + +/** + * Create a grid table that displays the server capabilities with the namespaces + * listed below indicating their individual capabilities and how they interact + * with the servers own capabilities. + * @param server + * @constructor + */ +export const NamespaceCapabilitiesTable = ({ + namespace, + servers +}: NamespaceCapabilitiesTableProps) => { + return ( + + + + + + + + {namespace.path} Capabilities + + + + + + + + + + {servers && + servers + ?.sort((a, b) => a.name.localeCompare(b.name)) + ?.map((server) => ( + + + + + + + {server.name} + + + + + + + + + + ))} + + ); +}; diff --git a/web_ui/frontend/types.ts b/web_ui/frontend/types.ts index cfcf71d66..6577d7fed 100644 --- a/web_ui/frontend/types.ts +++ b/web_ui/frontend/types.ts @@ -25,6 +25,8 @@ export interface Namespace { tokenGeneration: TokenGeneration[]; tokenIssuer: TokenIssuer[]; fromTopology: boolean; + caches: string[]; + origins: string[]; } interface ServerBase { @@ -35,7 +37,7 @@ interface ServerBase { brokerUrl: string; url: string; webUrl: string; - type: string; + type: 'Origin' | 'Cache'; latitude: number; longitude: number; capabilities: Capabilities; From 0ac23a21ce326b3cca2f8af08936ae1b93165687 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Mon, 18 Nov 2024 16:13:14 -0600 Subject: [PATCH 20/86] Add Namespaces to the director UI - Add iconography to the server list to indicate type --- .../director/components/NamespaceDropdown.tsx | 4 +- .../components/Namespace/NamespaceIcon.tsx | 70 ++++++++++++++----- .../components/NamespaceCapabilitiesTable.tsx | 11 ++- web_ui/frontend/types.ts | 4 +- 4 files changed, 65 insertions(+), 24 deletions(-) diff --git a/web_ui/frontend/app/director/components/NamespaceDropdown.tsx b/web_ui/frontend/app/director/components/NamespaceDropdown.tsx index 7400a2365..35b2276a5 100644 --- a/web_ui/frontend/app/director/components/NamespaceDropdown.tsx +++ b/web_ui/frontend/app/director/components/NamespaceDropdown.tsx @@ -22,7 +22,7 @@ export const NamespaceDropdown = ({ - {namespace.tokenGeneration.map((tg) => + {namespace.tokenGeneration?.map((tg) => @@ -31,7 +31,7 @@ export const NamespaceDropdown = ({ )} - {namespace.tokenIssuer.map((ti) => + {namespace.tokenIssuer?.map((ti) => diff --git a/web_ui/frontend/components/Namespace/NamespaceIcon.tsx b/web_ui/frontend/components/Namespace/NamespaceIcon.tsx index 051098af2..9fae82c1f 100644 --- a/web_ui/frontend/components/Namespace/NamespaceIcon.tsx +++ b/web_ui/frontend/components/Namespace/NamespaceIcon.tsx @@ -1,66 +1,102 @@ import { Avatar, Box, Tooltip } from '@mui/material'; import { FolderOpen, Storage, TripOrigin } from '@mui/icons-material'; -import React from 'react'; +import React, { useMemo } from 'react'; const NamespaceIcon = ({ - serverType: prefixType, + serverType, + size, + color = 'white', + bgcolor = 'primary.main', }: { serverType: 'origin' | 'cache' | 'namespace'; + size?: 'large' | 'medium' | 'small'; + color?: string; + bgcolor?: string; }) => { - if (prefixType == 'namespace') { + + const avatarPixelSize = useMemo(() => { + switch (size) { + case 'large': + return 50; + case 'medium': + return 30; + case 'small': + return 20; + default: + return 30; + } + }, [size]) + + const iconPixelSize = useMemo(() => { + switch (size) { + case 'large': + return 30; + case 'medium': + return 24; + case 'small': + return 15; + default: + return 24; + } + }, []) + + if (serverType == 'namespace') { return ( - + ); } - if (prefixType == 'origin') { + if (serverType == 'origin') { return ( - + ); } - if (prefixType == 'cache') { + if (serverType == 'cache') { return ( - + diff --git a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx index 263ea656a..b7cb70501 100644 --- a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx +++ b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx @@ -7,9 +7,10 @@ */ import { Namespace, ServerDetailed, ServerGeneral } from '@/types'; -import { Box, Grid, Typography } from '@mui/material'; +import { Box, Grid, Typography, useTheme } from '@mui/material'; import { CapabilitiesRow } from '@/app/director/components/DirectorDropdown'; import { grey } from '@mui/material/colors'; +import { NamespaceIcon } from '@/components/Namespace'; interface NamespaceCapabilitiesTableProps { namespace: Namespace @@ -27,6 +28,9 @@ export const NamespaceCapabilitiesTable = ({ namespace, servers }: NamespaceCapabilitiesTableProps) => { + + const theme = useTheme(); + return ( @@ -41,7 +45,7 @@ export const NamespaceCapabilitiesTable = ({ - {namespace.path} Capabilities + Namespace Capabilities @@ -60,7 +64,8 @@ export const NamespaceCapabilitiesTable = ({ - + + {server.name} diff --git a/web_ui/frontend/types.ts b/web_ui/frontend/types.ts index 6577d7fed..f1b461b26 100644 --- a/web_ui/frontend/types.ts +++ b/web_ui/frontend/types.ts @@ -22,8 +22,8 @@ export interface TokenIssuer { export interface Namespace { path: string; capabilities: Capabilities; - tokenGeneration: TokenGeneration[]; - tokenIssuer: TokenIssuer[]; + tokenGeneration: TokenGeneration[] | null; + tokenIssuer: TokenIssuer[] | null; fromTopology: boolean; caches: string[]; origins: string[]; From 483dbf020876fe1ea93d0a51d43b73d499ddb0f4 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 19 Nov 2024 08:40:22 -0600 Subject: [PATCH 21/86] Run prettier --- .../app/director/components/NamespaceCard.tsx | 25 ++++---- .../director/components/NamespaceCardList.tsx | 4 +- .../director/components/NamespaceDropdown.tsx | 62 +++++++++++++------ web_ui/frontend/app/director/page.tsx | 20 ++++-- .../frontend/components/InformationSpan.tsx | 21 ++++--- .../components/Namespace/NamespaceIcon.tsx | 17 +++-- .../components/NamespaceCapabilitiesTable.tsx | 22 +++++-- 7 files changed, 105 insertions(+), 66 deletions(-) diff --git a/web_ui/frontend/app/director/components/NamespaceCard.tsx b/web_ui/frontend/app/director/components/NamespaceCard.tsx index b12706b83..7e8215f43 100644 --- a/web_ui/frontend/app/director/components/NamespaceCard.tsx +++ b/web_ui/frontend/app/director/components/NamespaceCard.tsx @@ -1,10 +1,6 @@ import { secureFetch } from '@/helpers/login'; import React, { useState } from 'react'; -import { - Box, - Paper, - Typography, -} from '@mui/material'; +import { Box, Paper, Typography } from '@mui/material'; import { NamespaceIcon } from '@/components/Namespace/index'; import { NamespaceDropdown } from './NamespaceDropdown'; import { Namespace, ServerDetailed, ServerGeneral } from '@/types'; @@ -15,7 +11,9 @@ export interface NamespaceCardProps { export const NamespaceCard = ({ namespace }: NamespaceCardProps) => { const [dropdownOpen, setDropdownOpen] = useState(false); - const [servers, setServers] = useState(undefined); + const [servers, setServers] = useState( + undefined + ); return ( <> @@ -39,9 +37,7 @@ export const NamespaceCard = ({ namespace }: NamespaceCardProps) => { }} > - + {namespace.path} @@ -56,16 +52,17 @@ export const NamespaceCard = ({ namespace }: NamespaceCardProps) => { }; const getAssociatedServers = async (namespace: Namespace) => { - const servers = await Promise.all([...namespace.origins, ...namespace.caches].map(getServer)); + const servers = await Promise.all( + [...namespace.origins, ...namespace.caches].map(getServer) + ); // Alert the console if any servers are undefined, as this is unlikely to happen naturally - if(servers.some((s) => s === undefined)) { - console.error("Failed to fetch all servers, some are undefined"); + if (servers.some((s) => s === undefined)) { + console.error('Failed to fetch all servers, some are undefined'); } return servers.filter((s) => s !== undefined) as ServerDetailed[]; - -} +}; // TODO: Consolidate this when https://github.com/PelicanPlatform/pelican/pull/1687 is merged const getServer = async (name: string): Promise => { diff --git a/web_ui/frontend/app/director/components/NamespaceCardList.tsx b/web_ui/frontend/app/director/components/NamespaceCardList.tsx index e6580d112..a13a20fad 100644 --- a/web_ui/frontend/app/director/components/NamespaceCardList.tsx +++ b/web_ui/frontend/app/director/components/NamespaceCardList.tsx @@ -1,6 +1,4 @@ -import React, { - useState -} from 'react'; +import React, { useState } from 'react'; import { Box, TextField } from '@mui/material'; import { NamespaceCard, NamespaceCardProps } from './'; import { CardList } from '@/components'; diff --git a/web_ui/frontend/app/director/components/NamespaceDropdown.tsx b/web_ui/frontend/app/director/components/NamespaceDropdown.tsx index 35b2276a5..7ebad9872 100644 --- a/web_ui/frontend/app/director/components/NamespaceDropdown.tsx +++ b/web_ui/frontend/app/director/components/NamespaceDropdown.tsx @@ -13,7 +13,7 @@ interface NamespaceDropdownProps { export const NamespaceDropdown = ({ namespace, servers, - transition + transition, }: NamespaceDropdownProps) => { return ( <> @@ -22,33 +22,57 @@ export const NamespaceDropdown = ({ - {namespace.tokenGeneration?.map((tg) => + {namespace.tokenGeneration?.map((tg) => ( - - - + + + - )} + ))} - {namespace.tokenIssuer?.map((ti) => + {namespace.tokenIssuer?.map((ti) => ( - - {ti.basePaths.map((bp) => - - )} - { ti.restrictedPaths && ( + + {ti.basePaths.map((bp) => ( + + ))} + {ti.restrictedPaths && ( <> - - {ti.restrictedPaths?.map((rp) => - - )} + + {ti.restrictedPaths?.map((rp) => ( + + ))} - ) - } + )} - )} + ))} diff --git a/web_ui/frontend/app/director/page.tsx b/web_ui/frontend/app/director/page.tsx index 78f8bec42..7d88242b4 100644 --- a/web_ui/frontend/app/director/page.tsx +++ b/web_ui/frontend/app/director/page.tsx @@ -31,7 +31,10 @@ import { NamespaceCardList } from './components/NamespaceCardList'; export default function Page() { const { data } = useSWR('getServers', getServers); - const { data: namespaces } = useSWR('getNamespaces', getNamespaces); + const { data: namespaces } = useSWR( + 'getNamespaces', + getNamespaces + ); const { data: user, error } = useSWR('getUser', getUser); @@ -87,9 +90,11 @@ export default function Page() { {cacheData ? ( { - return {namespace} - }) || []} + data={ + namespaces?.map((namespace) => { + return { namespace }; + }) || [] + } /> ) : ( @@ -122,7 +127,10 @@ const getServers = async () => { }; const getNamespaces = async () => { - const url = new URL('/api/v1.0/director_ui/namespaces', window.location.origin); + const url = new URL( + '/api/v1.0/director_ui/namespaces', + window.location.origin + ); let response = await fetch(url); if (response.ok) { @@ -132,4 +140,4 @@ const getNamespaces = async () => { } throw new Error('Failed to fetch servers'); -} +}; diff --git a/web_ui/frontend/components/InformationSpan.tsx b/web_ui/frontend/components/InformationSpan.tsx index bf2d55ab7..46ddd89b4 100644 --- a/web_ui/frontend/components/InformationSpan.tsx +++ b/web_ui/frontend/components/InformationSpan.tsx @@ -4,10 +4,10 @@ import React, { ReactNode } from 'react'; export const InformationSpanHeader = ({ title, - indent = 0 + indent = 0, }: { - title: string, - indent?: number + title: string; + indent?: number; }) => { return ( - {"\u00A0\u00A0\u00A0\u00A0".repeat(Math.max(indent - 1, 0))}{indent > 0 ? "↳\u00A0" : ""}{title} - - + {'\u00A0\u00A0\u00A0\u00A0'.repeat(Math.max(indent - 1, 0))} + {indent > 0 ? '↳\u00A0' : ''} + {title} + ); -} +}; export const InformationSpan = ({ name, value, - indent = 0 + indent = 0, }: { name: string; value: string; @@ -58,7 +59,9 @@ export const InformationSpan = ({ }} > - {"\u00A0\u00A0\u00A0\u00A0".repeat(Math.max(indent - 1, 0))}{indent > 0 ? "↳\u00A0" : ""}{name}: + {'\u00A0\u00A0\u00A0\u00A0'.repeat(Math.max(indent - 1, 0))} + {indent > 0 ? '↳\u00A0' : ''} + {name}: {value} diff --git a/web_ui/frontend/components/Namespace/NamespaceIcon.tsx b/web_ui/frontend/components/Namespace/NamespaceIcon.tsx index 9fae82c1f..ba8bcd19c 100644 --- a/web_ui/frontend/components/Namespace/NamespaceIcon.tsx +++ b/web_ui/frontend/components/Namespace/NamespaceIcon.tsx @@ -13,7 +13,6 @@ const NamespaceIcon = ({ color?: string; bgcolor?: string; }) => { - const avatarPixelSize = useMemo(() => { switch (size) { case 'large': @@ -25,7 +24,7 @@ const NamespaceIcon = ({ default: return 30; } - }, [size]) + }, [size]); const iconPixelSize = useMemo(() => { switch (size) { @@ -38,7 +37,7 @@ const NamespaceIcon = ({ default: return 24; } - }, []) + }, []); if (serverType == 'namespace') { return ( @@ -51,10 +50,10 @@ const NamespaceIcon = ({ width: avatarPixelSize, my: 'auto', mr: 1, - bgcolor + bgcolor, }} > - + @@ -72,10 +71,10 @@ const NamespaceIcon = ({ width: avatarPixelSize, my: 'auto', mr: 1, - bgcolor + bgcolor, }} > - + @@ -93,10 +92,10 @@ const NamespaceIcon = ({ width: avatarPixelSize, my: 'auto', mr: 1, - bgcolor + bgcolor, }} > - + diff --git a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx index b7cb70501..66b1dc43a 100644 --- a/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx +++ b/web_ui/frontend/components/NamespaceCapabilitiesTable.tsx @@ -13,8 +13,8 @@ import { grey } from '@mui/material/colors'; import { NamespaceIcon } from '@/components/Namespace'; interface NamespaceCapabilitiesTableProps { - namespace: Namespace - servers?: ServerDetailed[] + namespace: Namespace; + servers?: ServerDetailed[]; } /** @@ -26,9 +26,8 @@ interface NamespaceCapabilitiesTableProps { */ export const NamespaceCapabilitiesTable = ({ namespace, - servers + servers, }: NamespaceCapabilitiesTableProps) => { - const theme = useTheme(); return ( @@ -64,8 +63,19 @@ export const NamespaceCapabilitiesTable = ({ - - + + {server.name} From cf287a24800b6552c7cd3e32084bffc220eaa821 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 19 Nov 2024 11:31:12 -0600 Subject: [PATCH 22/86] Run prettier --- .../frontend/app/registry/components/util.tsx | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/web_ui/frontend/app/registry/components/util.tsx b/web_ui/frontend/app/registry/components/util.tsx index 4477f9c5c..cc4a5e0ab 100644 --- a/web_ui/frontend/app/registry/components/util.tsx +++ b/web_ui/frontend/app/registry/components/util.tsx @@ -128,25 +128,26 @@ export const postGeneralNamespace = async ( export const putGeneralNamespace = async ( data: Namespace ): Promise => { - // If an access_token is in the URL, add it to the request - const url = new URL(`/api/v1.0/registry_ui/namespaces/${data.id}`, window.location.origin); - const accessToken = new URLSearchParams(window.location.search).get('access_token'); + const url = new URL( + `/api/v1.0/registry_ui/namespaces/${data.id}`, + window.location.origin + ); + const accessToken = new URLSearchParams(window.location.search).get( + 'access_token' + ); if (accessToken) { url.searchParams.append('access_token', accessToken); } - return await handleRequestAlert( - url.toString(), - { - body: JSON.stringify(data), - method: 'PUT', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - } - ); + return await handleRequestAlert(url.toString(), { + body: JSON.stringify(data), + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + }); }; export const submitNamespaceForm = async ( From a8062d7d07f6fd1d015fcb212eca50f16eb9febc Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 19 Nov 2024 11:58:10 -0600 Subject: [PATCH 23/86] Prettier --- web_ui/frontend/helpers/api.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts index 4bb2c3a0d..24b5903e4 100644 --- a/web_ui/frontend/helpers/api.ts +++ b/web_ui/frontend/helpers/api.ts @@ -161,7 +161,6 @@ export const postGeneralNamespace = async ( export const putGeneralNamespace = async ( data: Namespace ): Promise => { - // If an access_token is in the URL, add it to the request const url = new URL( `/api/v1.0/registry_ui/namespaces/${data.id}`, From e180b90740c0a99480cd161941d17c7ec2331ba6 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 20:47:55 +0000 Subject: [PATCH 24/86] Attempt longer ctx deadline in flaky localcache test --- local_cache/cache_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/local_cache/cache_test.go b/local_cache/cache_test.go index f57582ce4..b03654a1d 100644 --- a/local_cache/cache_test.go +++ b/local_cache/cache_test.go @@ -376,7 +376,10 @@ func TestLargeFile(t *testing.T) { viper.Set("Client.MaximumDownloadSpeed", 40*1024*1024) ft := fed_test_utils.NewFedTest(t, pubOriginCfg) - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + // Set a custom timeout for this test to see if we can make it less flaky + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancelTest, egrp := test_utils.TestContext(ctx, t) + te, err := client.NewTransferEngine(ctx) require.NoError(t, err) @@ -399,6 +402,7 @@ func TestLargeFile(t *testing.T) { t.Cleanup(func() { cancel() + cancelTest() if err := egrp.Wait(); err != nil && err != context.Canceled && err != http.ErrServerClosed { require.NoError(t, err) } @@ -408,7 +412,6 @@ func TestLargeFile(t *testing.T) { // Throw in a config.Reset for good measure. Keeps our env squeaky clean! server_utils.ResetTestState() }) - } // Create a federation then SIGSTOP the origin to prevent it from responding. From 0124123337cae843b8e7ac662c639c2664b1b275 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 19 Nov 2024 14:51:39 -0600 Subject: [PATCH 25/86] Prettier --- .../app/director/components/DirectorCard.tsx | 6 +++--- .../app/director/components/NamespaceCard.tsx | 5 +++-- web_ui/frontend/app/director/page.tsx | 13 +++++++++++-- web_ui/frontend/app/registry/components/PutPage.tsx | 4 +++- web_ui/frontend/helpers/api.ts | 3 +-- web_ui/frontend/helpers/get.ts | 11 +++++------ web_ui/frontend/index.ts | 1 - 7 files changed, 26 insertions(+), 17 deletions(-) diff --git a/web_ui/frontend/app/director/components/DirectorCard.tsx b/web_ui/frontend/app/director/components/DirectorCard.tsx index 479f2a4d1..f99652142 100644 --- a/web_ui/frontend/app/director/components/DirectorCard.tsx +++ b/web_ui/frontend/app/director/components/DirectorCard.tsx @@ -67,12 +67,12 @@ export const DirectorCard = ({ server, authenticated }: DirectorCardProps) => { if (detailedServer === undefined) { alertOnError( async () => { - const response = await getDirectorServer(server.name) + const response = await getDirectorServer(server.name); setDetailedServer(await response.json()); }, - "Failed to fetch server details", + 'Failed to fetch server details', dispatch - ) + ); } }} > diff --git a/web_ui/frontend/app/director/components/NamespaceCard.tsx b/web_ui/frontend/app/director/components/NamespaceCard.tsx index 35ecd8e76..057013ee3 100644 --- a/web_ui/frontend/app/director/components/NamespaceCard.tsx +++ b/web_ui/frontend/app/director/components/NamespaceCard.tsx @@ -13,7 +13,6 @@ export interface NamespaceCardProps { } export const NamespaceCard = ({ namespace }: NamespaceCardProps) => { - const dispatch = useContext(AlertDispatchContext); const [dropdownOpen, setDropdownOpen] = useState(false); const [servers, setServers] = useState( @@ -62,7 +61,9 @@ export const NamespaceCard = ({ namespace }: NamespaceCardProps) => { const getAssociatedServers = async (namespace: DirectorNamespace) => { const servers = await Promise.all( - [...namespace.origins, ...namespace.caches].map(async (name) => (await getDirectorServer(name)).json()) + [...namespace.origins, ...namespace.caches].map(async (name) => + (await getDirectorServer(name)).json() + ) ); // Alert the console if any servers are undefined, as this is unlikely to happen naturally diff --git a/web_ui/frontend/app/director/page.tsx b/web_ui/frontend/app/director/page.tsx index 60124fc64..a9665b1f5 100644 --- a/web_ui/frontend/app/director/page.tsx +++ b/web_ui/frontend/app/director/page.tsx @@ -38,12 +38,21 @@ export default function Page() { const { data } = useSWR( 'getDirectorServers', async () => - await alertOnError(getDirectorServers, 'Failed to fetch servers', dispatch) + await alertOnError( + getDirectorServers, + 'Failed to fetch servers', + dispatch + ) ); const { data: namespaces } = useSWR( 'getDirectorNamespaces', - async () => await alertOnError(getDirectorNamespaces, "Faild to fetch Namespaces", dispatch) + async () => + await alertOnError( + getDirectorNamespaces, + 'Faild to fetch Namespaces', + dispatch + ) ); const { data: user, error } = useSWR('getUser', () => diff --git a/web_ui/frontend/app/registry/components/PutPage.tsx b/web_ui/frontend/app/registry/components/PutPage.tsx index ca7cd07f8..2cd369777 100644 --- a/web_ui/frontend/app/registry/components/PutPage.tsx +++ b/web_ui/frontend/app/registry/components/PutPage.tsx @@ -47,7 +47,9 @@ import { alertOnError } from '@/helpers/util'; const PutPage = ({ update }: NamespaceFormPage) => { const [id, setId] = useState(undefined); const [fromUrl, setFromUrl] = useState(undefined); - const [namespace, setNamespace] = useState(undefined); + const [namespace, setNamespace] = useState( + undefined + ); const dispatch = useContext(AlertDispatchContext); diff --git a/web_ui/frontend/helpers/api.ts b/web_ui/frontend/helpers/api.ts index f61e90f9a..2d42a83db 100644 --- a/web_ui/frontend/helpers/api.ts +++ b/web_ui/frontend/helpers/api.ts @@ -133,7 +133,7 @@ export const getDirectorServer = async (name: string): Promise => { ); return await fetchApi(async () => await fetch(url)); -} +}; /** * Get namespaces from director @@ -147,7 +147,6 @@ export const getDirectorNamespaces = async () => { return await fetchApi(async () => await fetch(url)); }; - /** * Get namespaces */ diff --git a/web_ui/frontend/helpers/get.ts b/web_ui/frontend/helpers/get.ts index 1e2f95687..f127c5b3e 100644 --- a/web_ui/frontend/helpers/get.ts +++ b/web_ui/frontend/helpers/get.ts @@ -9,7 +9,7 @@ import { getDirectorNamespaces as getDirectorNamespacesResponse, getDirectorServers as getDirectorServersResponse, getConfig as getConfigResponse, - getNamespaces + getNamespaces, } from '@/helpers/api'; import { flattenObject } from '@/app/config/util'; import { DirectorNamespace } from '@/types'; @@ -25,22 +25,21 @@ import { ServerGeneral } from '@/types'; * Get and sort director servers */ export const getDirectorServers = async () => { - const response = await getDirectorServersResponse() + const response = await getDirectorServersResponse(); const responseData: ServerGeneral[] = await response.json(); responseData.sort((a, b) => a.name.localeCompare(b.name)); return responseData; -} +}; /** * Get and sort director namespaces */ export const getDirectorNamespaces = async () => { - const response = await getDirectorNamespacesResponse() + const response = await getDirectorNamespacesResponse(); const responseData: DirectorNamespace[] = await response.json(); responseData.sort((a, b) => a.path.localeCompare(b.path)); return responseData; -} - +}; export const getConfig = async (): Promise => { let response = await getConfigResponse(); diff --git a/web_ui/frontend/index.ts b/web_ui/frontend/index.ts index ab47845da..dfcdcae95 100644 --- a/web_ui/frontend/index.ts +++ b/web_ui/frontend/index.ts @@ -27,7 +27,6 @@ export interface Server { namespacePrefixes: string[]; } - export type StringTree = { [key: string]: StringTree | true }; export interface Alert { From f2187e21497cf7eba843c3c6b89720578d4db741 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Thu, 21 Nov 2024 18:02:29 +0000 Subject: [PATCH 26/86] Removed the "network" labels from the metrics --- director/director.go | 7 ------- director/sort.go | 10 ---------- metrics/director.go | 4 ++-- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/director/director.go b/director/director.go index 07dd24cb9..1ba4d9b55 100644 --- a/director/director.go +++ b/director/director.go @@ -1362,7 +1362,6 @@ func collectDirectorRedirectionMetric(ctx *gin.Context, destination string) { "destination": destination, "status_code": strconv.Itoa(ctx.Writer.Status()), "version": "", - "network": "", } version, _, err := extractVersionAndService(ctx) @@ -1376,12 +1375,6 @@ func collectDirectorRedirectionMetric(ctx *gin.Context, destination string) { labels["version"] = "unknown" } - maskedIp, ok := utils.ApplyIPMask(ctx.ClientIP()) - if ok { - labels["network"] = maskedIp - } else { - labels["network"] = "unknown" - } metrics.PelicanDirectorRedirectionsTotal.With(labels).Inc() } diff --git a/director/sort.go b/director/sort.go index 6a11e3361..b0f7305a3 100644 --- a/director/sort.go +++ b/director/sort.go @@ -37,7 +37,6 @@ import ( "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" - "github.com/pelicanplatform/pelican/utils" ) type ( @@ -157,19 +156,10 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 } labels := prometheus.Labels{ - "network": "", "source": "", "proj": "", } - network, ok := utils.ApplyIPMask(addr.String()) - if !ok { - log.Warningf("Failed to apply IP mask to address %s", ip.String()) - labels["network"] = "unknown" - } else { - labels["network"] = network - } - project, ok := ctx.Value(ProjectContextKey{}).(string) if !ok || project == "" { log.Warningf("Failed to get project from context") diff --git a/metrics/director.go b/metrics/director.go index 7be3927a8..e9ee1a699 100644 --- a/metrics/director.go +++ b/metrics/director.go @@ -95,10 +95,10 @@ var ( PelicanDirectorRedirectionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pelican_director_redirections_total", Help: "The total number of redirections the director issued.", - }, []string{"destination", "status_code", "version", "network"}) + }, []string{"destination", "status_code", "version"}) PelicanDirectorGeoIPErrors = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pelican_director_geoip_errors", Help: "The total number of errors encountered trying to resolve coordinates using the GeoIP MaxMind database", - }, []string{"network", "source", "proj"}) + }, []string{"source", "proj"}) ) From a09f635a3f33ef761691705e67c459a1e38c3583 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Thu, 21 Nov 2024 18:16:30 +0000 Subject: [PATCH 27/86] Fixed linter error in director/sort.go --- director/sort.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/director/sort.go b/director/sort.go index b0f7305a3..b261f6d08 100644 --- a/director/sort.go +++ b/director/sort.go @@ -156,8 +156,8 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 } labels := prometheus.Labels{ - "source": "", - "proj": "", + "source": "", + "proj": "", } project, ok := ctx.Value(ProjectContextKey{}).(string) From ed3d4ab458e3138c28757563162c7e4374e88541 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Thu, 21 Nov 2024 20:46:20 +0000 Subject: [PATCH 28/86] Removed new metrics that seem to causing Prometheus to use a very large amount of memory --- director/director.go | 46 -------------------------------------------- director/sort.go | 23 ++-------------------- metrics/director.go | 10 ---------- 3 files changed, 2 insertions(+), 77 deletions(-) diff --git a/director/director.go b/director/director.go index 07dd24cb9..f3db5a34b 100644 --- a/director/director.go +++ b/director/director.go @@ -333,18 +333,6 @@ func checkRedirectQuery(query url.Values) error { func redirectToCache(ginCtx *gin.Context) { reqVer, service, _ := extractVersionAndService(ginCtx) - // Flag to indicate if the request was redirected to a cache - // For metric collection purposes - // see collectDirectorRedirectionMetric - redirectedToCache := true - defer func() { - if !redirectedToCache { - collectDirectorRedirectionMetric(ginCtx, "origin") - } else { - collectDirectorRedirectionMetric(ginCtx, "cache") - } - }() - defer collectDirectorRedirectionMetric(ginCtx, "cache") err := versionCompatCheck(reqVer, service) if err != nil { log.Warningf("A version incompatibility was encountered while redirecting to a cache and no response was served: %v", err) @@ -485,11 +473,6 @@ func redirectToCache(ginCtx *gin.Context) { }) return } - // At this point, the cacheAds is full of originAds - // We need to indicate that we are redirecting to an origin and not a cache - // This is for the purpose of metrics - // See collectDirectorRedirectionMetric - redirectedToCache = false } ctx := context.Background() @@ -556,7 +539,6 @@ func redirectToCache(ginCtx *gin.Context) { func redirectToOrigin(ginCtx *gin.Context) { reqVer, service, _ := extractVersionAndService(ginCtx) - defer collectDirectorRedirectionMetric(ginCtx, "origin") err := versionCompatCheck(reqVer, service) if err != nil { log.Warningf("A version incompatibility was encountered while redirecting to an origin and no response was served: %v", err) @@ -1357,34 +1339,6 @@ func collectClientVersionMetric(reqVer *version.Version, service string) { metrics.PelicanDirectorClientVersionTotal.With(prometheus.Labels{"version": shortendVersion, "service": service}).Inc() } -func collectDirectorRedirectionMetric(ctx *gin.Context, destination string) { - labels := prometheus.Labels{ - "destination": destination, - "status_code": strconv.Itoa(ctx.Writer.Status()), - "version": "", - "network": "", - } - - version, _, err := extractVersionAndService(ctx) - if err != nil { - log.Warningf("Failed to extract version and service from request: %v", err) - return - } - if version != nil { - labels["version"] = version.String() - } else { - labels["version"] = "unknown" - } - - maskedIp, ok := utils.ApplyIPMask(ctx.ClientIP()) - if ok { - labels["network"] = maskedIp - } else { - labels["network"] = "unknown" - } - metrics.PelicanDirectorRedirectionsTotal.With(labels).Inc() -} - func RegisterDirectorAPI(ctx context.Context, router *gin.RouterGroup) { directorAPIV1 := router.Group("/api/v1.0/director") { diff --git a/director/sort.go b/director/sort.go index 6a11e3361..37dab4d69 100644 --- a/director/sort.go +++ b/director/sort.go @@ -34,10 +34,8 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" - "github.com/pelicanplatform/pelican/utils" ) type ( @@ -157,17 +155,8 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 } labels := prometheus.Labels{ - "network": "", - "source": "", - "proj": "", - } - - network, ok := utils.ApplyIPMask(addr.String()) - if !ok { - log.Warningf("Failed to apply IP mask to address %s", ip.String()) - labels["network"] = "unknown" - } else { - labels["network"] = network + "source": "", + "proj": "", } project, ok := ctx.Value(ProjectContextKey{}).(string) @@ -182,14 +171,10 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 reader := maxMindReader.Load() if reader == nil { err = errors.New("No GeoIP database is available") - labels["source"] = "server" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() return } record, err := reader.City(ip) if err != nil { - labels["source"] = "server" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() return } lat = record.Location.Latitude @@ -200,8 +185,6 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 // comes from a private range. if lat == 0 && long == 0 { log.Warningf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) - labels["source"] = "client" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() } // MaxMind provides an accuracy radius in kilometers. When it actually has no clue how to resolve a valid, public @@ -213,8 +196,6 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null.", ip.String(), record.Location.AccuracyRadius) lat = 0 long = 0 - labels["source"] = "client" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() } return diff --git a/metrics/director.go b/metrics/director.go index 7be3927a8..5a4731f80 100644 --- a/metrics/director.go +++ b/metrics/director.go @@ -91,14 +91,4 @@ var ( Name: "pelican_director_client_version_total", Help: "The total number of requests from client versions.", }, []string{"version", "service"}) - - PelicanDirectorRedirectionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pelican_director_redirections_total", - Help: "The total number of redirections the director issued.", - }, []string{"destination", "status_code", "version", "network"}) - - PelicanDirectorGeoIPErrors = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pelican_director_geoip_errors", - Help: "The total number of errors encountered trying to resolve coordinates using the GeoIP MaxMind database", - }, []string{"network", "source", "proj"}) ) From ccba46238f059f5b7cef031ee9de9eacc46b54aa Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 21 Nov 2024 21:38:43 +0000 Subject: [PATCH 29/86] Use Transport.ResponseHeaderTimeout to set pelican.timeout --- local_cache/cache_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/local_cache/cache_test.go b/local_cache/cache_test.go index b03654a1d..eac121845 100644 --- a/local_cache/cache_test.go +++ b/local_cache/cache_test.go @@ -373,13 +373,15 @@ func TestLargeFile(t *testing.T) { tmpDir := t.TempDir() server_utils.ResetTestState() - viper.Set("Client.MaximumDownloadSpeed", 40*1024*1024) - ft := fed_test_utils.NewFedTest(t, pubOriginCfg) - // Set a custom timeout for this test to see if we can make it less flaky - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - ctx, cancelTest, egrp := test_utils.TestContext(ctx, t) + clientConfig := map[string]interface{}{ + "Client.MaximumDownloadSpeed": 40 * 1024 * 1024, + "Transport.ResponseHeaderTimeout": "1000s", + } + test_utils.InitClient(t, clientConfig) + ft := fed_test_utils.NewFedTest(t, pubOriginCfg) + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) te, err := client.NewTransferEngine(ctx) require.NoError(t, err) @@ -402,7 +404,6 @@ func TestLargeFile(t *testing.T) { t.Cleanup(func() { cancel() - cancelTest() if err := egrp.Wait(); err != nil && err != context.Canceled && err != http.ErrServerClosed { require.NoError(t, err) } From edbd5e92bed84307f5fca3a5d22add1b3ef2eb4f Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 21 Nov 2024 21:55:57 +0000 Subject: [PATCH 30/86] Speedup ubuntu tests by passing --single-target to goreleaser --- .github/workflows/test.yml | 2 +- local_cache/cache_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d0e56fc3..bbc670a52 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -132,7 +132,7 @@ jobs: # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser version: latest - args: --clean --snapshot + args: --clean --snapshot --single-target - name: Copy files (Ubuntu) run: | cp dist/pelican_linux_amd64_v1/pelican ./ diff --git a/local_cache/cache_test.go b/local_cache/cache_test.go index eac121845..d6f1e7066 100644 --- a/local_cache/cache_test.go +++ b/local_cache/cache_test.go @@ -375,7 +375,7 @@ func TestLargeFile(t *testing.T) { server_utils.ResetTestState() clientConfig := map[string]interface{}{ - "Client.MaximumDownloadSpeed": 40 * 1024 * 1024, + "Client.MaximumDownloadSpeed": 40 * 1024 * 1024, "Transport.ResponseHeaderTimeout": "1000s", } test_utils.InitClient(t, clientConfig) From e60a8b144145a6dcffa9cfebd0bee78168a8ddcb Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 21 Nov 2024 22:15:35 +0000 Subject: [PATCH 31/86] Fix goreleaser --single-target arg --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbc670a52..5adb86ac2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -132,7 +132,7 @@ jobs: # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser version: latest - args: --clean --snapshot --single-target + args: build --single-target --clean --snapshot - name: Copy files (Ubuntu) run: | cp dist/pelican_linux_amd64_v1/pelican ./ From 138e90bdd53dc69f6c84e33f030662a543421487 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 22 Nov 2024 08:20:29 -0600 Subject: [PATCH 32/86] =?UTF-8?q?Revert=20"Removed=20new=20metrics=20that?= =?UTF-8?q?=20seem=20to=20causing=20Prometheus=20to=20use=20a=20very=20lar?= =?UTF-8?q?=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- director/director.go | 39 ++++++++++++++++++++++++++++++++++++++- director/sort.go | 9 +++++++++ metrics/director.go | 10 ++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/director/director.go b/director/director.go index 9479e0f7a..1aeb8e0dc 100644 --- a/director/director.go +++ b/director/director.go @@ -333,7 +333,17 @@ func checkRedirectQuery(query url.Values) error { func redirectToCache(ginCtx *gin.Context) { reqVer, service, _ := extractVersionAndService(ginCtx) - + // Flag to indicate if the request was redirected to a cache + // For metric collection purposes + // see collectDirectorRedirectionMetric + redirectedToCache := true + defer func() { + if !redirectedToCache { + collectDirectorRedirectionMetric(ginCtx, "origin") + } else { + collectDirectorRedirectionMetric(ginCtx, "cache") + } + }() err := versionCompatCheck(reqVer, service) if err != nil { log.Warningf("A version incompatibility was encountered while redirecting to a cache and no response was served: %v", err) @@ -474,6 +484,11 @@ func redirectToCache(ginCtx *gin.Context) { }) return } + // At this point, the cacheAds is full of originAds + // We need to indicate that we are redirecting to an origin and not a cache + // This is for the purpose of metrics + // See collectDirectorRedirectionMetric + redirectedToCache = false } ctx := context.Background() @@ -540,6 +555,7 @@ func redirectToCache(ginCtx *gin.Context) { func redirectToOrigin(ginCtx *gin.Context) { reqVer, service, _ := extractVersionAndService(ginCtx) + defer collectDirectorRedirectionMetric(ginCtx, "origin") err := versionCompatCheck(reqVer, service) if err != nil { log.Warningf("A version incompatibility was encountered while redirecting to an origin and no response was served: %v", err) @@ -1340,6 +1356,27 @@ func collectClientVersionMetric(reqVer *version.Version, service string) { metrics.PelicanDirectorClientVersionTotal.With(prometheus.Labels{"version": shortendVersion, "service": service}).Inc() } +func collectDirectorRedirectionMetric(ctx *gin.Context, destination string) { + labels := prometheus.Labels{ + "destination": destination, + "status_code": strconv.Itoa(ctx.Writer.Status()), + "version": "", + } + + version, _, err := extractVersionAndService(ctx) + if err != nil { + log.Warningf("Failed to extract version and service from request: %v", err) + return + } + if version != nil { + labels["version"] = version.String() + } else { + labels["version"] = "unknown" + } + + metrics.PelicanDirectorRedirectionsTotal.With(labels).Inc() +} + func RegisterDirectorAPI(ctx context.Context, router *gin.RouterGroup) { directorAPIV1 := router.Group("/api/v1.0/director") { diff --git a/director/sort.go b/director/sort.go index 37dab4d69..b261f6d08 100644 --- a/director/sort.go +++ b/director/sort.go @@ -34,6 +34,7 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" ) @@ -171,10 +172,14 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 reader := maxMindReader.Load() if reader == nil { err = errors.New("No GeoIP database is available") + labels["source"] = "server" + metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() return } record, err := reader.City(ip) if err != nil { + labels["source"] = "server" + metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() return } lat = record.Location.Latitude @@ -185,6 +190,8 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 // comes from a private range. if lat == 0 && long == 0 { log.Warningf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) + labels["source"] = "client" + metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() } // MaxMind provides an accuracy radius in kilometers. When it actually has no clue how to resolve a valid, public @@ -196,6 +203,8 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null.", ip.String(), record.Location.AccuracyRadius) lat = 0 long = 0 + labels["source"] = "client" + metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() } return diff --git a/metrics/director.go b/metrics/director.go index 5a4731f80..e9ee1a699 100644 --- a/metrics/director.go +++ b/metrics/director.go @@ -91,4 +91,14 @@ var ( Name: "pelican_director_client_version_total", Help: "The total number of requests from client versions.", }, []string{"version", "service"}) + + PelicanDirectorRedirectionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pelican_director_redirections_total", + Help: "The total number of redirections the director issued.", + }, []string{"destination", "status_code", "version"}) + + PelicanDirectorGeoIPErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pelican_director_geoip_errors", + Help: "The total number of errors encountered trying to resolve coordinates using the GeoIP MaxMind database", + }, []string{"source", "proj"}) ) From aaa982ab8fa891693f06bbdc154f0cac4f330f02 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 22 Nov 2024 15:26:30 +0000 Subject: [PATCH 33/86] Fixed macos failing tests: Pinned macos xrootd to last working version Added an error check in LocalCache.Stat() --- github_scripts/osx_install.sh | 3 ++- local_cache/local_cache.go | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/github_scripts/osx_install.sh b/github_scripts/osx_install.sh index ac9eff625..635aaf6a2 100755 --- a/github_scripts/osx_install.sh +++ b/github_scripts/osx_install.sh @@ -43,8 +43,9 @@ popd # Build XRootD from source # Add patches to xrootd source code if needed -git clone --depth=1 https://github.com/xrootd/xrootd.git +git clone --depth=10 https://github.com/xrootd/xrootd.git pushd xrootd +git checkout d74b2af36d04c398a91235bd2de32ee1283e92fe patch -p1 < $scriptdir/pelican_protocol.patch mkdir xrootd_build cd xrootd_build diff --git a/local_cache/local_cache.go b/local_cache/local_cache.go index 71df19cdf..1cfd77dfb 100644 --- a/local_cache/local_cache.go +++ b/local_cache/local_cache.go @@ -718,6 +718,9 @@ func (lc *LocalCache) Stat(path, token string) (uint64, error) { dUrl.Path = path dUrl.Scheme = "pelican" statInfo, err := client.DoStat(context.Background(), dUrl.String(), client.WithToken(token)) + if err != nil { + return 0, err + } return uint64(statInfo.Size), err } From 705457e01f10c5cc721fdb5824e9dc99fdc874d6 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 22 Nov 2024 17:55:09 +0000 Subject: [PATCH 34/86] Pinning the xrootd build in macos to a specific tag --- github_scripts/osx_install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_scripts/osx_install.sh b/github_scripts/osx_install.sh index 635aaf6a2..c702b414c 100755 --- a/github_scripts/osx_install.sh +++ b/github_scripts/osx_install.sh @@ -43,9 +43,9 @@ popd # Build XRootD from source # Add patches to xrootd source code if needed -git clone --depth=10 https://github.com/xrootd/xrootd.git +git clone https://github.com/xrootd/xrootd.git pushd xrootd -git checkout d74b2af36d04c398a91235bd2de32ee1283e92fe +git checkout tags/v5.7.1 patch -p1 < $scriptdir/pelican_protocol.patch mkdir xrootd_build cd xrootd_build From 0ecb8be19ad6271b6c55cea0797926de693932fd Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 22 Nov 2024 12:42:12 -0600 Subject: [PATCH 35/86] Revert "Removed the "network" labels from the metrics" --- director/director.go | 7 +++++++ director/sort.go | 14 ++++++++++++-- metrics/director.go | 4 ++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/director/director.go b/director/director.go index 1aeb8e0dc..f8f8f6344 100644 --- a/director/director.go +++ b/director/director.go @@ -1361,6 +1361,7 @@ func collectDirectorRedirectionMetric(ctx *gin.Context, destination string) { "destination": destination, "status_code": strconv.Itoa(ctx.Writer.Status()), "version": "", + "network": "", } version, _, err := extractVersionAndService(ctx) @@ -1374,6 +1375,12 @@ func collectDirectorRedirectionMetric(ctx *gin.Context, destination string) { labels["version"] = "unknown" } + maskedIp, ok := utils.ApplyIPMask(ctx.ClientIP()) + if ok { + labels["network"] = maskedIp + } else { + labels["network"] = "unknown" + } metrics.PelicanDirectorRedirectionsTotal.With(labels).Inc() } diff --git a/director/sort.go b/director/sort.go index b261f6d08..6a11e3361 100644 --- a/director/sort.go +++ b/director/sort.go @@ -37,6 +37,7 @@ import ( "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" + "github.com/pelicanplatform/pelican/utils" ) type ( @@ -156,8 +157,17 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 } labels := prometheus.Labels{ - "source": "", - "proj": "", + "network": "", + "source": "", + "proj": "", + } + + network, ok := utils.ApplyIPMask(addr.String()) + if !ok { + log.Warningf("Failed to apply IP mask to address %s", ip.String()) + labels["network"] = "unknown" + } else { + labels["network"] = network } project, ok := ctx.Value(ProjectContextKey{}).(string) diff --git a/metrics/director.go b/metrics/director.go index e9ee1a699..7be3927a8 100644 --- a/metrics/director.go +++ b/metrics/director.go @@ -95,10 +95,10 @@ var ( PelicanDirectorRedirectionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pelican_director_redirections_total", Help: "The total number of redirections the director issued.", - }, []string{"destination", "status_code", "version"}) + }, []string{"destination", "status_code", "version", "network"}) PelicanDirectorGeoIPErrors = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pelican_director_geoip_errors", Help: "The total number of errors encountered trying to resolve coordinates using the GeoIP MaxMind database", - }, []string{"source", "proj"}) + }, []string{"network", "source", "proj"}) ) From 6cfbe120a57887a5abdce4df65011a5dd8bd17af Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Fri, 22 Nov 2024 21:32:09 +0000 Subject: [PATCH 36/86] Use reasonable header timeout now that bug is found --- local_cache/cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_cache/cache_test.go b/local_cache/cache_test.go index d6f1e7066..04963cda7 100644 --- a/local_cache/cache_test.go +++ b/local_cache/cache_test.go @@ -376,7 +376,7 @@ func TestLargeFile(t *testing.T) { clientConfig := map[string]interface{}{ "Client.MaximumDownloadSpeed": 40 * 1024 * 1024, - "Transport.ResponseHeaderTimeout": "1000s", + "Transport.ResponseHeaderTimeout": "60s", } test_utils.InitClient(t, clientConfig) ft := fed_test_utils.NewFedTest(t, pubOriginCfg) From a4a7042443837394252cc5b95b4e7e9e94e0790e Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 18:20:06 +0000 Subject: [PATCH 37/86] Trigger lotman package build on compatible systems with 'lotman' build flag Regardless of the underlying host arch/os, this commit makes sure we only build Lotman if the `lotman` build flag is passed to go. This setup will allow us to trigger the build of a secondary binary with lotman enabled. --- lotman/lotman.go | 2 +- lotman/lotman_linux.go | 3 +-- lotman/lotman_test.go | 3 +-- lotman/lotman_ui.go | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lotman/lotman.go b/lotman/lotman.go index ee3544ffd..69ac4d302 100644 --- a/lotman/lotman.go +++ b/lotman/lotman.go @@ -1,4 +1,4 @@ -//go:build windows || darwin || linux +//go:build !lotman || (lotman && linux && ppc64le) || !linux // For now we're shutting off LotMan due to weirdness with purego. When we return to this, remember // that purego doesn't support (linux && ppc64le), so we'll need to add that back here. diff --git a/lotman/lotman_linux.go b/lotman/lotman_linux.go index 910859f90..2bbd80233 100644 --- a/lotman/lotman_linux.go +++ b/lotman/lotman_linux.go @@ -1,5 +1,4 @@ -//go:build false -// For now we're shutting off LotMan due to weirdness with purego +//go:build lotman && linux && !ppc64le /*************************************************************** * diff --git a/lotman/lotman_test.go b/lotman/lotman_test.go index 61975f6b1..38292d699 100644 --- a/lotman/lotman_test.go +++ b/lotman/lotman_test.go @@ -1,6 +1,5 @@ -//go:build false +//go:build lotman && linux && !ppc64le -//linux && !ppc64le /*************************************************************** * * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research diff --git a/lotman/lotman_ui.go b/lotman/lotman_ui.go index 57d44a856..6425e20eb 100644 --- a/lotman/lotman_ui.go +++ b/lotman/lotman_ui.go @@ -1,5 +1,4 @@ -//go:build false -//linux && !ppc64le +//go:build lotman && linux && !ppc64le /*************************************************************** * From b55d4e1f3b2d09650a6da2c754c6bfe1db51e2eb Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 18:29:57 +0000 Subject: [PATCH 38/86] Perform standard federation metadata discovery for determining fed issuer Between the writing and compiling out of this code, we've hidden direct access through the param package to the 'Federation.DirectorUrl' key. Instead, this now calls `config.GetFederation()`, which tries to perform metadata discovery in many cases. To fix this, the commit adjusts the way we grab this information, as well as fixes tests that now mock the metadata discovery as well. --- lotman/lotman_linux.go | 26 ++++++++--- lotman/lotman_test.go | 103 ++++++++++++++++++++++++++++++----------- lotman/lotman_ui.go | 15 ++++-- 3 files changed, 106 insertions(+), 38 deletions(-) diff --git a/lotman/lotman_linux.go b/lotman/lotman_linux.go index 2bbd80233..855d0963b 100644 --- a/lotman/lotman_linux.go +++ b/lotman/lotman_linux.go @@ -24,6 +24,7 @@ package lotman import ( "bytes" + "context" "encoding/json" "fmt" "os" @@ -35,6 +36,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" ) @@ -298,13 +300,17 @@ func GetAuthorizedCallers(lotName string) (*[]string, error) { // 1. The federation's discovery url // 2. The federation's director url // TODO: Consider what happens to the lot if either of these values change in the future after the lot is created? -func getFederationIssuer() string { - federationIssuer := param.Federation_DiscoveryUrl.GetString() +func getFederationIssuer() (string, error) { + fedInfo, err := config.GetFederation(context.Background()) + if err != nil { + return "", err + } + federationIssuer := fedInfo.DiscoveryEndpoint if federationIssuer == "" { - federationIssuer = param.Federation_DirectorUrl.GetString() + federationIssuer = fedInfo.DirectorEndpoint } - return federationIssuer + return federationIssuer, nil } // Initialize the LotMan library and bind its functions to the global vars @@ -358,14 +364,22 @@ func InitLotman() bool { log.Warningf("Error while unmarshaling Lots from config: %v", err) } - federationIssuer := getFederationIssuer() + federationIssuer, err := getFederationIssuer() + if err != nil { + log.Errorf("Error getting federation issuer: %v", err) + return false + } + if federationIssuer == "" { + log.Errorln("Unable to determine federation issuer which is needed by Lotman to determine lot ownership") + return false + } callerMutex.Lock() defer callerMutex.Unlock() ret = LotmanSetContextStr("caller", federationIssuer, &errMsg) if ret != 0 { trimBuf(&errMsg) - log.Errorf("Error setting context for default lot: %s", string(errMsg)) + log.Errorf("Error setting caller context to %s for default lot: %s", federationIssuer, string(errMsg)) return false } diff --git a/lotman/lotman_test.go b/lotman/lotman_test.go index 38292d699..b5a12c504 100644 --- a/lotman/lotman_test.go +++ b/lotman/lotman_test.go @@ -25,6 +25,8 @@ import ( _ "embed" "encoding/json" "fmt" + "net/http" + "net/http/httptest" "os" "strings" "testing" @@ -36,19 +38,33 @@ import ( ) //go:embed resources/lots-config.yaml - var yamlMockup string -func setupLotmanFromConf(t *testing.T, readConfig bool, name string) (bool, func()) { - // Load in our config - if readConfig { - viper.Set("Federation.DiscoveryUrl", "https://fake-federation.com") - viper.SetConfigType("yaml") - err := viper.ReadConfig(strings.NewReader(yamlMockup)) - if err != nil { - t.Fatalf("Error reading config: %v", err) - } - } +// Initialize Lotman +// If we read from the embedded yaml, we need to override the SHOULD_OVERRIDE keys with the discUrl +// so that underlying metadata discovery can happen against the mock discovery host +func setupLotmanFromConf(t *testing.T, readConfig bool, name string, discUrl string) (bool, func()) { + // Load in our config and handle overriding the SHOULD_OVERRIDE keys with the discUrl + // Load in our config + if readConfig { + viper.SetConfigType("yaml") + err := viper.ReadConfig(strings.NewReader(yamlMockup)) + if err != nil { + t.Fatalf("Error reading config: %v", err) + } + + // Traverse the settings and modify the "Owner" keys directly in Viper + lots := viper.Get("Lotman.Lots").([]interface{}) + for i, lot := range lots { + if lotMap, ok := lot.(map[string]interface{}); ok { + if owner, ok := lotMap["owner"].(string); ok && owner == "SHOULD_OVERRIDE" { + lotMap["owner"] = discUrl + lots[i] = lotMap + } + } + } + viper.Set("Lotman.Lots", lots) + } tmpPathPattern := name + "*" tmpPath, err := os.MkdirTemp("", tmpPathPattern) @@ -62,21 +78,43 @@ func setupLotmanFromConf(t *testing.T, readConfig bool, name string) (bool, func } } +// Create a mock discovery host that returns the servers URL as the value for each pelican-configuration key +func getMockDiscoveryHost() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/pelican-configuration" { + w.Header().Set("Content-Type", "application/json") + serverURL := r.Host + response := fmt.Sprintf(`{ + "director_endpoint": "https://%s/osdf-director.osg-htc.org", + "namespace_registration_endpoint": "https://%s/osdf-registry.osg-htc.org", + "jwks_uri": "https://%s/osdf/public_signing_key.jwks" +}`, serverURL, serverURL, serverURL) + w.Write([]byte(response)) + } else { + http.NotFound(w, r) + } + })) +} + // Test the library initializer. NOTE: this also tests CreateLot, which is a part of initialization. func TestLotmanInit(t *testing.T) { server_utils.ResetTestState() t.Run("TestBadInit", func(t *testing.T) { // We haven't set various bits needed to create the lots, like discovery URL - success, cleanup := setupLotmanFromConf(t, false, "LotmanBadInit") + success, cleanup := setupLotmanFromConf(t, false, "LotmanBadInit", "") defer cleanup() require.False(t, success) }) t.Run("TestGoodInit", func(t *testing.T) { viper.Set("Log.Level", "debug") - viper.Set("Federation.DiscoveryUrl", "https://fake-federation.com") - success, cleanup := setupLotmanFromConf(t, false, "LotmanGoodInit") + server := getMockDiscoveryHost() + // Set the Federation.DiscoveryUrl to the test server's URL + // Lotman uses the discovered URLs/keys to determine some aspects of lot ownership + viper.Set("Federation.DiscoveryUrl", server.URL) + + success, cleanup := setupLotmanFromConf(t, false, "LotmanGoodInit", server.URL) defer cleanup() require.True(t, success) @@ -94,7 +132,7 @@ func TestLotmanInit(t *testing.T) { err := json.Unmarshal(defaultOutput, &defaultLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling default lot JSON: %s", string(defaultOutput))) require.Equal(t, "default", defaultLot.LotName) - require.Equal(t, "https://fake-federation.com", defaultLot.Owner) + require.Equal(t, server.URL, defaultLot.Owner) require.Equal(t, "default", defaultLot.Parents[0]) require.Equal(t, 0.0, *(defaultLot.MPA.DedicatedGB)) require.Equal(t, int64(0), (defaultLot.MPA.MaxNumObjects.Value)) @@ -110,7 +148,7 @@ func TestLotmanInit(t *testing.T) { err = json.Unmarshal(rootOutput, &rootLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling root lot JSON: %s", string(rootOutput))) require.Equal(t, "root", rootLot.LotName) - require.Equal(t, "https://fake-federation.com", rootLot.Owner) + require.Equal(t, server.URL, rootLot.Owner) require.Equal(t, "root", rootLot.Parents[0]) require.Equal(t, 0.0, *(rootLot.MPA.DedicatedGB)) require.Equal(t, int64(0), (rootLot.MPA.MaxNumObjects.Value)) @@ -119,8 +157,9 @@ func TestLotmanInit(t *testing.T) { func TestLotmanInitFromConfig(t *testing.T) { server_utils.ResetTestState() - - success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf", server.URL) defer cleanup() require.True(t, success) @@ -139,7 +178,7 @@ func TestLotmanInitFromConfig(t *testing.T) { err := json.Unmarshal(defaultOutput, &defaultLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling default lot JSON: %s", string(defaultOutput))) require.Equal(t, "default", defaultLot.LotName) - require.Equal(t, "https://fake-federation.com", defaultLot.Owner) + require.Equal(t, server.URL, defaultLot.Owner) require.Equal(t, "default", defaultLot.Parents[0]) require.Equal(t, 100.0, *(defaultLot.MPA.DedicatedGB)) require.Equal(t, int64(1000), (defaultLot.MPA.MaxNumObjects.Value)) @@ -156,7 +195,7 @@ func TestLotmanInitFromConfig(t *testing.T) { err = json.Unmarshal(rootOutput, &rootLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling root lot JSON: %s", string(rootOutput))) require.Equal(t, "root", rootLot.LotName) - require.Equal(t, "https://fake-federation.com", rootLot.Owner) + require.Equal(t, server.URL, rootLot.Owner) require.Equal(t, "root", rootLot.Parents[0]) require.Equal(t, 1.0, *(rootLot.MPA.DedicatedGB)) require.Equal(t, int64(10), rootLot.MPA.MaxNumObjects.Value) @@ -219,7 +258,9 @@ func TestGetLotmanLib(t *testing.T) { func TestGetAuthzCallers(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanGetAuthzCalleres") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanGetAuthzCalleres", server.URL) defer cleanup() require.True(t, success) @@ -228,7 +269,7 @@ func TestGetAuthzCallers(t *testing.T) { authzedCallers, err := GetAuthorizedCallers("test-2") require.NoError(t, err, "Failed to get authorized callers") require.Equal(t, 2, len(*authzedCallers)) - require.Contains(t, *authzedCallers, "https://fake-federation.com") + require.Contains(t, *authzedCallers, server.URL) require.Contains(t, *authzedCallers, "https://different-fake-federation.com") // test with a non-existent lot @@ -238,7 +279,9 @@ func TestGetAuthzCallers(t *testing.T) { func TestGetLot(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanGetLot") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanGetLot", server.URL) defer cleanup() require.True(t, success) @@ -250,7 +293,7 @@ func TestGetLot(t *testing.T) { require.Contains(t, lot.Parents, "root") require.Contains(t, lot.Parents, "test-1") require.Equal(t, 3, len(lot.Owners)) - require.Contains(t, lot.Owners, "https://fake-federation.com") + require.Contains(t, lot.Owners, server.URL) require.Contains(t, lot.Owners, "https://different-fake-federation.com") require.Contains(t, lot.Owners, "https://another-fake-federation.com") require.Equal(t, 1.11, *(lot.MPA.DedicatedGB)) @@ -261,7 +304,9 @@ func TestGetLot(t *testing.T) { func TestUpdateLot(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf", server.URL) defer cleanup() require.True(t, success) @@ -284,7 +329,7 @@ func TestUpdateLot(t *testing.T) { }, } - err := UpdateLot(&lotUpdate, "https://fake-federation.com") + err := UpdateLot(&lotUpdate, server.URL) require.NoError(t, err, "Failed to update lot") // Now check that the update was successful @@ -299,12 +344,14 @@ func TestUpdateLot(t *testing.T) { func TestDeleteLotsRec(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf", server.URL) defer cleanup() require.True(t, success) // Delete test-1, then verify both it and test-2 are gone - err := DeleteLotsRecursive("test-1", "https://fake-federation.com") + err := DeleteLotsRecursive("test-1", server.URL) require.NoError(t, err, "Failed to delete lot") // Now check that the delete was successful diff --git a/lotman/lotman_ui.go b/lotman/lotman_ui.go index 6425e20eb..b5dad97b8 100644 --- a/lotman/lotman_ui.go +++ b/lotman/lotman_ui.go @@ -36,7 +36,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/pelicanplatform/pelican/config" - "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/token" @@ -117,7 +116,10 @@ func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { if len(lot.Parents) != 0 && lot.Parents[0] == "root" { // We check that the token is signed by the federation // First check for discovery URL and then for director URL, both of which should host the federation's pubkey - issuerUrl := getFederationIssuer() + issuerUrl, err := getFederationIssuer() + if err != nil { + return false, err + } kSet, err := server_utils.GetJWKSFromIssUrl(issuerUrl) if err != nil { @@ -200,7 +202,12 @@ func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { // Get the namespace by querying the director and checking the headers errMsgPrefix := "the provided token is acceptible, but no owner could be determined because " - directorUrlStr := param.Federation_DirectorUrl.GetString() + + fedInfo, err := config.GetFederation(context.Background()) + if err != nil { + return false, errors.Wrap(err, errMsgPrefix+"the federation information could not be retrieved") + } + directorUrlStr := fedInfo.DirectorEndpoint if directorUrlStr == "" { return false, errors.New(errMsgPrefix + "the federation director URL is not set") } @@ -211,7 +218,7 @@ func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { directorUrl.Path, err = url.JoinPath("/api/v1.0/director/object", path) if err != nil { - return false, errors.Wrap(err, errMsgPrefix+"the director URL could not be joined with the path") + return false, errors.Wrap(err, errMsgPrefix+"the director's object path could not be constructed") } // Get the namespace by querying the director and checking the headers. The client should NOT From 200811331a6489559eb5a5ee17e399c23bd5e014 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 18:33:42 +0000 Subject: [PATCH 39/86] Enable a few extra debugging tools for lotman --- lotman/lotman_linux.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lotman/lotman_linux.go b/lotman/lotman_linux.go index 855d0963b..1719911e3 100644 --- a/lotman/lotman_linux.go +++ b/lotman/lotman_linux.go @@ -59,6 +59,7 @@ var ( // Auxilliary functions LotmanLotExists func(lotName string, errMsg *[]byte) int32 LotmanSetContextStr func(contextKey string, contextValue string, errMsg *[]byte) int32 + LotmanGetContextStr func(key string, output *[]byte, errMsg *[]byte) int32 // Functions that would normally take a char *** as an argument take an *unsafe.Pointer instead because // these functions are responsible for allocating and deallocating the memory for the char ***. The Go // runtime will handle the memory management for the *unsafe.Pointer. @@ -339,6 +340,7 @@ func InitLotman() bool { // Auxilliary functions purego.RegisterLibFunc(&LotmanLotExists, lotmanLib, "lotman_lot_exists") purego.RegisterLibFunc(&LotmanSetContextStr, lotmanLib, "lotman_set_context_str") + purego.RegisterLibFunc(&LotmanGetContextStr, lotmanLib, "lotman_get_context_str") purego.RegisterLibFunc(&LotmanGetLotOwners, lotmanLib, "lotman_get_owners") purego.RegisterLibFunc(&LotmanGetLotParents, lotmanLib, "lotman_get_parent_names") purego.RegisterLibFunc(&LotmanGetLotsFromDir, lotmanLib, "lotman_get_lots_from_dir") @@ -540,6 +542,7 @@ func InitLotman() bool { if ret != 0 { trimBuf(&errMsg) log.Errorf("Error creating lot %s: %s", lot.LotName, string(errMsg)) + log.Infoln("Full lot JSON passed to Lotman for lot creation:", string(lotJSON)) return false } } From 5aa710e48ac65b2af3afcadeb4930ec4a56ca33f Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 18:34:20 +0000 Subject: [PATCH 40/86] Adjust mock lot config to reflect metadata discovery --- lotman/resources/lots-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lotman/resources/lots-config.yaml b/lotman/resources/lots-config.yaml index 874b0b7a3..9fdf31edb 100644 --- a/lotman/resources/lots-config.yaml +++ b/lotman/resources/lots-config.yaml @@ -20,7 +20,7 @@ Lotman: Lots: - LotName: "default" - Owner: "https://fake-federation.com" + Owner: "SHOULD_OVERRIDE" Parents: - "default" ManagementPolicyAttrs: @@ -38,7 +38,7 @@ Lotman: Value: 123456 - LotName: "root" - Owner: "https://fake-federation.com" + Owner: "SHOULD_OVERRIDE" Parents: - "root" Paths: From a81a7ac9d7a8d789dad14ed1105034820b08fe27 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 19:46:43 +0000 Subject: [PATCH 41/86] Create Lotman-enabled server binary and plumb through to prod cache containers This creates a second `pelican-server` binary that passes the `lotman` build flag to go, thereby enabling the previously-shimmed Lotman wrapper package. Since we only envision this being used in caches (at least for the time being), this new binary only replaces the binaries for the `cache` and `osdf-cache` targeted Docker builds. Along with this, the entrypoints for these containers have been modified to accept `pelican-server` and `osdf-server` arguments. --- .goreleaser.yml | 18 +++++++++++++++++- images/Dockerfile | 13 +++++++++---- images/entrypoint.sh | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index dc184d93a..f6d0174b9 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -50,7 +50,23 @@ builds: goarch: ppc64le - goos: darwin goarch: ppc64le - + # Set things up to build a second server binary that enables Lotman. Eventually + # we'll also use this to filter which moduels are built into the binary. + - env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - "amd64" + - "arm64" + id: "pelican-server" + dir: ./cmd + binary: pelican-server + tags: + - forceposix + - lotman + ldflags: + - -s -w -X github.com/pelicanplatform/pelican/config.commit={{.Commit}} -X github.com/pelicanplatform/pelican/config.date={{.Date}} -X github.com/pelicanplatform/pelican/config.builtBy=goreleaser -X github.com/pelicanplatform/pelican/config.version={{.Version}} archives: - id: pelican name_template: >- diff --git a/images/Dockerfile b/images/Dockerfile index 4689158fd..30836869f 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -220,8 +220,11 @@ RUN rm -rf /pelican/pelican #################### FROM pelican-base AS cache - -ENTRYPOINT [ "/entrypoint.sh", "pelican", "cache"] +RUN rm -rf /pelican/pelican +COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /pelican/pelican-server +RUN chmod +x /pelican/pelican-server +# For now, we're only using pelican-server in the cache, but eventually we'll use it in all servers +ENTRYPOINT [ "/entrypoint.sh", "pelican-server", "cache"] CMD [ "serve" ] #################### @@ -260,8 +263,10 @@ CMD [ "serve" ] #################### FROM osdf-base AS osdf-cache - -ENTRYPOINT [ "/entrypoint.sh" ,"osdf", "cache"] +RUN rm -rf /pelican/osdf +COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /pelican/osdf-server +RUN chmod +x /pelican/osdf-server +ENTRYPOINT [ "/entrypoint.sh" ,"osdf-server", "cache"] CMD [ "serve" ] #################### diff --git a/images/entrypoint.sh b/images/entrypoint.sh index d7f5943de..4d8b24037 100644 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -119,6 +119,15 @@ if [ $# -ne 0 ]; then echo >&2 "Exec of tini failed!" exit 1 ;; + pelican-server) + # Our server-specific binary which may come with additional + # features/system requirements (like Lotman) + echo "Running pelican-server with arguments: $@" + exec tini -- /pelican/pelican-server "$@" + # we shouldn't get here + echo >&2 "Exec of tini failed!" + exit 1 + ;; osdf) # Run osdf with the rest of the arguments echo "Running osdf with arguments: $@" @@ -127,6 +136,13 @@ if [ $# -ne 0 ]; then echo >&2 "Exec of tini failed!" exit 1 ;; + osdf-server) + echo "Running osdf-server with arguments: $@" + exec tini -- /pelican/osdf-server "$@" + # we shouldn't get here + echo >&2 "Exec of tini failed!" + exit 1 + ;; *) # Default case if the program selector does not match echo "Unknown program: $program_selector" From 76f3f94b388f976d4c693181a34f7800f8e8a4ff Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 19 Nov 2024 20:16:14 +0000 Subject: [PATCH 42/86] Attempt running dual test suites -- one for server binary, one for regular Github Actions workflows are not my forte, but the attempt here is to re-use the workflow file (now test-template.yml) to duplicate all the tests but with the new binary and with the lotman tests. Since GHA doesn't provide a way to mock the action environment in a local dry-run, I'll be developing through commits/pushes. Let's see how it goes... --- .github/workflows/test-template.yml | 101 ++++++++++++++++++++++++++ .github/workflows/test.yml | 107 +++------------------------- 2 files changed, 112 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/test-template.yml diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml new file mode 100644 index 000000000..53b311fab --- /dev/null +++ b/.github/workflows/test-template.yml @@ -0,0 +1,101 @@ +name: Test Template + +on: + workflow_call: + inputs: + tags: + required: true + type: string + coverprofile: + required: true + type: string + binary_name: + required: true + type: string + +jobs: + test: + runs-on: ubuntu-latest + container: + image: hub.opensciencegrid.org/pelican_platform/pelican-dev:latest-itb + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Fetch tags + run: | + git config --global --add safe.directory /__w/pelican/pelican + git fetch --force --tags + - name: Cache Next.js + uses: actions/cache@v4 + with: + path: | + ~/.npm + ${{ github.workspace }}/.next/cache + key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx') }} + restore-keys: | + ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- + - name: Test + run: | + make web-build + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + - name: Get total code coverage + if: github.event_name == 'pull_request' + id: cc + run: | + set -x + cc_total=`go tool cover -func=${{ inputs.coverprofile }} | grep total | grep -Eo '[0-9]+\.[0-9]+'` + echo "cc_total=$cc_total" >> $GITHUB_OUTPUT + - name: Restore base test coverage + id: base-coverage + if: github.event.pull_request.base.sha != '' + uses: actions/cache@v4 + with: + path: | + unit-base.txt + key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} + - name: Run test for base code + if: steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != '' + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch origin main ${{ github.event.pull_request.base.sha }} + HEAD=$(git rev-parse HEAD) + git reset --hard ${{ github.event.pull_request.base.sha }} + make web-build + go generate ./... + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./... + go tool cover -func=base_coverage.out > unit-base.txt + git reset --hard $HEAD + - name: Get base code coverage value + if: github.event_name == 'pull_request' + id: cc_b + run: | + set -x + cc_base_total=`grep total ./unit-base.txt | grep -Eo '[0-9]+\.[0-9]+'` + echo "cc_base_total=$cc_base_total" >> $GITHUB_OUTPUT + - name: Add coverage information to action summary + if: github.event_name == 'pull_request' + run: echo 'Code coverage ' ${{steps.cc.outputs.cc_total}}'% Prev ' ${{steps.cc_b.outputs.cc_base_total}}'%' >> $GITHUB_STEP_SUMMARY + - name: Run GoReleaser for Ubuntu + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: build --single-target --clean --snapshot + - name: Copy files (Ubuntu) + run: | + cp dist/pelican_linux_amd64_v1/${{ inputs.binary_name }} ./ + - name: Run Integration Tests + run: ./github_scripts/citests.sh + - name: Run End-to-End Test for Object get/put + run: ./github_scripts/get_put_test.sh + - name: Run End-to-End Test for Director stat + run: ./github_scripts/stat_test.sh + - name: Run End-to-End Test of x509 access + run: ./github_scripts/x509_test.sh + - name: Run End-to-End Test for --version flag + run: ./github_scripts/version_test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5adb86ac2..aa3ab6734 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,8 +13,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - # Do fetch depth 0 here because otherwise goreleaser might not work properly: - # https://goreleaser.com/ci/actions/?h=tag#workflow fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -25,9 +23,7 @@ jobs: path: | ~/.npm ${{ github.workspace }}/.next/cache - # Generate a new cache whenever packages or source files change. key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx', '!**/node_modules/**') }} - # If source files changed but packages didn't, rebuild from a prior cache. restore-keys: | ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- - name: Install Go @@ -50,99 +46,18 @@ jobs: - name: Run GoReleaser for Non-Ubuntu uses: goreleaser/goreleaser-action@v5 with: - # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser version: latest args: build --single-target --clean --snapshot test-ubuntu: - runs-on: ubuntu-latest - container: - image: hub.opensciencegrid.org/pelican_platform/pelican-dev:latest-itb - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - # See above for why fetch depth is 0 here - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20 - # Fetch the tags is essential so that goreleaser can build the correct version. Workaround found here: - # https://github.com/actions/checkout/issues/290 - - name: Fetch tags - run: | - git config --global --add safe.directory /__w/pelican/pelican - git fetch --force --tags - - name: Cache Next.js - uses: actions/cache@v4 - with: - path: | - ~/.npm - ${{ github.workspace }}/.next/cache - # Generate a new cache whenever packages or source files change. - key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx') }} - # If source files changed but packages didn't, rebuild from a prior cache. - restore-keys: | - ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- - - name: Test - run: | - make web-build - go test -timeout 15m -coverpkg=./... -coverprofile=coverage.out -covermode=count ./... - - name: Get total code coverage - if: github.event_name == 'pull_request' - id: cc - run: | - set -x - cc_total=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'` - echo "cc_total=$cc_total" >> $GITHUB_OUTPUT - - name: Restore base test coverage - id: base-coverage - if: github.event.pull_request.base.sha != '' - uses: actions/cache@v4 - with: - path: | - unit-base.txt - # Use base sha for PR or new commit hash for master/main push in test result key. - key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} - - name: Run test for base code - if: steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != '' - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git fetch origin main ${{ github.event.pull_request.base.sha }} - HEAD=$(git rev-parse HEAD) - git reset --hard ${{ github.event.pull_request.base.sha }} - make web-build - go generate ./... - go test -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./... - go tool cover -func=base_coverage.out > unit-base.txt - git reset --hard $HEAD - - name: Get base code coverage value - if: github.event_name == 'pull_request' - id: cc_b - run: | - set -x - cc_base_total=`grep total ./unit-base.txt | grep -Eo '[0-9]+\.[0-9]+'` - echo "cc_base_total=$cc_base_total" >> $GITHUB_OUTPUT - - name: Add coverage information to action summary - if: github.event_name == 'pull_request' - run: echo 'Code coverage ' ${{steps.cc.outputs.cc_total}}'% Prev ' ${{steps.cc_b.outputs.cc_base_total}}'%' >> $GITHUB_STEP_SUMMARY - - name: Run GoReleaser for Ubuntu - uses: goreleaser/goreleaser-action@v5 - with: - # either 'goreleaser' (default) or 'goreleaser-pro' - distribution: goreleaser - version: latest - args: build --single-target --clean --snapshot - - name: Copy files (Ubuntu) - run: | - cp dist/pelican_linux_amd64_v1/pelican ./ - - name: Run Integration Tests - run: ./github_scripts/citests.sh - - name: Run End-to-End Test for Object get/put - run: ./github_scripts/get_put_test.sh - - name: Run End-to-End Test for Director stat - run: ./github_scripts/stat_test.sh - - name: Run End-to-End Test of x509 access - run: ./github_scripts/x509_test.sh - - name: Run End-to-End Test for --version flag - run: ./github_scripts/version_test.sh + uses: ./.github/workflows/test-template.yml + with: + tags: "" + coverprofile: "coverage.out" + binary_name: "pelican" + test-ubuntu-server: + uses: ./.github/workflows/test-template.yml + with: + tags: "lotman" + coverprofile: "coverage-server.out" + binary_name: "pelican-server" From 26414086a2db0702a8c2eb88f64d703f3a673fbe Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 21 Nov 2024 16:30:44 +0000 Subject: [PATCH 43/86] Split binary archives to ameliorate goreleaser warning --- .github/workflows/test-template.yml | 2 +- .github/workflows/test.yml | 5 +++++ .goreleaser.yml | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 53b311fab..d4b9a6711 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -88,7 +88,7 @@ jobs: args: build --single-target --clean --snapshot - name: Copy files (Ubuntu) run: | - cp dist/pelican_linux_amd64_v1/${{ inputs.binary_name }} ./ + cp dist/${{ inputs.binary_name }}_linux_amd64_v1/${{ inputs.binary_name }} ./ - name: Run Integration Tests run: ./github_scripts/citests.sh - name: Run End-to-End Test for Object get/put diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa3ab6734..0804df90f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: + # Do fetch depth 0 here because otherwise goreleaser might not work properly: + # https://goreleaser.com/ci/actions/?h=tag#workflow fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -23,7 +25,9 @@ jobs: path: | ~/.npm ${{ github.workspace }}/.next/cache + # Generate a new cache whenever packages or source files change. key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx', '!**/node_modules/**') }} + # If source files changed but packages didn't, rebuild from a prior cache. restore-keys: | ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- - name: Install Go @@ -46,6 +50,7 @@ jobs: - name: Run GoReleaser for Non-Ubuntu uses: goreleaser/goreleaser-action@v5 with: + # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser version: latest args: build --single-target --clean --snapshot diff --git a/.goreleaser.yml b/.goreleaser.yml index f6d0174b9..5ae8b446f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -67,8 +67,13 @@ builds: - lotman ldflags: - -s -w -X github.com/pelicanplatform/pelican/config.commit={{.Commit}} -X github.com/pelicanplatform/pelican/config.date={{.Date}} -X github.com/pelicanplatform/pelican/config.builtBy=goreleaser -X github.com/pelicanplatform/pelican/config.version={{.Version}} +# Goreleaser complains if there's a different number of binaries built for different architectures +# in the same archive. Instead of plopping pelican-server in the same archive as pelican, split the +# builds into separate archives. archives: - id: pelican + builds: + - pelican name_template: >- {{ .ProjectName }}_ {{- title .Os }}_ @@ -78,6 +83,15 @@ archives: - goos: windows format: zip wrap_in_directory: '{{ .ProjectName }}-{{ trimsuffix .Version "-next" }}' + - id: pelican-server + builds: + - pelican-server + name_template: >- + {{ .ProjectName }}-server_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else }}{{ .Arch }}{{ end }} + wrap_in_directory: '{{ .ProjectName }}-server-{{ trimsuffix .Version "-next" }}' checksum: name_template: 'checksums.txt' snapshot: From 3cbbe7a1bc7370d961743bd747fed40c4176b71c Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Fri, 22 Nov 2024 22:00:49 +0000 Subject: [PATCH 44/86] Use binary name 'pelican' and not 'pelican-server' for all CI tests --- .github/workflows/test-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index d4b9a6711..0109c967e 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -88,7 +88,7 @@ jobs: args: build --single-target --clean --snapshot - name: Copy files (Ubuntu) run: | - cp dist/${{ inputs.binary_name }}_linux_amd64_v1/${{ inputs.binary_name }} ./ + cp dist/${{ inputs.binary_name }}_linux_amd64_v1/${{ inputs.binary_name }} ./pelican - name: Run Integration Tests run: ./github_scripts/citests.sh - name: Run End-to-End Test for Object get/put From b70a7698900a4e4e8dd115348e656ee64a5ae160 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 11 Nov 2024 16:13:43 +0000 Subject: [PATCH 45/86] Move project label setting code to its own function --- director/sort.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/director/sort.go b/director/sort.go index 6a11e3361..724988aa9 100644 --- a/director/sort.go +++ b/director/sort.go @@ -148,6 +148,17 @@ func checkOverrides(addr net.IP) (coordinate *Coordinate) { return nil } +func setProjectLabel(ctx context.Context, labels prometheus.Labels) bool { + project, ok := ctx.Value(ProjectContextKey{}).(string) + if !ok || project == "" { + labels["proj"] = "unknown" + return false + } else { + labels["proj"] = project + return true + } +} + func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64, err error) { ip := net.IP(addr.AsSlice()) override := checkOverrides(ip) @@ -170,13 +181,10 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 labels["network"] = network } - project, ok := ctx.Value(ProjectContextKey{}).(string) - if !ok || project == "" { + ok = setProjectLabel(ctx, labels) + if !ok { log.Warningf("Failed to get project from context") - labels["proj"] = "unknown" labels["source"] = "server" - } else { - labels["proj"] = project } reader := maxMindReader.Load() From e4dfb88e6d3c3d1ca5a6eb411ce460ac019844e5 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 17:05:38 +0000 Subject: [PATCH 46/86] Added custom error types --- director/sort.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/director/sort.go b/director/sort.go index 724988aa9..6d012df4e 100644 --- a/director/sort.go +++ b/director/sort.go @@ -21,6 +21,7 @@ package director import ( "cmp" "context" + "fmt" "math/rand" "net" "net/netip" @@ -59,8 +60,50 @@ type ( IP string `mapstructure:"IP"` Coordinate Coordinate `mapstructure:"Coordinate"` } + + GeoIPMaskError struct { + IpStr string + } + GeoIPProjectError struct{} + GeoIPMaxMindError struct{} + GeoIPNullError struct { + IpStr string + } + GeoIPReaderError struct { + errStr string + } + GeoIPAccuracyError struct { + IpStr string + AcuRadius uint16 + } ) +func (e GeoIPMaskError) Error() string { + return fmt.Sprintf("Failed to apply IP mask to address %s", e.IpStr) +} + +func (e GeoIPProjectError) Error() string { + return "Failed to get project from context" +} + +func (e GeoIPMaxMindError) Error() string { + return "No GeoIP database is available" +} + +func (e GeoIPNullError) Error() string { + return fmt.Sprintf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", e.IpStr) +} + +func (e GeoIPReaderError) Error() string { + return e.errStr +} + +func (e GeoIPAccuracyError) Error() string { + errStr := fmt.Sprintf("GeoIP resolution of the address %s resulted in a suspiciously large accuracy radius of %d km. ", e.IpStr, e.AcuRadius) + errStr += "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null." + return errStr +} + var ( invalidOverrideLogOnce = map[string]bool{} geoIPOverrides []GeoIPOverride @@ -176,6 +219,7 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 network, ok := utils.ApplyIPMask(addr.String()) if !ok { log.Warningf("Failed to apply IP mask to address %s", ip.String()) + err = GeoIPMaskError{IpStr: ip.String()} labels["network"] = "unknown" } else { labels["network"] = network @@ -184,18 +228,20 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 ok = setProjectLabel(ctx, labels) if !ok { log.Warningf("Failed to get project from context") + err = GeoIPProjectError{} labels["source"] = "server" } reader := maxMindReader.Load() if reader == nil { - err = errors.New("No GeoIP database is available") + err = GeoIPMaxMindError{} labels["source"] = "server" metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() return } record, err := reader.City(ip) if err != nil { + err = GeoIPReaderError{errStr: err.Error()} labels["source"] = "server" metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() return @@ -208,6 +254,7 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 // comes from a private range. if lat == 0 && long == 0 { log.Warningf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) + err = GeoIPNullError{IpStr: ip.String()} labels["source"] = "client" metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() } @@ -219,6 +266,7 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 if record.Location.AccuracyRadius >= 900 { log.Warningf("GeoIP resolution of the address %s resulted in a suspiciously large accuracy radius of %d km. "+ "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null.", ip.String(), record.Location.AccuracyRadius) + err = GeoIPAccuracyError{IpStr: ip.String(), AcuRadius: record.Location.AccuracyRadius} lat = 0 long = 0 labels["source"] = "client" From 2bc73b9d426346b3ef954314876e72018eaeae86 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 17:49:06 +0000 Subject: [PATCH 47/86] Refactor GeoIP error types to single type --- director/sort.go | 63 +++++++++--------------------------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/director/sort.go b/director/sort.go index 6d012df4e..73acdbea5 100644 --- a/director/sort.go +++ b/director/sort.go @@ -61,47 +61,14 @@ type ( Coordinate Coordinate `mapstructure:"Coordinate"` } - GeoIPMaskError struct { - IpStr string - } - GeoIPProjectError struct{} - GeoIPMaxMindError struct{} - GeoIPNullError struct { - IpStr string - } - GeoIPReaderError struct { - errStr string - } - GeoIPAccuracyError struct { - IpStr string - AcuRadius uint16 + GeoIPError struct { + labels prometheus.Labels + errorMsg string } ) -func (e GeoIPMaskError) Error() string { - return fmt.Sprintf("Failed to apply IP mask to address %s", e.IpStr) -} - -func (e GeoIPProjectError) Error() string { - return "Failed to get project from context" -} - -func (e GeoIPMaxMindError) Error() string { - return "No GeoIP database is available" -} - -func (e GeoIPNullError) Error() string { - return fmt.Sprintf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", e.IpStr) -} - -func (e GeoIPReaderError) Error() string { - return e.errStr -} - -func (e GeoIPAccuracyError) Error() string { - errStr := fmt.Sprintf("GeoIP resolution of the address %s resulted in a suspiciously large accuracy radius of %d km. ", e.IpStr, e.AcuRadius) - errStr += "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null." - return errStr +func (e GeoIPError) Error() string { + return e.errorMsg } var ( @@ -219,7 +186,6 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 network, ok := utils.ApplyIPMask(addr.String()) if !ok { log.Warningf("Failed to apply IP mask to address %s", ip.String()) - err = GeoIPMaskError{IpStr: ip.String()} labels["network"] = "unknown" } else { labels["network"] = network @@ -227,23 +193,19 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 ok = setProjectLabel(ctx, labels) if !ok { - log.Warningf("Failed to get project from context") - err = GeoIPProjectError{} labels["source"] = "server" } reader := maxMindReader.Load() if reader == nil { - err = GeoIPMaxMindError{} labels["source"] = "server" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() + err = GeoIPError{labels: labels, errorMsg: "No GeoIP database is available"} return } record, err := reader.City(ip) if err != nil { - err = GeoIPReaderError{errStr: err.Error()} labels["source"] = "server" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() + err = GeoIPError{labels: labels, errorMsg: err.Error()} return } lat = record.Location.Latitude @@ -253,10 +215,10 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 // There's likely a problem with the GeoIP database or the IP address. Usually this just means the IP address // comes from a private range. if lat == 0 && long == 0 { - log.Warningf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) - err = GeoIPNullError{IpStr: ip.String()} + errMsg := fmt.Sprintf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) + log.Warningf(errMsg) labels["source"] = "client" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() + err = GeoIPError{labels: labels, errorMsg: errMsg} } // MaxMind provides an accuracy radius in kilometers. When it actually has no clue how to resolve a valid, public @@ -264,13 +226,12 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 // should be very suspicious of the data, and mark it as appearing at the null lat/long (and provide a warning in // the Director), which also triggers random weighting in our sort algorithms. if record.Location.AccuracyRadius >= 900 { - log.Warningf("GeoIP resolution of the address %s resulted in a suspiciously large accuracy radius of %d km. "+ + errMsg := fmt.Sprintf("GeoIP resolution of the address %s resulted in a suspiciously large accuracy radius of %d km. "+ "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null.", ip.String(), record.Location.AccuracyRadius) - err = GeoIPAccuracyError{IpStr: ip.String(), AcuRadius: record.Location.AccuracyRadius} lat = 0 long = 0 labels["source"] = "client" - metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() + err = GeoIPError{labels: labels, errorMsg: errMsg} } return From b2ef725e280640ace396a05dc0aa69fb8f2010be Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 17:50:53 +0000 Subject: [PATCH 48/86] Update getClientLatLong to return error, had to reflect in tests as well. Also use GeoIPError to update metric --- director/sort.go | 16 +++++++++++++--- director/sort_test.go | 8 ++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/director/sort.go b/director/sort.go index 73acdbea5..4c9c05500 100644 --- a/director/sort.go +++ b/director/sort.go @@ -246,8 +246,7 @@ func assignRandBoundedCoord(minLat, maxLat, minLong, maxLong float64) (lat, long // Given a client address, attempt to get the lat/long of the client. If the address is invalid or // the lat/long is not resolvable, assign a random location in the contiguous US. -func getClientLatLong(ctx context.Context, addr netip.Addr) (coord Coordinate) { - var err error +func getClientLatLong(ctx context.Context, addr netip.Addr) (coord Coordinate, err error) { if !addr.IsValid() { log.Warningf("Unable to sort servers based on client-server distance. Invalid client IP address: %s", addr.String()) coord.Lat, coord.Long = assignRandBoundedCoord(usLatMin, usLatMax, usLongMin, usLongMax) @@ -303,7 +302,18 @@ func sortServerAds(ctx context.Context, clientAddr netip.Addr, ads []server_stru weights := make(SwapMaps, len(ads)) sortMethod := param.Director_CacheSortMethod.GetString() // This will handle the case where the client address is invalid or the lat/long is not resolvable. - clientCoord := getClientLatLong(ctx, clientAddr) + clientCoord, err := getClientLatLong(ctx, clientAddr) + if err != nil { + // If it is a geoIP error, then we get the labels and increment the error counter + // Otherwise we log the error and continue + switch err := err.(type) { + case GeoIPError: + labels := err.labels + metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() + default: + log.Warningf("Error while getting the client IP address: %v", err) + } + } // For each ad, we apply the configured sort method to determine a priority weight. for idx, ad := range ads { diff --git a/director/sort_test.go b/director/sort_test.go index ae1b64fbc..77c90689c 100644 --- a/director/sort_test.go +++ b/director/sort_test.go @@ -472,7 +472,7 @@ func TestGetClientLatLong(t *testing.T) { assert.False(t, clientIpCache.Has(clientIp)) ctx := context.Background() ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord1 := getClientLatLong(ctx, clientIp) + coord1, _ := getClientLatLong(ctx, clientIp) assert.True(t, coord1.Lat <= usLatMax && coord1.Lat >= usLatMin) assert.True(t, coord1.Long <= usLongMax && coord1.Long >= usLongMin) @@ -482,7 +482,7 @@ func TestGetClientLatLong(t *testing.T) { // Get it again to make sure it's coming from the cache ctx = context.Background() ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord2 := getClientLatLong(ctx, clientIp) + coord2, _ := getClientLatLong(ctx, clientIp) assert.Equal(t, coord1.Lat, coord2.Lat) assert.Equal(t, coord1.Long, coord2.Long) assert.Contains(t, logOutput.String(), "Retrieving pre-assigned lat/long for unresolved client IP") @@ -498,7 +498,7 @@ func TestGetClientLatLong(t *testing.T) { assert.False(t, clientIpCache.Has(clientIp)) ctx := context.Background() ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord1 := getClientLatLong(ctx, clientIp) + coord1, _ := getClientLatLong(ctx, clientIp) assert.True(t, coord1.Lat <= usLatMax && coord1.Lat >= usLatMin) assert.True(t, coord1.Long <= usLongMax && coord1.Long >= usLongMin) @@ -508,7 +508,7 @@ func TestGetClientLatLong(t *testing.T) { // Get it again to make sure it's coming from the cache ctx = context.Background() ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord2 := getClientLatLong(ctx, clientIp) + coord2, _ := getClientLatLong(ctx, clientIp) assert.Equal(t, coord1.Lat, coord2.Lat) assert.Equal(t, coord1.Long, coord2.Long) assert.Contains(t, logOutput.String(), "Retrieving pre-assigned lat/long for client IP") From 0d0aa3f9af4aad9ab4e41cea3d47ee24086f6dc6 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 17:55:06 +0000 Subject: [PATCH 49/86] Use GeoIPError in updateLatLong --- director/cache_ads.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/director/cache_ads.go b/director/cache_ads.go index 9b19cc686..f0aa97bc3 100644 --- a/director/cache_ads.go +++ b/director/cache_ads.go @@ -33,6 +33,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/utils" @@ -82,7 +83,13 @@ func (f filterType) String() string { // 5. Return the updated ServerAd. The ServerAd passed in will not be modified func recordAd(ctx context.Context, sAd server_structs.ServerAd, namespaceAds *[]server_structs.NamespaceAdV2) (updatedAd server_structs.ServerAd) { if err := updateLatLong(ctx, &sAd); err != nil { - log.Debugln("Failed to lookup GeoIP coordinates for host", sAd.URL.Host) + switch err := err.(type) { + case GeoIPError: + labels := err.labels + metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() + default: + log.Debugln("Failed to lookup GeoIP coordinates for host", sAd.URL.Host) + } } if sAd.URL.String() == "" { From 063e98b307d97aa1eb3962cde7831d795c4e0aaf Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 18:21:12 +0000 Subject: [PATCH 50/86] Move project label setting up in the call stack. This allows for the removal of some context drilling --- director/cache_ads.go | 2 +- director/sort.go | 24 +++++++++--------------- director/sort_test.go | 16 ++++------------ 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/director/cache_ads.go b/director/cache_ads.go index f0aa97bc3..4ec499ef0 100644 --- a/director/cache_ads.go +++ b/director/cache_ads.go @@ -264,7 +264,7 @@ func updateLatLong(ctx context.Context, ad *server_structs.ServerAd) error { } // NOTE: If GeoIP resolution of this address fails, lat/long are set to 0.0 (the null lat/long) // This causes the server to be sorted to the end of the list whenever the Director requires distance-aware sorting. - lat, long, err := getLatLong(ctx, addr) + lat, long, err := getLatLong(addr) if err != nil { return err } diff --git a/director/sort.go b/director/sort.go index 4c9c05500..0b6f5f094 100644 --- a/director/sort.go +++ b/director/sort.go @@ -158,18 +158,16 @@ func checkOverrides(addr net.IP) (coordinate *Coordinate) { return nil } -func setProjectLabel(ctx context.Context, labels prometheus.Labels) bool { +func setProjectLabel(ctx context.Context, labels *prometheus.Labels) { project, ok := ctx.Value(ProjectContextKey{}).(string) if !ok || project == "" { - labels["proj"] = "unknown" - return false + (*labels)["proj"] = "unknown" } else { - labels["proj"] = project - return true + (*labels)["proj"] = project } } -func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64, err error) { +func getLatLong(addr netip.Addr) (lat float64, long float64, err error) { ip := net.IP(addr.AsSlice()) override := checkOverrides(ip) if override != nil { @@ -180,7 +178,7 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 labels := prometheus.Labels{ "network": "", "source": "", - "proj": "", + "proj": "", // this will be set in the setProjectLabel function } network, ok := utils.ApplyIPMask(addr.String()) @@ -191,11 +189,6 @@ func getLatLong(ctx context.Context, addr netip.Addr) (lat float64, long float64 labels["network"] = network } - ok = setProjectLabel(ctx, labels) - if !ok { - labels["source"] = "server" - } - reader := maxMindReader.Load() if reader == nil { labels["source"] = "server" @@ -246,7 +239,7 @@ func assignRandBoundedCoord(minLat, maxLat, minLong, maxLong float64) (lat, long // Given a client address, attempt to get the lat/long of the client. If the address is invalid or // the lat/long is not resolvable, assign a random location in the contiguous US. -func getClientLatLong(ctx context.Context, addr netip.Addr) (coord Coordinate, err error) { +func getClientLatLong(addr netip.Addr) (coord Coordinate, err error) { if !addr.IsValid() { log.Warningf("Unable to sort servers based on client-server distance. Invalid client IP address: %s", addr.String()) coord.Lat, coord.Long = assignRandBoundedCoord(usLatMin, usLatMax, usLongMin, usLongMax) @@ -260,7 +253,7 @@ func getClientLatLong(ctx context.Context, addr netip.Addr) (coord Coordinate, e return } - coord.Lat, coord.Long, err = getLatLong(ctx, addr) + coord.Lat, coord.Long, err = getLatLong(addr) if err != nil || (coord.Lat == 0 && coord.Long == 0) { if err != nil { log.Warningf("Error while getting the client IP address: %v", err) @@ -302,13 +295,14 @@ func sortServerAds(ctx context.Context, clientAddr netip.Addr, ads []server_stru weights := make(SwapMaps, len(ads)) sortMethod := param.Director_CacheSortMethod.GetString() // This will handle the case where the client address is invalid or the lat/long is not resolvable. - clientCoord, err := getClientLatLong(ctx, clientAddr) + clientCoord, err := getClientLatLong(clientAddr) if err != nil { // If it is a geoIP error, then we get the labels and increment the error counter // Otherwise we log the error and continue switch err := err.(type) { case GeoIPError: labels := err.labels + setProjectLabel(ctx, &labels) metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() default: log.Warningf("Error while getting the client IP address: %v", err) diff --git a/director/sort_test.go b/director/sort_test.go index 77c90689c..753ba752d 100644 --- a/director/sort_test.go +++ b/director/sort_test.go @@ -470,9 +470,7 @@ func TestGetClientLatLong(t *testing.T) { clientIp := netip.Addr{} assert.False(t, clientIpCache.Has(clientIp)) - ctx := context.Background() - ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord1, _ := getClientLatLong(ctx, clientIp) + coord1, _ := getClientLatLong(clientIp) assert.True(t, coord1.Lat <= usLatMax && coord1.Lat >= usLatMin) assert.True(t, coord1.Long <= usLongMax && coord1.Long >= usLongMin) @@ -480,9 +478,7 @@ func TestGetClientLatLong(t *testing.T) { assert.NotContains(t, logOutput.String(), "Retrieving pre-assigned lat/long") // Get it again to make sure it's coming from the cache - ctx = context.Background() - ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord2, _ := getClientLatLong(ctx, clientIp) + coord2, _ := getClientLatLong(clientIp) assert.Equal(t, coord1.Lat, coord2.Lat) assert.Equal(t, coord1.Long, coord2.Long) assert.Contains(t, logOutput.String(), "Retrieving pre-assigned lat/long for unresolved client IP") @@ -496,9 +492,7 @@ func TestGetClientLatLong(t *testing.T) { clientIp := netip.MustParseAddr("192.168.0.1") assert.False(t, clientIpCache.Has(clientIp)) - ctx := context.Background() - ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord1, _ := getClientLatLong(ctx, clientIp) + coord1, _ := getClientLatLong(clientIp) assert.True(t, coord1.Lat <= usLatMax && coord1.Lat >= usLatMin) assert.True(t, coord1.Long <= usLongMax && coord1.Long >= usLongMin) @@ -506,9 +500,7 @@ func TestGetClientLatLong(t *testing.T) { assert.NotContains(t, logOutput.String(), "Retrieving pre-assigned lat/long") // Get it again to make sure it's coming from the cache - ctx = context.Background() - ctx = context.WithValue(ctx, ProjectContextKey{}, "pelican-client/1.0.0 project/test") - coord2, _ := getClientLatLong(ctx, clientIp) + coord2, _ := getClientLatLong(clientIp) assert.Equal(t, coord1.Lat, coord2.Lat) assert.Equal(t, coord1.Long, coord2.Long) assert.Contains(t, logOutput.String(), "Retrieving pre-assigned lat/long for client IP") From 156010f65e0cb21d671b155c66af2e930ac82b38 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 18:26:22 +0000 Subject: [PATCH 51/86] Remove unnecessary context from updateLatLong --- director/cache_ads.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/director/cache_ads.go b/director/cache_ads.go index 4ec499ef0..684fb0e01 100644 --- a/director/cache_ads.go +++ b/director/cache_ads.go @@ -82,7 +82,7 @@ func (f filterType) String() string { // 4. Set up utilities for collecting origin/health server file transfer test status // 5. Return the updated ServerAd. The ServerAd passed in will not be modified func recordAd(ctx context.Context, sAd server_structs.ServerAd, namespaceAds *[]server_structs.NamespaceAdV2) (updatedAd server_structs.ServerAd) { - if err := updateLatLong(ctx, &sAd); err != nil { + if err := updateLatLong(&sAd); err != nil { switch err := err.(type) { case GeoIPError: labels := err.labels @@ -246,7 +246,7 @@ func recordAd(ctx context.Context, sAd server_structs.ServerAd, namespaceAds *[] return sAd } -func updateLatLong(ctx context.Context, ad *server_structs.ServerAd) error { +func updateLatLong(ad *server_structs.ServerAd) error { if ad == nil { return errors.New("Cannot provide a nil ad to UpdateLatLong") } From 02daec52bbb6a4a4adbc251c52c8e7f4053d21a0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 25 Nov 2024 19:18:05 +0000 Subject: [PATCH 52/86] Fix linter --- director/sort.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/director/sort.go b/director/sort.go index 0b6f5f094..80e9ccf10 100644 --- a/director/sort.go +++ b/director/sort.go @@ -209,7 +209,7 @@ func getLatLong(addr netip.Addr) (lat float64, long float64, err error) { // comes from a private range. if lat == 0 && long == 0 { errMsg := fmt.Sprintf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) - log.Warningf(errMsg) + log.Warning(errMsg) labels["source"] = "client" err = GeoIPError{labels: labels, errorMsg: errMsg} } @@ -221,6 +221,7 @@ func getLatLong(addr netip.Addr) (lat float64, long float64, err error) { if record.Location.AccuracyRadius >= 900 { errMsg := fmt.Sprintf("GeoIP resolution of the address %s resulted in a suspiciously large accuracy radius of %d km. "+ "This will be treated as GeoIP resolution failure and result in random server sorting. Setting lat/long to null.", ip.String(), record.Location.AccuracyRadius) + log.Warning(errMsg) lat = 0 long = 0 labels["source"] = "client" From b02d3d644b8dd7706f0daa66f64227ae0a82606a Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Mon, 4 Nov 2024 19:38:58 +0000 Subject: [PATCH 53/86] Update/clarify Cache.Locations documentation --- docs/parameters.yaml | 46 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 028a661d2..dd065d425 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -1078,7 +1078,34 @@ components: ["cache"] --- name: Cache.LocalRoot description: |+ - The location where the filesystem namespace is actually rooted + A path to the directory where xrootd will create its default namespace, `meta`, and `data` directories. For example, + setting `Cache.LocalRoot=/run/pelican/cache` without specifying further `Cache.DataLocations` or `Cache.MetaLocations` + values will result in the cache creating a directory structure like: + ``` + . + └── /run/pelican/cache/ + ├── data/ + │ ├── 00 #hexadecimal values + │ ├── 01 + │ ├── ... + │ └── FF + ├── meta/ + │ ├── 00 #hexadecimal values + │ ├── 01 + │ ├── ... + │ └── FF + ├── namespace1/ + │ ├── foo1.txt --> /run/pelican/cache/data/00 + │ └── foo2.txt --> /run/pelican/cache/data/01 + └── namespace2/ + └── bar.txt --> /run/pelican/cache/data/FF + ``` + In this setup, actual data files live at `/run/pelican/cache/data` and are given hexadecimal names, while + references (symbolic links) to those files are stored in `/run/pelican/cache/namespace1`, `/run/pelican/cache/namespace2`, + etc. The `meta` directory is used for object metadata. Object requests to XRootD will be served from the namespace directories, and + resolve the underlying object through these symbolic links. + + We recommend tying the `Cache.LocalRoot` to a fast storage device, such as an SSD, to ensure optimal cache performance. type: string root_default: /run/pelican/cache default: $XDG_RUNTIME_DIR/pelican/cache @@ -1094,8 +1121,13 @@ components: ["cache"] --- name: Cache.DataLocations description: |+ - A list of directories for the locations of the cache data files - this is where the actual data in the cache is stored. - These paths should *not* be in same path as XrootD.Mount or else it will expose the data files as part of the files within the cache. + A list of filesystem paths/directories where the cache's object data will be stored. This list of directories can be used to string together + multiple storage devices to increase the cache's storage capacity, as long as each of the directories is accessible by the cache service. + For example, setting `Cache.DataLocations=["/mnt/cache1", "/mnt/cache2"]` will result in splitting cache data between two mounted drives, + `/mnt/cache1` and `/mnt/cache2`. As such, these drives should be fast storage devices, such as SSDs. + + For more information, see the [xrootd oss documentation](https://xrootd.slac.stanford.edu/doc/dev56/ofs_config.pdf) for the `oss.space` directive + as well as the [xrootd pfc documentation](https://xrootd.slac.stanford.edu/doc/dev56/pss_config.pdf) for the `pfc.spaces` directive. type: stringSlice root_default: ["/run/pelican/cache/data"] default: ["$XDG_RUNTIME_DIR/pelican/cache/data"] @@ -1103,8 +1135,12 @@ components: ["cache"] --- name: Cache.MetaLocations description: |+ - A list of directories for the locations of the cache metadata. These paths should *not* be in the same path as XrootD.Mount or else it - will expose the metadata files as part of the files within the cache + A list of filesystem paths/directories where the cache's object metadata will be stored. Values in this list may point to separate drives as long + as they're accessible by the cache service. For example, setting `Cache.MetaLocations=["/mnt/meta1", "/mnt/meta2"]` will result in + splitting cache metadata between two the mounted drives. As such, these drives should be fast storage devices, such as SSDs. + + For more information, see the [xrootd oss documentation](https://xrootd.slac.stanford.edu/doc/dev56/ofs_config.pdf) for the `oss.space` directive + as well as the [xrootd pfc documentation](https://xrootd.slac.stanford.edu/doc/dev56/pss_config.pdf) for the `pfc.spaces` directive. type: stringSlice root_default: ["/run/pelican/cache/meta"] default: ["$XDG_RUNTIME_DIR/pelican/cache/meta"] From 71968d4e9ff847b2154d0a718926a4956fc72426 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Wed, 6 Nov 2024 16:23:24 +0000 Subject: [PATCH 54/86] Specify that paths for Cache. Date: Fri, 8 Nov 2024 19:18:52 +0000 Subject: [PATCH 55/86] Correct cache namespace/data/meta location behavior This commit introduces `Cache.StorageLocation` and `Cache.NamespaceLocation`, two new variables used to correct some of the behavior around setting locations for the cache's namespace, meta, and data directories. One crucial change here is that the data/meta directories are no longer exported under the namespace directory, which previously resulted in exposing these internal file representations to users who new to look for them. This also ties the namespace/meta/data directories to `Cache.StorageLocation`, allowing you to set all three to the same directory through a single config. --- config/config.go | 46 ++++++--------- docs/parameters.yaml | 97 +++++++++++++++++++++---------- param/parameters.go | 5 +- param/parameters_struct.go | 4 ++ xrootd/resources/xrootd-cache.cfg | 2 +- xrootd/xrootd_config.go | 65 +++++++++++---------- xrootd/xrootd_config_test.go | 28 +++++++++ 7 files changed, 156 insertions(+), 91 deletions(-) diff --git a/config/config.go b/config/config.go index 245c2edba..56ec8891c 100644 --- a/config/config.go +++ b/config/config.go @@ -952,18 +952,16 @@ func SetServerDefaults(v *viper.Viper) error { if IsRootExecution() { v.SetDefault(param.Origin_RunLocation.GetName(), filepath.Join("/run", "pelican", "xrootd", "origin")) v.SetDefault(param.Cache_RunLocation.GetName(), filepath.Join("/run", "pelican", "xrootd", "cache")) - // To ensure Cache.DataLocation still works, we default Cache.LocalRoot to Cache.DataLocation - // The logic is extracted from handleDeprecatedConfig as we manually set the default value here - v.SetDefault(param.Cache_DataLocation.GetName(), "/run/pelican/cache") - v.SetDefault(param.Cache_LocalRoot.GetName(), v.GetString(param.Cache_DataLocation.GetName())) - - if v.IsSet(param.Cache_DataLocation.GetName()) { - v.SetDefault(param.Cache_DataLocations.GetName(), []string{filepath.Join(v.GetString(param.Cache_DataLocation.GetName()), "data")}) - v.SetDefault(param.Cache_MetaLocations.GetName(), []string{filepath.Join(v.GetString(param.Cache_DataLocation.GetName()), "meta")}) - } else { - v.SetDefault(param.Cache_DataLocations.GetName(), []string{"/run/pelican/cache/data"}) - v.SetDefault(param.Cache_MetaLocations.GetName(), []string{"/run/pelican/cache/meta"}) + + // Several deprecated keys point to Cache.StorageLocation, and by the time here we've already mapped those + // keys in handleDeprecatedConfig(). To prevent overriding potentially-mapped deprecated keys, we only re-set + // the default here if this key is not set. + if !v.IsSet(param.Cache_StorageLocation.GetName()) { + v.SetDefault(param.Cache_StorageLocation.GetName(), filepath.Join("/run", "pelican", "cache")) } + v.SetDefault(param.Cache_NamespaceLocation.GetName(), filepath.Join(param.Cache_StorageLocation.GetString(), "namespace")) + v.SetDefault(param.Cache_DataLocations.GetName(), []string{filepath.Join(param.Cache_StorageLocation.GetString(), "data")}) + v.SetDefault(param.Cache_MetaLocations.GetName(), []string{filepath.Join(param.Cache_StorageLocation.GetString(), "meta")}) v.SetDefault(param.LocalCache_RunLocation.GetName(), filepath.Join("/run", "pelican", "localcache")) v.SetDefault(param.Origin_Multiuser.GetName(), true) @@ -1011,18 +1009,17 @@ func SetServerDefaults(v *viper.Viper) error { } v.SetDefault(param.Origin_GlobusConfigLocation.GetName(), filepath.Join(runtimeDir, "xrootd", "origin", "globus")) - // To ensure Cache.DataLocation still works, we default Cache.LocalRoot to Cache.DataLocation - // The logic is extracted from handleDeprecatedConfig as we manually set the default value here - v.SetDefault(param.Cache_DataLocation.GetName(), filepath.Join(runtimeDir, "cache")) - v.SetDefault(param.Cache_LocalRoot.GetName(), v.GetString(param.Cache_DataLocation.GetName())) - - if v.IsSet(param.Cache_DataLocation.GetName()) { - v.SetDefault(param.Cache_DataLocations.GetName(), []string{filepath.Join(v.GetString(param.Cache_DataLocation.GetName()), "data")}) - v.SetDefault(param.Cache_MetaLocations.GetName(), []string{filepath.Join(v.GetString(param.Cache_DataLocation.GetName()), "meta")}) - } else { - v.SetDefault(param.Cache_DataLocations.GetName(), []string{filepath.Join(runtimeDir, "pelican/cache/data")}) - v.SetDefault(param.Cache_MetaLocations.GetName(), []string{filepath.Join(runtimeDir, "pelican/cache/meta")}) + + // Several deprecated keys point to Cache.StorageLocation, and by the time here we've already mapped those + // keys in handleDeprecatedConfig(). To prevent overriding potentially-mapped deprecated keys, we only re-set + // the default here if this key is not set. + if !viper.IsSet(param.Cache_StorageLocation.GetName()) { + viper.SetDefault(param.Cache_StorageLocation.GetName(), filepath.Join(runtimeDir, "cache")) } + viper.SetDefault(param.Cache_NamespaceLocation.GetName(), filepath.Join(param.Cache_StorageLocation.GetString(), "namespace")) + viper.SetDefault(param.Cache_DataLocations.GetName(), []string{filepath.Join(param.Cache_StorageLocation.GetString(), "data")}) + viper.SetDefault(param.Cache_MetaLocations.GetName(), []string{filepath.Join(param.Cache_StorageLocation.GetString(), "meta")}) + v.SetDefault(param.LocalCache_RunLocation.GetName(), filepath.Join(runtimeDir, "cache")) v.SetDefault(param.Origin_Multiuser.GetName(), false) } @@ -1154,11 +1151,6 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e } } - if param.Cache_DataLocation.IsSet() { - log.Warningf("Deprecated configuration key %s is set. Please migrate to use %s instead", param.Cache_DataLocation.GetName(), param.Cache_LocalRoot.GetName()) - log.Warningf("Will attempt to use the value of %s as default for %s", param.Cache_DataLocation.GetName(), param.Cache_LocalRoot.GetName()) - } - if err := SetServerDefaults(viper.GetViper()); err != nil { return err } diff --git a/docs/parameters.yaml b/docs/parameters.yaml index c0b11b844..e097f3cca 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -1066,59 +1066,61 @@ components: ["localcache"] ############################ # Cache-level configs # ############################ -name: Cache.DataLocation -description: |+ - [Deprecated] Cache.DataLocation is being deprecated and will be removed in a future release. It is replaced by a combination of Cache.DataLocations and Cache.MetaLocations -type: string -root_default: /run/pelican/xcache -default: $XDG_RUNTIME_DIR/pelican/xcache -deprecated: true -replacedby: Cache.LocalRoot -components: ["cache"] ---- -name: Cache.LocalRoot +name: Cache.StorageLocation description: |+ An absolute path to the directory where xrootd will create its default namespace, `meta`, and `data` directories. For example, - setting `Cache.LocalRoot=/run/pelican/cache` without specifying further `Cache.DataLocations` or `Cache.MetaLocations` + setting `Cache.StorageLocation=/run/pelican/cache` without specifying further `Cache.DataLocations` or `Cache.MetaLocations` values will result in the cache creating a directory structure like: ``` . └── /run/pelican/cache/ ├── data/ - │ ├── 00 #hexadecimal values + │ ├── 00 # hexadecimal name values │ ├── 01 │ ├── ... │ └── FF ├── meta/ - │ ├── 00 #hexadecimal values + │ ├── 00 # hexadecimal name values │ ├── 01 │ ├── ... │ └── FF - ├── namespace1/ - │ ├── foo1.txt --> /run/pelican/cache/data/00 - │ └── foo2.txt --> /run/pelican/cache/data/01 - └── namespace2/ - └── bar.txt --> /run/pelican/cache/data/FF + └── namespace/ + ├── namespace1/ + │ ├── foo1.txt --> /run/pelican/cache/data/00 + │ └── foo2.txt --> /run/pelican/cache/data/01 + └── namespace2/ + └── bar.txt --> /run/pelican/cache/data/FF ``` In this setup, actual data files live at `/run/pelican/cache/data` and are given hexadecimal names, while - references (symbolic links) to those files are stored in `/run/pelican/cache/namespace1`, `/run/pelican/cache/namespace2`, - etc. The `meta` directory is used for object metadata. Object requests to XRootD will be served from the namespace directories, and + references (symbolic links) to those files are stored in `/run/pelican/cache/namespace`. The `meta` directory + is used for object metadata. Object requests to XRootD will be served from the namespace directories, and resolve the underlying object through these symbolic links. - We recommend tying the `Cache.LocalRoot` to a fast storage device, such as an SSD, to ensure optimal cache performance. - + We recommend tying the `Cache.StorageLocation` to a fast storage device, such as an SSD, to ensure optimal cache performance. If this directory does not already exist, it will be created by Pelican. + + WARNING: The default value of /var/run/pelican should _never_ be used for production caches, as this directory is typically + cleared on system restarts, and may interfere with system services if it becomes full. Running a cache with the default value + set will generate a warning at cache startup. type: string root_default: /run/pelican/cache default: $XDG_RUNTIME_DIR/pelican/cache components: ["cache"] --- -name: Cache.ExportLocation +name: Cache.NamespaceLocation description: |+ - The location of the export directory. Everything under this directory will be exposed as part of the cache. This is - relative to the mount location. + A cache's namespace directory is used to duplicate/recreate the federation's namespace structure, and stores symbolic links from + object names to the actual data files (see `Cache.StorageLocation` for extra information). For example, requesting `/foo/bar.txt` from a + cache will check for the existence of a symbolic link at `${Cache.NamespaceLocation}/foo/bar.txt`, and if it exists, the cache will serve + the data file at the location the symbolic link points to. + + If this directory does not already exist, it will be created by Pelican. + + WARNING: It's important that any values for `Cache.DataLocations` and `Cache.MetaLocations` are NOT subdirectories of `Cache.NamespaceLocation`, + as this will make the raw data/meta files accessible through the cache's namespace structure, which is undefined behavior. type: string -default: / +default: ${Cache.StorageLocation}/namespace + components: ["cache"] --- name: Cache.DataLocations @@ -1132,9 +1134,11 @@ description: |+ as well as the [xrootd pfc documentation](https://xrootd.slac.stanford.edu/doc/dev56/pss_config.pdf) for the `pfc.spaces` directive. If this directory does not already exist, it will be created by Pelican. + + WARNING: It's important that any values for `Cache.DataLocations` are NOT subdirectories of `Cache.NamespaceLocation`, + as this will make the raw data files accessible through the cache's namespace structure, which is undefined behavior. type: stringSlice -root_default: ["/run/pelican/cache/data"] -default: ["$XDG_RUNTIME_DIR/pelican/cache/data"] +default: ["${Cache.StorageLocation}/data"] components: ["cache"] --- name: Cache.MetaLocations @@ -1147,9 +1151,40 @@ description: |+ as well as the [xrootd pfc documentation](https://xrootd.slac.stanford.edu/doc/dev56/pss_config.pdf) for the `pfc.spaces` directive. If this directory does not already exist, it will be created by Pelican. + + WARNING: It's important that any values for `Cache.MetaLocations` are NOT subdirectories of `Cache.NamespaceLocation`, + as this will make the raw metadata files accessible through the cache's namespace structure, which is undefined behavior. type: stringSlice -root_default: ["/run/pelican/cache/meta"] -default: ["$XDG_RUNTIME_DIR/pelican/cache/meta"] +default: ["${Cache.StorageLocation}/meta"] +components: ["cache"] +--- +name: Cache.LocalRoot +description: |+ + [Deprecated] Cache.LocalRoot is deprecated and replaced by Cache.StorageLocation. +type: string +root_default: /run/pelican/cache +default: $XDG_RUNTIME_DIR/pelican/cache +deprecated: true +replacedby: "Cache.StorageLocation" +components: ["cache"] +--- +name: Cache.DataLocation +description: |+ + [Deprecated] Cache.DataLocation is being deprecated and will be removed in a future release. It is replaced by Cache.StorageLocation +type: string +root_default: /run/pelican/cache +default: $XDG_RUNTIME_DIR/pelican/cache +deprecated: true +replacedby: Cache.StorageLocation +components: ["cache"] +--- +name: Cache.ExportLocation +description: |+ + A path that's relative to the `Cache.NamespaceLocation` where the cache will expose its contents. This path can be used to + control which namespaces are available through the cache. For example, setting `Cache.ExportLocation: /foo` will only expose + the `/foo` namespace to clients. +type: string +default: / components: ["cache"] --- name: Cache.RunLocation diff --git a/param/parameters.go b/param/parameters.go index f915935ef..ee8f3dbaf 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -51,7 +51,8 @@ type ObjectParam struct { func GetDeprecated() map[string][]string { return map[string][]string{ - "Cache.DataLocation": {"Cache.LocalRoot"}, + "Cache.DataLocation": {"Cache.StorageLocation"}, + "Cache.LocalRoot": {"Cache.StorageLocation"}, "Director.EnableStat": {"Director.CheckOriginPresence"}, "DisableHttpProxy": {"Client.DisableHttpProxy"}, "DisableProxyFallback": {"Client.DisableProxyFallback"}, @@ -147,8 +148,10 @@ var ( Cache_HighWaterMark = StringParam{"Cache.HighWaterMark"} Cache_LocalRoot = StringParam{"Cache.LocalRoot"} Cache_LowWatermark = StringParam{"Cache.LowWatermark"} + Cache_NamespaceLocation = StringParam{"Cache.NamespaceLocation"} Cache_RunLocation = StringParam{"Cache.RunLocation"} Cache_SentinelLocation = StringParam{"Cache.SentinelLocation"} + Cache_StorageLocation = StringParam{"Cache.StorageLocation"} Cache_Url = StringParam{"Cache.Url"} Cache_XRootDPrefix = StringParam{"Cache.XRootDPrefix"} Director_CacheSortMethod = StringParam{"Director.CacheSortMethod"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 8f9165ca1..7b6fa07a7 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -37,12 +37,14 @@ type Config struct { LocalRoot string `mapstructure:"localroot" yaml:"LocalRoot"` LowWatermark string `mapstructure:"lowwatermark" yaml:"LowWatermark"` MetaLocations []string `mapstructure:"metalocations" yaml:"MetaLocations"` + NamespaceLocation string `mapstructure:"namespacelocation" yaml:"NamespaceLocation"` PermittedNamespaces []string `mapstructure:"permittednamespaces" yaml:"PermittedNamespaces"` Port int `mapstructure:"port" yaml:"Port"` RunLocation string `mapstructure:"runlocation" yaml:"RunLocation"` SelfTest bool `mapstructure:"selftest" yaml:"SelfTest"` SelfTestInterval time.Duration `mapstructure:"selftestinterval" yaml:"SelfTestInterval"` SentinelLocation string `mapstructure:"sentinellocation" yaml:"SentinelLocation"` + StorageLocation string `mapstructure:"storagelocation" yaml:"StorageLocation"` Url string `mapstructure:"url" yaml:"Url"` XRootDPrefix string `mapstructure:"xrootdprefix" yaml:"XRootDPrefix"` } `mapstructure:"cache" yaml:"Cache"` @@ -341,12 +343,14 @@ type configWithType struct { LocalRoot struct { Type string; Value string } LowWatermark struct { Type string; Value string } MetaLocations struct { Type string; Value []string } + NamespaceLocation struct { Type string; Value string } PermittedNamespaces struct { Type string; Value []string } Port struct { Type string; Value int } RunLocation struct { Type string; Value string } SelfTest struct { Type string; Value bool } SelfTestInterval struct { Type string; Value time.Duration } SentinelLocation struct { Type string; Value string } + StorageLocation struct { Type string; Value string } Url struct { Type string; Value string } XRootDPrefix struct { Type string; Value string } } diff --git a/xrootd/resources/xrootd-cache.cfg b/xrootd/resources/xrootd-cache.cfg index 7d1cd8a19..60883b15a 100644 --- a/xrootd/resources/xrootd-cache.cfg +++ b/xrootd/resources/xrootd-cache.cfg @@ -66,7 +66,7 @@ http.tlsrequiredprefix {{$Prefix}} throttle.throttle concurrency {{.Cache.Concurrency}} {{end}} pss.origin {{.Cache.PSSOrigin}} -oss.localroot {{.Cache.LocalRoot}} +oss.localroot {{.Cache.NamespaceLocation}} pfc.spaces data meta {{- range $value := .Cache.DataLocations}} oss.space data {{$value}} diff --git a/xrootd/xrootd_config.go b/xrootd/xrootd_config.go index e0de0703f..b0553387d 100644 --- a/xrootd/xrootd_config.go +++ b/xrootd/xrootd_config.go @@ -115,7 +115,7 @@ type ( RunLocation string DataLocations []string MetaLocations []string - LocalRoot string + NamespaceLocation string PSSOrigin string BlocksToPrefetch int Concurrency int @@ -274,32 +274,32 @@ func CheckOriginXrootdEnv(exportPath string, server server_structs.XRootDServer, return nil } -func CheckCacheXrootdEnv(exportPath string, server server_structs.XRootDServer, uid int, gid int) (string, error) { - viper.Set("Xrootd.Mount", exportPath) - filepath.Join(exportPath, "/") - err := config.MkdirAll(exportPath, 0775, uid, gid) - if err != nil { - return "", errors.Wrapf(err, "Unable to create export directory %v", - filepath.Dir(exportPath)) +func CheckCacheXrootdEnv(server server_structs.XRootDServer, uid int, gid int) error { + storageLocation := param.Cache_StorageLocation.GetString() + if err := config.MkdirAll(storageLocation, 0775, uid, gid); err != nil { + return errors.Wrapf(err, "Unable to create the cache's storage directory '%s'", storageLocation) + } + // Setting Cache.StorageLocation to /run/pelican/cache is a default we use for testing, but it shouldn't ever be used + // in a production setting. If the user hasn't overridden the default, log a warning. + if storageLocation == filepath.Join("/run", "pelican", "cache") { + log.Warnf("%s is set to the default /run/pelican/cache. This default is to allow quick testing but should not be used in production.", param.Cache_StorageLocation.GetName()) } - localRoot := param.Cache_LocalRoot.GetString() - - localRoot = filepath.Clean(localRoot) - err = config.MkdirAll(localRoot, 0775, uid, gid) - - if err != nil { - return "", errors.Wrapf(err, "Unable to create local root %v", - filepath.Dir(localRoot)) + namespaceLocation := param.Cache_NamespaceLocation.GetString() + if err := config.MkdirAll(namespaceLocation, 0775, uid, gid); err != nil { + return errors.Wrapf(err, "Unable to create the cache's storage directory '%s'", storageLocation) } dataPaths := param.Cache_DataLocations.GetStringSlice() for _, dPath := range dataPaths { dataPath := filepath.Clean(dPath) - err = config.MkdirAll(dataPath, 0775, uid, gid) + // Data locations should never be below the namespace location + if strings.HasPrefix(dPath, namespaceLocation) { + return errors.Errorf("A configured data location '%s' is a subdirectory of the namespace location '%s'. Please ensure these directories are not nested.", dPath, namespaceLocation) + } - if err != nil { - return "", errors.Wrapf(err, "Unable to create data directory %v", + if err := config.MkdirAll(dataPath, 0775, uid, gid); err != nil { + return errors.Wrapf(err, "Unable to create data directory %v", filepath.Dir(dataPath)) } } @@ -307,17 +307,20 @@ func CheckCacheXrootdEnv(exportPath string, server server_structs.XRootDServer, metaPaths := param.Cache_MetaLocations.GetStringSlice() for _, mPath := range metaPaths { metaPath := filepath.Clean(mPath) - err = config.MkdirAll(metaPath, 0775, uid, gid) + // Similar to data locations, meta locations should never be below the namespace location + if strings.HasPrefix(mPath, namespaceLocation) { + return errors.Errorf("The configured meta location '%s' is a subdirectory of the namespace location '%s'. Please ensure these directories are not nested.", mPath, namespaceLocation) + } - if err != nil { - return "", errors.Wrapf(err, "Unable to create meta directory %v", + if err := config.MkdirAll(metaPath, 0775, uid, gid); err != nil { + return errors.Wrapf(err, "Unable to create meta directory %v", filepath.Dir(metaPath)) } } fedInfo, err := config.GetFederation(context.Background()) if err != nil { - return "", errors.Wrap(err, "Failed to pull information from the federation") + return errors.Wrap(err, "Failed to pull information from the federation") } if discoveryUrlStr := param.Federation_DiscoveryUrl.GetString(); discoveryUrlStr != "" { @@ -328,14 +331,14 @@ func CheckCacheXrootdEnv(exportPath string, server server_structs.XRootDServer, discoveryUrl.Host = discoveryUrl.Path discoveryUrl.Path = "" } else if discoveryUrl.Path != "" && discoveryUrl.Path != "/" { - return "", errors.New("The Federation.DiscoveryUrl's path is non-empty, ensure the Federation.DiscoveryUrl has the format :") + return errors.New("The Federation.DiscoveryUrl's path is non-empty, ensure the Federation.DiscoveryUrl has the format :") } discoveryUrl.Scheme = "pelican" discoveryUrl.Path = "" discoveryUrl.RawQuery = "" viper.Set("Cache.PSSOrigin", discoveryUrl.String()) } else { - return "", errors.Wrapf(err, "Failed to parse discovery URL %s", discoveryUrlStr) + return errors.Wrapf(err, "Failed to parse discovery URL %s", discoveryUrlStr) } } @@ -344,27 +347,27 @@ func CheckCacheXrootdEnv(exportPath string, server server_structs.XRootDServer, if err == nil { log.Debugln("Parsing director URL for 'pss.origin' setting:", directorUrlStr) if directorUrl.Path != "" && directorUrl.Path != "/" { - return "", errors.New("The Federation.DirectorUrl's path is non-empty, ensure the Federation.DirectorUrl has the format :") + return errors.New("The Federation.DirectorUrl's path is non-empty, ensure the Federation.DirectorUrl has the format :") } directorUrl.Scheme = "pelican" viper.Set("Cache.PSSOrigin", directorUrl.String()) } else { - return "", errors.Wrapf(err, "Failed to parse director URL %s", directorUrlStr) + return errors.Wrapf(err, "Failed to parse director URL %s", directorUrlStr) } } if viper.GetString("Cache.PSSOrigin") == "" { - return "", errors.New("One of Federation.DiscoveryUrl or Federation.DirectorUrl must be set to configure a cache") + return errors.New("One of Federation.DiscoveryUrl or Federation.DirectorUrl must be set to configure a cache") } if cacheServer, ok := server.(*cache.CacheServer); ok { err := WriteCacheScitokensConfig(cacheServer.GetNamespaceAds()) if err != nil { - return "", errors.Wrap(err, "Failed to create scitokens configuration for the cache") + return errors.Wrap(err, "Failed to create scitokens configuration for the cache") } } - return exportPath, nil + return nil } func CheckXrootdEnv(server server_structs.XRootDServer) error { @@ -446,7 +449,7 @@ func CheckXrootdEnv(server server_structs.XRootDServer) error { if server.GetServerType().IsEnabled(server_structs.OriginType) { err = CheckOriginXrootdEnv(exportPath, server, uid, gid, groupname) } else { - exportPath, err = CheckCacheXrootdEnv(exportPath, server, uid, gid) + err = CheckCacheXrootdEnv(server, uid, gid) } if err != nil { return err diff --git a/xrootd/xrootd_config_test.go b/xrootd/xrootd_config_test.go index 1e192451b..235ad53a3 100644 --- a/xrootd/xrootd_config_test.go +++ b/xrootd/xrootd_config_test.go @@ -38,6 +38,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" + "github.com/pelicanplatform/pelican/cache" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/origin" "github.com/pelicanplatform/pelican/param" @@ -626,6 +627,33 @@ func TestXrootDCacheConfig(t *testing.T) { assert.NoError(t, err) assert.NotContains(t, string(content), "http.tlsrequiredprefix") }) + + t.Run("TestNestedDataMetaNamespace", func(t *testing.T) { + testDir := t.TempDir() + viper.Set("Cache.StorageLocation", testDir) + namespaceLocation := filepath.Join(testDir, "namespace") + viper.Set("Cache.NamespaceLocation", namespaceLocation) + + cache := &cache.CacheServer{} + uid := os.Getuid() + gid := os.Getgid() + + // Data location test + nestedDataLocation := filepath.Join(namespaceLocation, "data") + viper.Set("Cache.DataLocations", []string{nestedDataLocation}) + err := CheckCacheXrootdEnv(cache, uid, gid) + require.Error(t, err) + require.Contains(t, err.Error(), "Please ensure these directories are not nested.") + // Now set to a valid location so we can hit the meta error in the next part of the test + viper.Set("Cache.DataLocations", []string{filepath.Join(testDir, "data")}) + + // Meta location test + nestedMetaLocation := filepath.Join(namespaceLocation, "meta") + viper.Set("Cache.MetaLocations", []string{nestedMetaLocation}) + err = CheckCacheXrootdEnv(cache, uid, gid) + require.Error(t, err) + require.Contains(t, err.Error(), "Please ensure these directories are not nested.") + }) } func TestUpdateAuth(t *testing.T) { From 20fbc9367c13ab2b2b3aa92328b157005050dcde Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 26 Nov 2024 21:34:07 +0000 Subject: [PATCH 56/86] Use updated params throughout tests Turns out some tests don't call init config, so they weren't handling the deprecated config. This updates the test to use the new params so they set the correct values. --- cache/cache_api.go | 8 ++++---- cache/cache_api_test.go | 4 ++-- cache/self_monitor.go | 8 ++++---- client/fed_test.go | 2 +- cmd/fed_serve_cache_test.go | 2 +- fed_test_utils/fed.go | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cache/cache_api.go b/cache/cache_api.go index ae4f38823..a47f18ce7 100644 --- a/cache/cache_api.go +++ b/cache/cache_api.go @@ -37,7 +37,7 @@ import ( func CheckCacheSentinelLocation() error { if param.Cache_SentinelLocation.IsSet() { sentinelPath := param.Cache_SentinelLocation.GetString() - dataLoc := param.Cache_LocalRoot.GetString() + dataLoc := param.Cache_StorageLocation.GetString() sentinelPath = path.Clean(sentinelPath) if path.Base(sentinelPath) != sentinelPath { return errors.Errorf("invalid Cache.SentinelLocation path. File must not contain a directory. Got %s", sentinelPath) @@ -51,17 +51,17 @@ func CheckCacheSentinelLocation() error { return nil } -// Periodically scan the //pelican/monitoring directory to clean up test files +// Periodically scan the ${Cache.StorageLocation}/pelican/monitoring directory to clean up test files // TODO: Director test files should be under /pelican/monitoring/directorTest and the file names // should have director-test- as the prefix func LaunchDirectorTestFileCleanup(ctx context.Context) { server_utils.LaunchWatcherMaintenance(ctx, - []string{filepath.Join(param.Cache_LocalRoot.GetString(), "pelican", "monitoring")}, + []string{filepath.Join(param.Cache_StorageLocation.GetString(), "pelican", "monitoring")}, "cache director-based health test clean up", time.Minute, func(notifyEvent bool) error { // We run this function regardless of notifyEvent to do the cleanup - dirPath := filepath.Join(param.Cache_LocalRoot.GetString(), "pelican", "monitoring") + dirPath := filepath.Join(param.Cache_StorageLocation.GetString(), "pelican", "monitoring") dirInfo, err := os.Stat(dirPath) if err != nil { return err diff --git a/cache/cache_api_test.go b/cache/cache_api_test.go index 22694622f..a9ce4c318 100644 --- a/cache/cache_api_test.go +++ b/cache/cache_api_test.go @@ -50,7 +50,7 @@ func TestCheckCacheSentinelLocation(t *testing.T) { tmpDir := t.TempDir() server_utils.ResetTestState() viper.Set(param.Cache_SentinelLocation.GetName(), "test.txt") - viper.Set(param.Cache_LocalRoot.GetName(), tmpDir) + viper.Set(param.Cache_StorageLocation.GetName(), tmpDir) err := CheckCacheSentinelLocation() require.Error(t, err) assert.Contains(t, err.Error(), "failed to open Cache.SentinelLocation") @@ -61,7 +61,7 @@ func TestCheckCacheSentinelLocation(t *testing.T) { server_utils.ResetTestState() viper.Set(param.Cache_SentinelLocation.GetName(), "test.txt") - viper.Set(param.Cache_LocalRoot.GetName(), tmpDir) + viper.Set(param.Cache_StorageLocation.GetName(), tmpDir) file, err := os.Create(filepath.Join(tmpDir, "test.txt")) require.NoError(t, err) diff --git a/cache/self_monitor.go b/cache/self_monitor.go index 0bd94706d..44a801dd5 100644 --- a/cache/self_monitor.go +++ b/cache/self_monitor.go @@ -59,7 +59,7 @@ func InitSelfTestDir() error { return err } - basePath := param.Cache_LocalRoot.GetString() + basePath := param.Cache_StorageLocation.GetString() pelicanMonPath := filepath.Join(basePath, "/pelican") monitoringPath := filepath.Join(pelicanMonPath, "/monitoring") selfTestPath := filepath.Join(monitoringPath, "/selfTest") @@ -80,9 +80,9 @@ func InitSelfTestDir() error { } func generateTestFile() (string, error) { - basePath := param.Cache_LocalRoot.GetString() + basePath := param.Cache_StorageLocation.GetString() if basePath == "" { - return "", errors.New("failed to generate self-test file for cache: Cache.LocalRoot is not set.") + return "", errors.New("failed to generate self-test file for cache: Cache.StorageLocation is not set.") } selfTestPath := filepath.Join(basePath, selfTestDir) _, err := os.Stat(selfTestPath) @@ -225,7 +225,7 @@ func downloadTestFile(ctx context.Context, fileUrl string) error { } func deleteTestFile(fileUrlStr string) error { - basePath := param.Cache_LocalRoot.GetString() + basePath := param.Cache_StorageLocation.GetString() fileUrl, err := url.Parse(fileUrlStr) if err != nil { return errors.Wrap(err, "invalid file url to remove the test file") diff --git a/client/fed_test.go b/client/fed_test.go index a066177fc..17b6cfb5f 100644 --- a/client/fed_test.go +++ b/client/fed_test.go @@ -623,7 +623,7 @@ func TestDirectReads(t *testing.T) { assert.Equal(t, transferResults[0].TransferredBytes, int64(17)) // Assert that the file was not cached - cacheDataLocation := param.Cache_LocalRoot.GetString() + export.FederationPrefix + cacheDataLocation := param.Cache_StorageLocation.GetString() + export.FederationPrefix filepath := filepath.Join(cacheDataLocation, filepath.Base(tempFile.Name())) _, err = os.Stat(filepath) assert.True(t, os.IsNotExist(err)) diff --git a/cmd/fed_serve_cache_test.go b/cmd/fed_serve_cache_test.go index 22a7b29c5..359c3b70d 100644 --- a/cmd/fed_serve_cache_test.go +++ b/cmd/fed_serve_cache_test.go @@ -77,7 +77,7 @@ func TestFedServeCache(t *testing.T) { viper.Set("ConfigDir", tmpPath) viper.Set("Origin.RunLocation", filepath.Join(tmpPath, "xOrigin")) viper.Set("Cache.RunLocation", filepath.Join(tmpPath, "xCache")) - viper.Set("Cache.LocalRoot", filepath.Join(tmpPath, "data")) + viper.Set("Cache.StorageLocation", filepath.Join(tmpPath, "data")) viper.Set("Origin.StoragePrefix", filepath.Join(origPath, "ns")) viper.Set("Origin.FederationPrefix", "/test") testFilePath := filepath.Join(origPath, "ns", "test-file.txt") diff --git a/fed_test_utils/fed.go b/fed_test_utils/fed.go index 300ffe27d..15973fab2 100644 --- a/fed_test_utils/fed.go +++ b/fed_test_utils/fed.go @@ -142,7 +142,7 @@ func NewFedTest(t *testing.T, originConfig string) (ft *FedTest) { viper.Set("Server.WebPort", 0) viper.Set("Origin.RunLocation", filepath.Join(tmpPath, "origin")) viper.Set("Cache.RunLocation", filepath.Join(tmpPath, "cache")) - viper.Set("Cache.LocalRoot", filepath.Join(tmpPath, "xcache-data")) + viper.Set("Cache.StorageLocation", filepath.Join(tmpPath, "xcache-data")) viper.Set("LocalCache.RunLocation", filepath.Join(tmpPath, "local-cache")) viper.Set("Registry.RequireOriginApproval", false) viper.Set("Registry.RequireCacheApproval", false) From fe03cc4ed0646768794672c5c0d2ac62cc69a4cf Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 26 Nov 2024 21:34:50 +0000 Subject: [PATCH 57/86] Use require.Error in test instead of assert.Error to prevent nil pointer deref Another issue fixed by a different commit exposed that failure to stop the test on this error generates a nil pointer dereference when we try to check the contents of a non-existent error. Using require.Error stops the test before reaching the dereference. --- local_cache/cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_cache/cache_test.go b/local_cache/cache_test.go index 04963cda7..5560f6d5a 100644 --- a/local_cache/cache_test.go +++ b/local_cache/cache_test.go @@ -330,7 +330,7 @@ func TestClient(t *testing.T) { _, err = client.DoGet(ctx, "pelican://"+param.Server_Hostname.GetString()+":"+strconv.Itoa(param.Server_WebPort.GetInt())+"/test/hello_world.txt.1", filepath.Join(tmpDir, "hello_world.txt.1"), false, client.WithToken(token), client.WithCaches(cacheUrl), client.WithAcquireToken(false)) - assert.Error(t, err) + require.Error(t, err) assert.Equal(t, "failed download from local-cache: server returned 404 Not Found", err.Error()) }) t.Cleanup(func() { From 3f979abdc334e2d5e98de58f9409bbcb44a49605 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Wed, 27 Nov 2024 17:03:05 +0000 Subject: [PATCH 58/86] Add small revisions to comments/docs to enhance concept clarity --- config/config.go | 12 ++++++------ docs/parameters.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index 56ec8891c..a51d53697 100644 --- a/config/config.go +++ b/config/config.go @@ -953,9 +953,9 @@ func SetServerDefaults(v *viper.Viper) error { v.SetDefault(param.Origin_RunLocation.GetName(), filepath.Join("/run", "pelican", "xrootd", "origin")) v.SetDefault(param.Cache_RunLocation.GetName(), filepath.Join("/run", "pelican", "xrootd", "cache")) - // Several deprecated keys point to Cache.StorageLocation, and by the time here we've already mapped those - // keys in handleDeprecatedConfig(). To prevent overriding potentially-mapped deprecated keys, we only re-set - // the default here if this key is not set. + // Several deprecated keys point to Cache.StorageLocation, and by the time we reach this section of code, we should + // have already mapped those keys in handleDeprecatedConfig(). To prevent overriding potentially-mapped deprecated keys, + // we only re-set he default here if this key is not set. if !v.IsSet(param.Cache_StorageLocation.GetName()) { v.SetDefault(param.Cache_StorageLocation.GetName(), filepath.Join("/run", "pelican", "cache")) } @@ -1010,9 +1010,9 @@ func SetServerDefaults(v *viper.Viper) error { v.SetDefault(param.Origin_GlobusConfigLocation.GetName(), filepath.Join(runtimeDir, "xrootd", "origin", "globus")) - // Several deprecated keys point to Cache.StorageLocation, and by the time here we've already mapped those - // keys in handleDeprecatedConfig(). To prevent overriding potentially-mapped deprecated keys, we only re-set - // the default here if this key is not set. + // Several deprecated keys point to Cache.StorageLocation, and by the time we reach this section of code, we should + // have already mapped those keys in handleDeprecatedConfig(). To prevent overriding potentially-mapped deprecated keys, + // we only re-set he default here if this key is not set. if !viper.IsSet(param.Cache_StorageLocation.GetName()) { viper.SetDefault(param.Cache_StorageLocation.GetName(), filepath.Join(runtimeDir, "cache")) } diff --git a/docs/parameters.yaml b/docs/parameters.yaml index e097f3cca..7920c532d 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -1068,7 +1068,7 @@ components: ["localcache"] ############################ name: Cache.StorageLocation description: |+ - An absolute path to the directory where xrootd will create its default namespace, `meta`, and `data` directories. For example, + An absolute path to the directory where xrootd will create its default `namespace`, `meta`, and `data` directories. For example, setting `Cache.StorageLocation=/run/pelican/cache` without specifying further `Cache.DataLocations` or `Cache.MetaLocations` values will result in the cache creating a directory structure like: ``` From 7f72c4b0261989988bc69d02a679021879a48b05 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 27 Nov 2024 10:53:23 -0600 Subject: [PATCH 59/86] Make xrootd startup wait time a tunable value If xrootd is running under valgrind, the startup time may be much more than 10s. Provide a hidden tunable to give valgrind more time. --- config/resources/defaults.yaml | 1 + docs/parameters.yaml | 9 +++++++++ param/parameters.go | 1 + param/parameters_struct.go | 2 ++ xrootd/launch.go | 6 +++--- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/config/resources/defaults.yaml b/config/resources/defaults.yaml index 8f4d49653..cd49f4a5b 100644 --- a/config/resources/defaults.yaml +++ b/config/resources/defaults.yaml @@ -100,6 +100,7 @@ Shoveler: PortHigher: 9999 AMQPExchange: shoveled-xrd Xrootd: + MaxStartupWait: "10s" Mount: "" ManagerPort: 1213 DetailedMonitoringPort: 9930 diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 028a661d2..acc62cc8a 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -2278,6 +2278,15 @@ type: string default: none components: ["origin"] --- +name: Xrootd.MaxStartupWait +description: |+ + The maximum amount of time pelican will wait for the xrootd daemons to + successfully start +type: duration +default: 10s +hidden: true +components: ["origin", "cache"] +--- ############################ # Monitoring-level configs # ############################ diff --git a/param/parameters.go b/param/parameters.go index f915935ef..cade9bfde 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -401,6 +401,7 @@ var ( Transport_ResponseHeaderTimeout = DurationParam{"Transport.ResponseHeaderTimeout"} Transport_TLSHandshakeTimeout = DurationParam{"Transport.TLSHandshakeTimeout"} Xrootd_AuthRefreshInterval = DurationParam{"Xrootd.AuthRefreshInterval"} + Xrootd_MaxStartupWait = DurationParam{"Xrootd.MaxStartupWait"} ) var ( diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 8f9165ca1..b7b70845e 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -315,6 +315,7 @@ type Config struct { MacaroonsKeyFile string `mapstructure:"macaroonskeyfile" yaml:"MacaroonsKeyFile"` ManagerHost string `mapstructure:"managerhost" yaml:"ManagerHost"` ManagerPort int `mapstructure:"managerport" yaml:"ManagerPort"` + MaxStartupWait time.Duration `mapstructure:"maxstartupwait" yaml:"MaxStartupWait"` Mount string `mapstructure:"mount" yaml:"Mount"` Port int `mapstructure:"port" yaml:"Port"` RobotsTxtFile string `mapstructure:"robotstxtfile" yaml:"RobotsTxtFile"` @@ -619,6 +620,7 @@ type configWithType struct { MacaroonsKeyFile struct { Type string; Value string } ManagerHost struct { Type string; Value string } ManagerPort struct { Type string; Value int } + MaxStartupWait struct { Type string; Value time.Duration } Mount struct { Type string; Value string } Port struct { Type string; Value int } RobotsTxtFile struct { Type string; Value string } diff --git a/xrootd/launch.go b/xrootd/launch.go index 78de57ff6..b404c0e69 100644 --- a/xrootd/launch.go +++ b/xrootd/launch.go @@ -165,7 +165,7 @@ func LaunchDaemons(ctx context.Context, launchers []daemon.Launcher, egrp *errgr return } - ticker := time.NewTicker(10 * time.Second) + ticker := time.NewTicker(param.Xrootd_MaxStartupWait.GetDuration()) defer ticker.Stop() select { case <-ctx.Done(): @@ -180,8 +180,8 @@ func LaunchDaemons(ctx context.Context, launchers []daemon.Launcher, egrp *errgr portStartCallback(port) } case <-ticker.C: - log.Errorln("XRootD did not startup after 10s of waiting") - err = errors.New("XRootD did not startup after 10s of waiting") + log.Errorln("XRootD did not startup after", param.Xrootd_MaxStartupWait.GetDuration().String(), "of waiting") + err = errors.New("XRootD did not startup after " + param.Xrootd_MaxStartupWait.GetDuration().String() + " of waiting") return } From 62a2854a17b3573870004a1adeba438ceb00353a Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 27 Nov 2024 10:55:22 -0600 Subject: [PATCH 60/86] Create a tunable for default cache timeout Under valgrind, its trivial to make the default cache timeouts hit frequently - make this adjustable so stress tests can succeed. Additionally, add a few helper RPMs for debugging under valgrind. --- config/resources/defaults.yaml | 1 + docs/parameters.yaml | 10 ++++++++++ images/dev.Dockerfile | 1 + param/parameters.go | 1 + param/parameters_struct.go | 2 ++ xrootd/launch.go | 2 ++ 6 files changed, 17 insertions(+) diff --git a/config/resources/defaults.yaml b/config/resources/defaults.yaml index cd49f4a5b..2e5216874 100644 --- a/config/resources/defaults.yaml +++ b/config/resources/defaults.yaml @@ -60,6 +60,7 @@ Director: CachePresenceTTL: 1m CachePresenceCapacity: 10000 Cache: + DefaultCacheTimeout: "9.5s" Port: 8442 SelfTest: true SelfTestInterval: 15s diff --git a/docs/parameters.yaml b/docs/parameters.yaml index acc62cc8a..6e2b99a74 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -1249,6 +1249,16 @@ type: int default: 0 components: ["cache"] --- +name: Cache.DefaultCacheTimeout +description: |+ + The default value of the cache operation timeout if one is not specified by the client. + + Newer clients should always specify a timeout; changing this default is rarely necessary. +type: duration +default: 9.5s +hidden: true +components: ["cache"] +--- ############################ # Director-level configs # ############################ diff --git a/images/dev.Dockerfile b/images/dev.Dockerfile index d048f6efb..7ca7cf8e1 100644 --- a/images/dev.Dockerfile +++ b/images/dev.Dockerfile @@ -56,6 +56,7 @@ RUN yum install -y --enablerepo=osg-testing goreleaser npm xrootd-devel xrootd-s xrdcl-http jq procps docker make curl-devel java-17-openjdk-headless git cmake3 gcc-c++ openssl-devel sqlite-devel libcap-devel sssd-client \ xrootd-multiuser \ zlib-devel \ + vim valgrind gdb gtest-devel \ && yum clean all # The ADD command with a api.github.com URL in the next couple of sections diff --git a/param/parameters.go b/param/parameters.go index cade9bfde..73f624ead 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -378,6 +378,7 @@ var ( ) var ( + Cache_DefaultCacheTimeout = DurationParam{"Cache.DefaultCacheTimeout"} Cache_SelfTestInterval = DurationParam{"Cache.SelfTestInterval"} Client_SlowTransferRampupTime = DurationParam{"Client.SlowTransferRampupTime"} Client_SlowTransferWindow = DurationParam{"Client.SlowTransferWindow"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index b7b70845e..9b64899fb 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -29,6 +29,7 @@ type Config struct { Concurrency int `mapstructure:"concurrency" yaml:"Concurrency"` DataLocation string `mapstructure:"datalocation" yaml:"DataLocation"` DataLocations []string `mapstructure:"datalocations" yaml:"DataLocations"` + DefaultCacheTimeout time.Duration `mapstructure:"defaultcachetimeout" yaml:"DefaultCacheTimeout"` EnableLotman bool `mapstructure:"enablelotman" yaml:"EnableLotman"` EnableOIDC bool `mapstructure:"enableoidc" yaml:"EnableOIDC"` EnableVoms bool `mapstructure:"enablevoms" yaml:"EnableVoms"` @@ -334,6 +335,7 @@ type configWithType struct { Concurrency struct { Type string; Value int } DataLocation struct { Type string; Value string } DataLocations struct { Type string; Value []string } + DefaultCacheTimeout struct { Type string; Value time.Duration } EnableLotman struct { Type string; Value bool } EnableOIDC struct { Type string; Value bool } EnableVoms struct { Type string; Value bool } diff --git a/xrootd/launch.go b/xrootd/launch.go index b404c0e69..78416b438 100644 --- a/xrootd/launch.go +++ b/xrootd/launch.go @@ -91,6 +91,8 @@ func makeUnprivilegedXrootdLauncher(daemonName string, configPath string, isCach if confDir := os.Getenv("XRD_PLUGINCONFDIR"); confDir != "" { result.ExtraEnv = append(result.ExtraEnv, "XRD_PLUGINCONFDIR="+confDir) } + result.ExtraEnv = append(result.ExtraEnv, "XRD_PELICANFEDERATIONMETADATATIMEOUT="+param.Cache_DefaultCacheTimeout.GetDuration().String()) + result.ExtraEnv = append(result.ExtraEnv, "XRD_PELICANDEFAULTHEADERTIMEOUT="+param.Cache_DefaultCacheTimeout.GetDuration().String()) } return } From 37b41e2549f8cab43ca0865eb8ba56a8b340cce5 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Wed, 27 Nov 2024 20:21:30 +0000 Subject: [PATCH 61/86] Add GHA that enforces labeling and linking in PRs --- .github/workflows/enforce-PR-labelling.yml | 55 ++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/enforce-PR-labelling.yml diff --git a/.github/workflows/enforce-PR-labelling.yml b/.github/workflows/enforce-PR-labelling.yml new file mode 100644 index 000000000..2c935c634 --- /dev/null +++ b/.github/workflows/enforce-PR-labelling.yml @@ -0,0 +1,55 @@ +name: PR and Issue Validation + +on: + pull_request: + # one limitation here is that there's no trigger to re-run any time we "connect" or "disconnect" an issue + types: [opened, edited, labeled, unlabeled, synchronize] + +jobs: + validate-pr: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v2 + + - name: Validate PR has labels + id: check_labels + run: | + PR_LABELS=$(jq -r '.pull_request.labels | length' $GITHUB_EVENT_PATH) + if [ "$PR_LABELS" -eq "0" ]; then + echo "No labels found on the pull request." + exit 1 + fi + + - name: Validate PR is linked to an issue + id: check_linked_issues + run: | + PR_NUMBER=$(jq -r '.pull_request.number' $GITHUB_EVENT_PATH) + REPO_OWNER=$(jq -r '.repository.owner.login' $GITHUB_EVENT_PATH) + REPO_NAME=$(jq -r '.repository.name' $GITHUB_EVENT_PATH) + TIMELINE_JSON=$(curl -s "https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/issues/$PR_NUMBER/timeline") + + # Count the number of times the timeline sees a "connected" event and subract the number of "disconnected" events + # We might also consider using the "cross-referenced" event in the future if actual connecting/disconnecting is too heavy-handed + LINKED_ISSUES=$(echo "$TIMELINE_JSON" | jq ' + reduce .[] as $event ( + 0; + if $event.event == "connected" then + . + 1 + elif $event.event == "disconnected" then + . - 1 + else + . + end + )') + + # If the sum is 0, then no linked issues were found + if [ "$LINKED_ISSUES" -eq "0" ]; then + echo "❌ No linked issues found in the pull request." + exit 1 + elif [ "$LINKED_ISSUES" -lt "0" ]; then + echo "Error: More disconnected events than connected events. This shouldn't be possible and likely indicates a big ol' 🪲" + exit 1 + else + echo "Linked issues found: $LINKED_ISSUES" + fi From f95e553e9dcff82bea4ee909b300a76173b27675 Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Fri, 8 Nov 2024 22:50:36 +0000 Subject: [PATCH 62/86] Print config at server startup after setting defaults --- launchers/launcher.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/launchers/launcher.go b/launchers/launcher.go index c8868b498..6206ac4f2 100644 --- a/launchers/launcher.go +++ b/launchers/launcher.go @@ -60,13 +60,6 @@ func LaunchModules(ctx context.Context, modules server_structs.ServerType) (serv config.LogPelicanVersion() - // Print Pelican config at server start if it's in debug or info level - if log.GetLevel() >= log.InfoLevel { - if err = config.PrintConfig(); err != nil { - return - } - } - egrp.Go(func() error { _ = config.RestartFlag log.Debug("Will shutdown process on signal") @@ -97,6 +90,13 @@ func LaunchModules(ctx context.Context, modules server_structs.ServerType) (serv return } + // Print Pelican config at server start if it's in debug or info level + if log.GetLevel() >= log.InfoLevel { + if err = config.PrintConfig(); err != nil { + return + } + } + // Set up necessary APIs to support Web UI, including auth and metrics if err = web_ui.ConfigureServerWebAPI(ctx, engine, egrp); err != nil { return From e5c2e7a77e090135a97654a3b9af7042c92603ae Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Sat, 9 Nov 2024 22:42:53 +0000 Subject: [PATCH 63/86] Add debug-level config print in InitClient --- config/config.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/config.go b/config/config.go index a51d53697..fc4ade31d 100644 --- a/config/config.go +++ b/config/config.go @@ -1515,6 +1515,13 @@ func InitClient() error { clientInitialized = true + // Print Pelican configuration after client config initialization if log level is set to debug + if log.GetLevel() == log.DebugLevel { + if err = PrintConfig(); err != nil { + return err + } + } + return nil } From eb99b23e01c4a55d15df81adf57d986e8ad43bc8 Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Fri, 15 Nov 2024 21:22:48 +0000 Subject: [PATCH 64/86] Add component based config filtering --- config/config.go | 112 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index fc4ade31d..2d5c123e7 100644 --- a/config/config.go +++ b/config/config.go @@ -32,6 +32,7 @@ import ( "net/url" "os" "path/filepath" + "reflect" "slices" "sort" "strconv" @@ -50,6 +51,7 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" + "github.com/pelicanplatform/pelican/docs" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/pelican_url" "github.com/pelicanplatform/pelican/server_structs" @@ -171,7 +173,8 @@ var ( "": true, } - clientInitialized = false + clientInitialized = false + printClientConfigOnce sync.Once ) func init() { @@ -922,6 +925,100 @@ func PrintConfig() error { return nil } +func contains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} + +// GetComponentConfig filters the full config and returns only the config parameters related to the given component. +// The filtering is based on whether the given component is part of the components in docs.parameters.yaml. +func GetComponentConfig(component string) (map[string]interface{}, error) { + rawConfig, err := param.UnmarshalConfig(viper.GetViper()) + if err != nil { + return nil, err + } + value, hasValue := filterConfigRecursive(reflect.ValueOf(rawConfig), "", component) + if hasValue { + return (*value).(map[string]interface{}), nil + } + return nil, nil +} + +// filterConfigRecursive is a helper function for GetComponentConfig. +// It recursively creates a nested config map of the parameters that relate to the given component. +func filterConfigRecursive(v reflect.Value, currentPath string, component string) (*interface{}, bool) { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + switch v.Kind() { + case reflect.Struct: + t := v.Type() + result := make(map[string]interface{}) + hasField := false + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + fieldType := t.Field(i) + if !fieldType.IsExported() { + continue + } + + fieldName := strings.ToLower(fieldType.Name) + + var newPath string + if currentPath == "" { + newPath = fieldName + } else { + newPath = currentPath + "." + fieldName + } + + fieldValue, fieldHasValue := filterConfigRecursive(field, newPath, component) + if fieldHasValue && fieldValue != nil { + result[fieldName] = *fieldValue + hasField = true + } + } + if hasField { + resultInterface := interface{}(result) + return &resultInterface, true + } + return nil, false + default: + lowerPath := strings.ToLower(currentPath) + paramDoc, exists := docs.ParsedParameters[lowerPath] + if exists && contains(paramDoc.Components, component) { + resultValue := v.Interface() + resultInterface := interface{}(resultValue) + return &resultInterface, true + } + return nil, false + } +} + +// PrintClientConfig prints the client config in JSON format to stderr. +func PrintClientConfig() error { + + clientConfig, err := GetComponentConfig("client") + if err != nil { + return err + } + + bytes, err := json.MarshalIndent(clientConfig, "", " ") + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, + "================ Pelican Client Configuration ================\n", + string(bytes), + "\n", + "============= End of Pelican Client Configuration ============") + return nil +} + func SetServerDefaults(v *viper.Viper) error { configDir := v.GetString("ConfigDir") v.SetConfigType("yaml") @@ -1515,11 +1612,16 @@ func InitClient() error { clientInitialized = true - // Print Pelican configuration after client config initialization if log level is set to debug - if log.GetLevel() == log.DebugLevel { - if err = PrintConfig(); err != nil { - return err + var printClientConfigErr error + printClientConfigOnce.Do(func() { + if log.GetLevel() == log.DebugLevel { + printClientConfigErr = PrintClientConfig() } + }) + + // Return any error encountered during PrintClientConfig + if printClientConfigErr != nil { + return printClientConfigErr } return nil From 6412cc4a60e0fb21df72622fcbd91fc729711a0f Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Mon, 18 Nov 2024 17:05:26 -0600 Subject: [PATCH 65/86] Remove extra new line --- config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.go b/config/config.go index 2d5c123e7..353cdec28 100644 --- a/config/config.go +++ b/config/config.go @@ -1001,7 +1001,6 @@ func filterConfigRecursive(v reflect.Value, currentPath string, component string // PrintClientConfig prints the client config in JSON format to stderr. func PrintClientConfig() error { - clientConfig, err := GetComponentConfig("client") if err != nil { return err From 9d307aba66b55b426a6fe8f8d16dc5ae7967436f Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Wed, 27 Nov 2024 20:52:33 +0000 Subject: [PATCH 66/86] Add issue labelling enforcement --- .github/workflows/enforce-PR-labelling.yml | 2 +- .github/workflows/enforce-issue-labelling.yml | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/enforce-issue-labelling.yml diff --git a/.github/workflows/enforce-PR-labelling.yml b/.github/workflows/enforce-PR-labelling.yml index 2c935c634..28fa6885f 100644 --- a/.github/workflows/enforce-PR-labelling.yml +++ b/.github/workflows/enforce-PR-labelling.yml @@ -1,4 +1,4 @@ -name: PR and Issue Validation +name: PR Validation on: pull_request: diff --git a/.github/workflows/enforce-issue-labelling.yml b/.github/workflows/enforce-issue-labelling.yml new file mode 100644 index 000000000..113fccd28 --- /dev/null +++ b/.github/workflows/enforce-issue-labelling.yml @@ -0,0 +1,32 @@ +name: Issue Validation + +on: + issues: + types: [closed] + +jobs: + validate-issue: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v2 + + - name: Validate issue has labels + id: check_labels + run: | + ISSUE_LABELS=$(jq -r '.issue.labels | length' $GITHUB_EVENT_PATH) + if [ "$ISSUE_LABELS" -eq "0" ]; then + echo "No labels found on the issue." + # Re-open the issue + ISSUE_NUMBER=$(jq -r '.issue.number' $GITHUB_EVENT_PATH) + REPO_OWNER=$(jq -r '.repository.owner.login' $GITHUB_EVENT_PATH) + REPO_NAME=$(jq -r '.repository.name' $GITHUB_EVENT_PATH) + curl -L \ + -X PATCH \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/issues/$ISSUE_NUMBER \ + -d '{"state":"open"}' + exit 1 + fi From 40d2e82858e76ce23c9ab62cb589c6c93b97d0d4 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Wed, 27 Nov 2024 15:11:46 -0600 Subject: [PATCH 67/86] Toggle dl webhook on release --- .github/workflows/post-release.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/post-release.yaml diff --git a/.github/workflows/post-release.yaml b/.github/workflows/post-release.yaml new file mode 100644 index 000000000..e6c048f71 --- /dev/null +++ b/.github/workflows/post-release.yaml @@ -0,0 +1,14 @@ +# Toggle webhook to pull latest release onto dl.pelicanplatform.org +name: post-release + +on: + release: + types: [published] + +jobs: + toggle-webhook: + runs-on: ubuntu-latest + steps: + - name: Toggle Webhook + run: | + curl -X POST https://dl.pelicanplatform.org/api/api/hooks/release-download-toggle From 9a032bc57b516a692ea8161047c7e3f8321331db Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Mon, 2 Dec 2024 16:41:51 -0600 Subject: [PATCH 68/86] Note the download repo in the documentation --- docs/pages/install.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/install.mdx b/docs/pages/install.mdx index 15b826f25..e0e221240 100644 --- a/docs/pages/install.mdx +++ b/docs/pages/install.mdx @@ -1,9 +1,14 @@ import DownloadsComponent from "/components/DownloadsComponent"; +import { Callout } from 'nextra/components' # Install Pelican This document lists Pelican's operating system requirements and explains how you can download and install the correct Pelican executable. + + Know what you want? Visit our [Downloads Repository](https://dl.pelicanplatform.org). + + ## Before Starting Pelican executables can run as a **client** or a **server**, and both are packaged in the same executable. However, if you intend to run a server, some non-RPM installations may require additional package dependencies. From d7fe07a577903d6a1515590499580042ee99a274 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Tue, 3 Dec 2024 13:05:08 -0600 Subject: [PATCH 69/86] Remove Broken graphs in favor of metrics page --- web_ui/frontend/app/cache/page.tsx | 117 ---------------------------- web_ui/frontend/app/origin/page.tsx | 56 ------------- 2 files changed, 173 deletions(-) diff --git a/web_ui/frontend/app/cache/page.tsx b/web_ui/frontend/app/cache/page.tsx index ad12e417a..6f9b4618d 100644 --- a/web_ui/frontend/app/cache/page.tsx +++ b/web_ui/frontend/app/cache/page.tsx @@ -39,123 +39,6 @@ export default function Home() { - - - - Storage - - - console.log(chart) - }, - }, - }, - }} - datasetOptions={[ - { - label: 'Total Storage (Gigabytes)', - borderColor: '#000000', - }, - { label: 'Free Storage (Gigabytes)', borderColor: '#54ff80' }, - ]} - datasetTransform={(dataset) => { - dataset.data = dataset.data.map((p) => { - let { x, y } = p as DataPoint; - y = y / 10 ** 9; - return { x: x, y: y }; - }); - - return dataset; - }} - /> - - - - - - - Transfer Rate - - - console.log(chart) - }, - }, - }, - }} - datasetOptions={[ - { label: 'Bytes Received (Bps)', borderColor: '#0071ff' }, - { label: 'Bytes Sent (Bps)', borderColor: '#54ff80' }, - ]} - /> - - - ); diff --git a/web_ui/frontend/app/origin/page.tsx b/web_ui/frontend/app/origin/page.tsx index 98e5fbcd6..9a775b6a7 100644 --- a/web_ui/frontend/app/origin/page.tsx +++ b/web_ui/frontend/app/origin/page.tsx @@ -87,62 +87,6 @@ export default function Home() { - - - - Transfer Rate - - - console.log(chart) - }, - }, - }, - }} - datasetOptions={[ - { label: 'Bytes Received (Bps)', borderColor: '#0071ff' }, - { label: 'Bytes Sent (Bps)', borderColor: '#54ff80' }, - ]} - /> - - - From 5943ed43f171507b3950551767387571d5d1164c Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Dec 2024 21:25:53 +0000 Subject: [PATCH 70/86] Replace os.stat sentinel file check with XRootD http GET --- launchers/launcher.go | 14 +++---- server_utils/origin.go | 50 +++++++++++++++++++++-- server_utils/origin_test.go | 60 --------------------------- xrootd/origin_test.go | 81 +++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 70 deletions(-) diff --git a/launchers/launcher.go b/launchers/launcher.go index c8868b498..9bab9a004 100644 --- a/launchers/launcher.go +++ b/launchers/launcher.go @@ -156,6 +156,13 @@ func LaunchModules(ctx context.Context, modules server_structs.ServerType) (serv if modules.IsEnabled(server_structs.OriginType) { + var server server_structs.XRootDServer + server, err = OriginServe(ctx, engine, egrp, modules) + if err != nil { + return + } + servers = append(servers, server) + var originExports []server_utils.OriginExport originExports, err = server_utils.GetOriginExports() if err != nil { @@ -167,13 +174,6 @@ func LaunchModules(ctx context.Context, modules server_structs.ServerType) (serv return } - var server server_structs.XRootDServer - server, err = OriginServe(ctx, engine, egrp, modules) - if err != nil { - return - } - servers = append(servers, server) - // Ordering: `LaunchBrokerListener` depends on the "right" value of Origin.FederationPrefix // which is possibly not set until `OriginServe` is called. // NOTE: Until the Broker supports multi-export origins, we've made the assumption that there diff --git a/server_utils/origin.go b/server_utils/origin.go index e158ce5d9..231478030 100644 --- a/server_utils/origin.go +++ b/server_utils/origin.go @@ -20,19 +20,22 @@ package server_utils import ( "fmt" - "os" + "net/http" "path" "path/filepath" "reflect" "strings" + "time" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" + "github.com/pelicanplatform/pelican/token" ) var originExports []OriginExport @@ -750,6 +753,29 @@ from S3 service URL. In this configuration, objects can be accessed at /federati return originExports, nil } +// Generate a test auth token for checking the sentinel location +func generateFileTestScitoken(resourceScope string) (string, error) { + issuerUrl := param.Server_ExternalWebUrl.GetString() + if issuerUrl == "" { // if both are empty, then error + return "", errors.New("failed to create token: invalid iss, Server_ExternalWebUrl is empty") + } + fTestTokenCfg := token.NewWLCGToken() + fTestTokenCfg.Lifetime = time.Minute + fTestTokenCfg.Issuer = issuerUrl + fTestTokenCfg.Subject = "origin" + fTestTokenCfg.Claims = map[string]string{"scope": fmt.Sprintf("storage.read:/%v", resourceScope)} + // For self-tests, the audience is the server itself + fTestTokenCfg.AddAudienceAny() + + // CreateToken also handles validation for us + tok, err := fTestTokenCfg.CreateToken() + if err != nil { + return "", errors.Wrap(err, "failed to create file test token") + } + + return tok, nil +} + // Check the sentinel files from Origin.Exports func CheckOriginSentinelLocations(exports []OriginExport) (ok bool, err error) { for _, export := range exports { @@ -758,11 +784,29 @@ func CheckOriginSentinelLocations(exports []OriginExport) (ok bool, err error) { if path.Base(sentinelPath) != sentinelPath { return false, errors.Errorf("invalid SentinelLocation path for StoragePrefix %s, file must not contain a directory. Got %s", export.StoragePrefix, export.SentinelLocation) } - fullPath := filepath.Join(export.StoragePrefix, sentinelPath) - _, err := os.Stat(fullPath) + + fullPath := filepath.Join(export.FederationPrefix, sentinelPath) + tkn, err := generateFileTestScitoken(sentinelPath) + if err != nil { + return false, errors.Wrap(err, "Failed to generate self-auth token for sentinel file check") + } + + sentinelUrl := fmt.Sprintf("%v%v", param.Origin_Url.GetString(), fullPath) + req, err := http.NewRequest(http.MethodGet, sentinelUrl, nil) + if err != nil { + return false, errors.Wrap(err, "Failed to create GET request for sentinel file check") + } + req.Header.Set("Authorization", "Bearer "+tkn) + + client := http.Client{Transport: config.GetTransport()} + resp, err := client.Do(req) if err != nil { return false, errors.Wrapf(err, "fail to open SentinelLocation %s for StoragePrefix %s. Collection check failed", export.SentinelLocation, export.StoragePrefix) } + + if resp.StatusCode != 200 { + return false, errors.New(fmt.Sprintf("Got non-200 response code %v when checking SentinelLocation %s for StoragePrefix %s", resp.StatusCode, export.SentinelLocation, export.StoragePrefix)) + } } } return true, nil diff --git a/server_utils/origin_test.go b/server_utils/origin_test.go index f31d9687a..9a5f25c3d 100644 --- a/server_utils/origin_test.go +++ b/server_utils/origin_test.go @@ -23,8 +23,6 @@ package server_utils import ( _ "embed" "fmt" - "os" - "path/filepath" "strings" "testing" @@ -395,64 +393,6 @@ func TestGetExports(t *testing.T) { }) } -func TestCheckOriginSentinelLocation(t *testing.T) { - tmpDir := t.TempDir() - tempStn := filepath.Join(tmpDir, "mock_sentinel") - file, err := os.Create(tempStn) - require.NoError(t, err) - err = file.Close() - require.NoError(t, err) - - mockExportNoStn := OriginExport{ - StoragePrefix: "/foo/bar", - FederationPrefix: "/demo/foo/bar", - Capabilities: server_structs.Capabilities{Reads: true}, - } - mockExportValidStn := OriginExport{ - StoragePrefix: tmpDir, - FederationPrefix: "/demo/foo/bar", - Capabilities: server_structs.Capabilities{Reads: true}, - SentinelLocation: "mock_sentinel", - } - mockExportInvalidStn := OriginExport{ - StoragePrefix: tmpDir, - FederationPrefix: "/demo/foo/bar", - Capabilities: server_structs.Capabilities{Reads: true}, - SentinelLocation: "sentinel_dne", - } - - t.Run("empty-sentinel-return-ok", func(t *testing.T) { - exports := make([]OriginExport, 0) - exports = append(exports, mockExportNoStn) - exports = append(exports, mockExportNoStn) - - ok, err := CheckOriginSentinelLocations(exports) - assert.NoError(t, err) - assert.True(t, ok) - }) - - t.Run("valid-sentinel-return-ok", func(t *testing.T) { - exports := make([]OriginExport, 0) - exports = append(exports, mockExportNoStn) - exports = append(exports, mockExportValidStn) - - ok, err := CheckOriginSentinelLocations(exports) - assert.NoError(t, err) - assert.True(t, ok) - }) - - t.Run("invalid-sentinel-return-error", func(t *testing.T) { - exports := make([]OriginExport, 0) - exports = append(exports, mockExportNoStn) - exports = append(exports, mockExportValidStn) - exports = append(exports, mockExportInvalidStn) - - ok, err := CheckOriginSentinelLocations(exports) - assert.Error(t, err) - assert.False(t, ok) - }) -} - func runBucketNameTest(t *testing.T, name string, valid bool) { t.Run(fmt.Sprintf("testBucketNameValidation-%s", name), func(t *testing.T) { err := validateBucketName(name) diff --git a/xrootd/origin_test.go b/xrootd/origin_test.go index e2c6a696d..b973b51e8 100644 --- a/xrootd/origin_test.go +++ b/xrootd/origin_test.go @@ -304,3 +304,84 @@ func TestS3OriginConfig(t *testing.T) { runS3Test(t, "", "path", "noaa-wod-pds/MD5SUMS") }) } + +func TestOriginWithSentinel(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + defer func() { require.NoError(t, egrp.Wait()) }() + defer cancel() + + server_utils.ResetTestState() + + defer server_utils.ResetTestState() + + tmpPathPattern := "XRD-Tst_Orgn*" + tmpPath, err := os.MkdirTemp("", tmpPathPattern) + require.NoError(t, err) + err = os.Chmod(tmpPath, 0755) + require.NoError(t, err) + + viper.Set("Origin.StoragePrefix", tmpPath) + viper.Set("Origin.FederationPrefix", "/test") + viper.Set("Origin.StorageType", "posix") + // Disable functionality we're not using (and is difficult to make work on Mac) + viper.Set("Origin.EnableCmsd", false) + viper.Set("Origin.EnableMacaroons", false) + viper.Set("Origin.EnableVoms", false) + viper.Set("Origin.Port", 0) + viper.Set("Server.WebPort", 0) + viper.Set("TLSSkipVerify", true) + viper.Set("Logging.Origin.Scitokens", "trace") + + mockupCancel := originMockup(ctx, egrp, t) + defer mockupCancel() + + mockExportValidStn := server_utils.OriginExport{ + StoragePrefix: viper.GetString("Origin.StoragePrefix"), + FederationPrefix: viper.GetString("Origin.FederationPrefix"), + Capabilities: server_structs.Capabilities{Reads: true}, + SentinelLocation: "mock_sentinel", + } + mockExportNoStn := server_utils.OriginExport{ + StoragePrefix: viper.GetString("Origin.StoragePrefix"), + FederationPrefix: viper.GetString("Origin.FederationPrefix"), + Capabilities: server_structs.Capabilities{Reads: true}, + } + mockExportInvalidStn := server_utils.OriginExport{ + StoragePrefix: viper.GetString("Origin.StoragePrefix"), + FederationPrefix: viper.GetString("Origin.FederationPrefix"), + Capabilities: server_structs.Capabilities{Reads: true}, + SentinelLocation: "sentinel_dne", + } + + tempStn := filepath.Join(mockExportValidStn.StoragePrefix, mockExportValidStn.SentinelLocation) + file, err := os.Create(tempStn) + require.NoError(t, err) + err = file.Close() + require.NoError(t, err) + err = os.Chmod(tempStn, 0755) + require.NoError(t, err) + + err = server_utils.WaitUntilWorking(ctx, "GET", param.Origin_Url.GetString(), "xrootd", 403, false) + if err != nil { + t.Fatalf("Unsuccessful test: Server encountered an error: %v", err) + } + require.NoError(t, err) + + t.Run("valid-sentinel-return-ok", func(t *testing.T) { + ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportValidStn}) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("empty-sentinel-return-ok", func(t *testing.T) { + ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportNoStn}) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("invalid-sentinel-return-error", func(t *testing.T) { + ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportInvalidStn}) + require.Error(t, err) + require.False(t, ok) + }) +} From fcd86def010a5c0864f08463761d25ae0aabeb3e Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Dec 2024 22:14:03 +0000 Subject: [PATCH 71/86] Test sentinel file against public S3 origin --- xrootd/origin_test.go | 92 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 12 deletions(-) diff --git a/xrootd/origin_test.go b/xrootd/origin_test.go index b973b51e8..c64fb53c6 100644 --- a/xrootd/origin_test.go +++ b/xrootd/origin_test.go @@ -231,16 +231,7 @@ func TestMultiExportOrigin(t *testing.T) { require.True(t, ok) } -func runS3Test(t *testing.T, bucketName, urlStyle, objectName string) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() - - server_utils.ResetTestState() - - defer server_utils.ResetTestState() - - federationPrefix := "/test" +func mockupS3Origin(ctx context.Context, egrp *errgroup.Group, t *testing.T, federationPrefix, bucketName, urlStyle string) context.CancelFunc { regionName := "us-east-1" serviceUrl := "https://s3.amazonaws.com" viper.Set("Origin.FederationPrefix", federationPrefix) @@ -260,7 +251,19 @@ func runS3Test(t *testing.T, bucketName, urlStyle, objectName string) { viper.Set("Server.WebPort", 0) viper.Set("TLSSkipVerify", true) - mockupCancel := originMockup(ctx, egrp, t) + return originMockup(ctx, egrp, t) +} + +func runS3Test(t *testing.T, bucketName, urlStyle, objectName string) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + defer func() { require.NoError(t, egrp.Wait()) }() + defer cancel() + server_utils.ResetTestState() + defer server_utils.ResetTestState() + + federationPrefix := "/test" + + mockupCancel := mockupS3Origin(ctx, egrp, t, federationPrefix, bucketName, urlStyle) defer mockupCancel() originEndpoint := param.Origin_Url.GetString() @@ -305,7 +308,67 @@ func TestS3OriginConfig(t *testing.T) { }) } -func TestOriginWithSentinel(t *testing.T) { +func TestS3OriginWithSentinel(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + defer func() { require.NoError(t, egrp.Wait()) }() + defer cancel() + server_utils.ResetTestState() + defer server_utils.ResetTestState() + + federationPrefix := "/test" + bucketName := "noaa-wod-pds" + + mockupCancel := mockupS3Origin(ctx, egrp, t, federationPrefix, bucketName, "path") + defer mockupCancel() + + mockExportValidStn := server_utils.OriginExport{ + StoragePrefix: viper.GetString("Origin.StoragePrefix"), + FederationPrefix: viper.GetString("Origin.FederationPrefix"), + Capabilities: server_structs.Capabilities{Reads: true}, + SentinelLocation: "MD5SUMS", + } + + originEndpoint := param.Origin_Url.GetString() + // At this point, a 403 means the server is running, which means its ready to grab objects from + err := server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 403, true) + if err != nil { + t.Fatalf("Unsuccessful test: Server encountered an error: %v", err) + } + + // mock export with no sentinel + mockExportNoStn := server_utils.OriginExport{ + StoragePrefix: viper.GetString("Origin.StoragePrefix"), + FederationPrefix: viper.GetString("Origin.FederationPrefix"), + Capabilities: server_structs.Capabilities{Reads: true}, + } + + // mock export with an invalid sentinel + mockExportInvalidStn := server_utils.OriginExport{ + StoragePrefix: viper.GetString("Origin.StoragePrefix"), + FederationPrefix: viper.GetString("Origin.FederationPrefix"), + Capabilities: server_structs.Capabilities{Reads: true}, + SentinelLocation: "MD5SUMS_dne", + } + + t.Run("valid-sentinel-return-ok", func(t *testing.T) { + ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportValidStn}) + require.NoError(t, err) + require.True(t, ok) + }) + t.Run("empty-sentinel-return-ok", func(t *testing.T) { + ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportNoStn}) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("invalid-sentinel-return-error", func(t *testing.T) { + ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportInvalidStn}) + require.Error(t, err) + require.False(t, ok) + }) +} + +func TestPosixOriginWithSentinel(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() defer cancel() @@ -314,6 +377,7 @@ func TestOriginWithSentinel(t *testing.T) { defer server_utils.ResetTestState() + // Create a test temp dir, ensure it's readable by XRootD tmpPathPattern := "XRD-Tst_Orgn*" tmpPath, err := os.MkdirTemp("", tmpPathPattern) require.NoError(t, err) @@ -335,17 +399,20 @@ func TestOriginWithSentinel(t *testing.T) { mockupCancel := originMockup(ctx, egrp, t) defer mockupCancel() + // mock export with a valid sentinel mockExportValidStn := server_utils.OriginExport{ StoragePrefix: viper.GetString("Origin.StoragePrefix"), FederationPrefix: viper.GetString("Origin.FederationPrefix"), Capabilities: server_structs.Capabilities{Reads: true}, SentinelLocation: "mock_sentinel", } + // mock export with no sentinel mockExportNoStn := server_utils.OriginExport{ StoragePrefix: viper.GetString("Origin.StoragePrefix"), FederationPrefix: viper.GetString("Origin.FederationPrefix"), Capabilities: server_structs.Capabilities{Reads: true}, } + // mock export with an invalid sentinel mockExportInvalidStn := server_utils.OriginExport{ StoragePrefix: viper.GetString("Origin.StoragePrefix"), FederationPrefix: viper.GetString("Origin.FederationPrefix"), @@ -353,6 +420,7 @@ func TestOriginWithSentinel(t *testing.T) { SentinelLocation: "sentinel_dne", } + // Create a sentinel file, ensure it's readable by XRootD tempStn := filepath.Join(mockExportValidStn.StoragePrefix, mockExportValidStn.SentinelLocation) file, err := os.Create(tempStn) require.NoError(t, err) From 0ef834c60b45fdc89f725b4f4a66011db51de31f Mon Sep 17 00:00:00 2001 From: Brian Aydemir Date: Wed, 4 Dec 2024 08:43:31 -0600 Subject: [PATCH 72/86] Fix documentation for the `Origin.Scitokens*` parameters --- docs/parameters.yaml | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index adaa76eb0..72bc601d6 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -763,35 +763,65 @@ components: ["origin"] --- name: Origin.ScitokensRestrictedPaths description: |+ - Enable the built-in issuer daemon for the origin. + This parameter is used to configure + [XRootD's SciTokens authorization plugin](https://github.com/xrootd/xrootd/tree/master/src/XrdSciTokens). + + Any restrictions on the paths that the issuer can authorize inside their + namespace. This is meant to be a mechanism to help with transitions, where + the underlying storage is setup such that an issuer's namespace contains + directories that should not be managed by the issuer. type: stringSlice default: [] components: ["origin"] --- name: Origin.ScitokensMapSubject description: |+ - Enable the built-in issuer daemon for the origin. + This parameter is used to configure + [XRootD's SciTokens authorization plugin](https://github.com/xrootd/xrootd/tree/master/src/XrdSciTokens). + + If set to `true`, the contents of the token's `sub` claim will be copied + into the XRootD username. When `Origin.Multiuser` is also set to `true`, + this will allow XRootD to read and write files using the Unix username + specified in the token. type: bool default: false components: ["origin"] --- name: Origin.ScitokensDefaultUser description: |+ - Enable the built-in issuer daemon for the origin. + This parameter is used to configure + [XRootD's SciTokens authorization plugin](https://github.com/xrootd/xrootd/tree/master/src/XrdSciTokens). + + If set, then all authorized operations will be performed under the + provided username when interacting with the file system. This is useful + when all files owned by an issuer should be mapped to a particular Unix + user account. type: string default: none components: ["origin"] --- name: Origin.ScitokensUsernameClaim description: |+ - Enable the built-in issuer daemon for the origin. + This parameter is used to configure + [XRootD's SciTokens authorization plugin](https://github.com/xrootd/xrootd/tree/master/src/XrdSciTokens). + + If set, then the provided claim will be used to determine the XRootD + username, and it will override the + `Origin.ScitokensMapSubject` and `Origin.ScitokensDefaultUser` parameters. type: string default: none components: ["origin"] --- name: Origin.ScitokensNameMapFile description: |+ - Enable the built-in issuer daemon for the origin. + This parameter is used to configure + [XRootD's SciTokens authorization plugin](https://github.com/xrootd/xrootd/tree/master/src/XrdSciTokens). + + If set, then the referenced file is parsed as a JSON object and the + specified mappings are applied to the username inside the XRootD + framework. See the + [XrdSciTokens documentation](https://github.com/xrootd/xrootd/tree/master/src/XrdSciTokens#mapfile-format) + for more information on the mapfile's format. type: string default: none components: ["origin"] From d4f42acf961dadcd789eabdb82269bd1f5ad1e25 Mon Sep 17 00:00:00 2001 From: Brian Aydemir Date: Wed, 4 Dec 2024 09:22:47 -0600 Subject: [PATCH 73/86] Fix documentation for the `Logging.Origin.Http` parameter --- docs/parameters.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 72bc601d6..0b04f0c06 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -211,9 +211,9 @@ components: ["origin"] --- name: Logging.Origin.Http description: |+ - Logging level for the HTTP component of the origin. Increasing to debug will cause the Xrootd daemon to log. + Logging level for the HTTP component of the origin. Increasing to debug + will cause the Xrootd daemon to log all headers and requests. Accepted values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `panic` - all headers and requests. type: string default: error components: ["origin"] From 870871ebee8699b4cd65d8f961e06603cddc3ab9 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Wed, 4 Dec 2024 12:42:03 -0600 Subject: [PATCH 74/86] Redirect Director Metrics if Not Admin - Redirect director metrics if not admin - Hide navigation options if not admin - Update AuthenicatedContent to consume static lists to enable easier SSR --- web_ui/frontend/app/director/layout.tsx | 19 +- web_ui/frontend/app/director/metrics/page.tsx | 167 +++++++++--------- .../app/origin/globus/callback/page.tsx | 2 +- web_ui/frontend/app/origin/globus/page.tsx | 5 +- web_ui/frontend/app/origin/page.tsx | 5 +- web_ui/frontend/app/registry/layout.tsx | 9 +- .../layout/AuthenticatedContent.tsx | 24 ++- 7 files changed, 126 insertions(+), 105 deletions(-) diff --git a/web_ui/frontend/app/director/layout.tsx b/web_ui/frontend/app/director/layout.tsx index 5d9325d30..55e01cd65 100644 --- a/web_ui/frontend/app/director/layout.tsx +++ b/web_ui/frontend/app/director/layout.tsx @@ -20,7 +20,8 @@ import { Box } from '@mui/material'; import { ButtonLink, Sidebar } from '@/components/layout/Sidebar'; import BuildIcon from '@mui/icons-material/Build'; import Main from '@/components/layout/Main'; -import { Dashboard, Equalizer, MapOutlined } from '@mui/icons-material'; +import { Block, Dashboard, Equalizer, MapOutlined } from '@mui/icons-material'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; export const metadata = { title: 'Pelican Director', @@ -38,15 +39,19 @@ export default function RootLayout({ - - - - - - + + + + + + + + + +
{children}
diff --git a/web_ui/frontend/app/director/metrics/page.tsx b/web_ui/frontend/app/director/metrics/page.tsx index 2d47410f7..52590a59a 100644 --- a/web_ui/frontend/app/director/metrics/page.tsx +++ b/web_ui/frontend/app/director/metrics/page.tsx @@ -9,96 +9,103 @@ import { } from '@/app/director/metrics/components/MetricBoxPlot'; import { StorageTable } from '@/app/director/metrics/components/StorageTable'; import { TransferBarGraph } from '@/app/director/metrics/components/TransferBarGraph'; +import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; const Page = () => { return ( - - - - {[ - , - , - ].map((component, index) => ( - - {component} - - ))} - - - - - - - - - - - - - - - + + + + + {[ + , + , + ].map((component, index) => ( + + {component} - - - - - - - - - - - - - - - - + ))} - - - {[ - , - , - ].map((component, index) => ( - - {component} + + + + + + + + + + + + + + + + + + + + + + + + - ))} + + + + + + + + + + {[ + , + , + ].map((component, index) => ( + + {component} + + ))} + - + ); }; diff --git a/web_ui/frontend/app/origin/globus/callback/page.tsx b/web_ui/frontend/app/origin/globus/callback/page.tsx index 2bcad0b8c..19b5c2ea0 100644 --- a/web_ui/frontend/app/origin/globus/callback/page.tsx +++ b/web_ui/frontend/app/origin/globus/callback/page.tsx @@ -96,7 +96,7 @@ export default function Home() { u?.role == 'admin'} + allowedRoles={['admin']} > u?.role == 'admin'} - > + Globus Exports diff --git a/web_ui/frontend/app/origin/page.tsx b/web_ui/frontend/app/origin/page.tsx index 98e5fbcd6..bcc00c005 100644 --- a/web_ui/frontend/app/origin/page.tsx +++ b/web_ui/frontend/app/origin/page.tsx @@ -51,10 +51,7 @@ export default function Home() { }; return ( - u?.role == 'admin'} - > + diff --git a/web_ui/frontend/app/registry/layout.tsx b/web_ui/frontend/app/registry/layout.tsx index 36e2b69cd..2c0febf4b 100644 --- a/web_ui/frontend/app/registry/layout.tsx +++ b/web_ui/frontend/app/registry/layout.tsx @@ -35,6 +35,7 @@ import SpeedDial, { } from '@/components/layout/SidebarSpeedDial'; import AuthenticatedContent from '@/components/layout/AuthenticatedContent'; import { PaddedContent } from '@/components/layout'; +import BuildIcon from '@mui/icons-material/Build'; export const metadata = { title: 'Pelican Registry', @@ -81,9 +82,11 @@ export default function RootLayout({ - - - + + + + +
{children} diff --git a/web_ui/frontend/components/layout/AuthenticatedContent.tsx b/web_ui/frontend/components/layout/AuthenticatedContent.tsx index f1382b828..0da52c668 100644 --- a/web_ui/frontend/components/layout/AuthenticatedContent.tsx +++ b/web_ui/frontend/components/layout/AuthenticatedContent.tsx @@ -41,18 +41,30 @@ interface AuthenticatedContentProps { promptLogin?: boolean; redirect?: boolean; trustThenValidate?: boolean; - children: React.ReactNode; boxProps?: BoxProps; - checkAuthentication?: (user: User) => boolean; + allowedRoles?: User['role'][]; + replace?: boolean; + children: React.ReactNode; } +/** + * AuthenticatedContent is a component that will show the children if the user is authenticated. + * @param promptLogin If true then the user will be prompted to login if they are not authenticated + * @param redirect If true then the user will be redirected to the login page if they are not authenticated + * @param trustThenValidate If true then the user will be shown the content if they are not authenticated but will be validated after + * @param boxProps The props to pass to the Box component + * @param allowedRoles The roles that are allowed to see the content + * @param replace If true then the + * @param children The content to show if the user is authenticated + * @constructor + */ const AuthenticatedContent = ({ promptLogin = false, redirect = false, trustThenValidate = false, children, boxProps, - checkAuthentication, + allowedRoles, }: AuthenticatedContentProps) => { if (redirect && promptLogin) { throw new Error('redirect XOR promptLogin must be true'); @@ -66,12 +78,12 @@ const AuthenticatedContent = ({ const [pageUrl, setPageUrl] = useState(''); const authenticated = useMemo(() => { - if (data && checkAuthentication) { - return checkAuthentication(data); + if (data && allowedRoles) { + return data?.role && allowedRoles.includes(data?.role); } else { return !!data?.authenticated; } - }, [data, checkAuthentication]); + }, [data, allowedRoles]); useEffect(() => { // Keep pathname as is since backend handles the redirect after logging in and needs the full path From 70dec0a31137360149932d41c0da53898097e562 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 4 Dec 2024 19:38:40 +0000 Subject: [PATCH 75/86] Resolve comments from Justin --- director/cache_ads.go | 8 +++----- director/sort.go | 20 +++++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/director/cache_ads.go b/director/cache_ads.go index 684fb0e01..2dea53d93 100644 --- a/director/cache_ads.go +++ b/director/cache_ads.go @@ -83,13 +83,11 @@ func (f filterType) String() string { // 5. Return the updated ServerAd. The ServerAd passed in will not be modified func recordAd(ctx context.Context, sAd server_structs.ServerAd, namespaceAds *[]server_structs.NamespaceAdV2) (updatedAd server_structs.ServerAd) { if err := updateLatLong(&sAd); err != nil { - switch err := err.(type) { - case GeoIPError: - labels := err.labels + if geoIPError, ok := err.(geoIPError); ok { + labels := geoIPError.labels metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() - default: - log.Debugln("Failed to lookup GeoIP coordinates for host", sAd.URL.Host) } + log.Debugln("Failed to lookup GeoIP coordinates for host", sAd.URL.Host) } if sAd.URL.String() == "" { diff --git a/director/sort.go b/director/sort.go index 80e9ccf10..cc7054cc1 100644 --- a/director/sort.go +++ b/director/sort.go @@ -61,13 +61,13 @@ type ( Coordinate Coordinate `mapstructure:"Coordinate"` } - GeoIPError struct { + geoIPError struct { labels prometheus.Labels errorMsg string } ) -func (e GeoIPError) Error() string { +func (e geoIPError) Error() string { return e.errorMsg } @@ -192,13 +192,13 @@ func getLatLong(addr netip.Addr) (lat float64, long float64, err error) { reader := maxMindReader.Load() if reader == nil { labels["source"] = "server" - err = GeoIPError{labels: labels, errorMsg: "No GeoIP database is available"} + err = geoIPError{labels: labels, errorMsg: "No GeoIP database is available"} return } record, err := reader.City(ip) if err != nil { labels["source"] = "server" - err = GeoIPError{labels: labels, errorMsg: err.Error()} + err = geoIPError{labels: labels, errorMsg: err.Error()} return } lat = record.Location.Latitude @@ -211,7 +211,7 @@ func getLatLong(addr netip.Addr) (lat float64, long float64, err error) { errMsg := fmt.Sprintf("GeoIP Resolution of the address %s resulted in the null lat/long. This will result in random server sorting.", ip.String()) log.Warning(errMsg) labels["source"] = "client" - err = GeoIPError{labels: labels, errorMsg: errMsg} + err = geoIPError{labels: labels, errorMsg: errMsg} } // MaxMind provides an accuracy radius in kilometers. When it actually has no clue how to resolve a valid, public @@ -225,7 +225,7 @@ func getLatLong(addr netip.Addr) (lat float64, long float64, err error) { lat = 0 long = 0 labels["source"] = "client" - err = GeoIPError{labels: labels, errorMsg: errMsg} + err = geoIPError{labels: labels, errorMsg: errMsg} } return @@ -300,14 +300,12 @@ func sortServerAds(ctx context.Context, clientAddr netip.Addr, ads []server_stru if err != nil { // If it is a geoIP error, then we get the labels and increment the error counter // Otherwise we log the error and continue - switch err := err.(type) { - case GeoIPError: - labels := err.labels + if geoIPError, ok := err.(geoIPError); ok { + labels := geoIPError.labels setProjectLabel(ctx, &labels) metrics.PelicanDirectorGeoIPErrors.With(labels).Inc() - default: - log.Warningf("Error while getting the client IP address: %v", err) } + log.Warningf("Error while getting the client IP address: %v", err) } // For each ad, we apply the configured sort method to determine a priority weight. From ed8050f4476e6db75e642f4caa968f640f40ae2d Mon Sep 17 00:00:00 2001 From: Cannon Lock <49032265+CannonLock@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:50:40 -0600 Subject: [PATCH 76/86] Update post-release.yaml --- .github/workflows/post-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/post-release.yaml b/.github/workflows/post-release.yaml index e6c048f71..1b9e6daa9 100644 --- a/.github/workflows/post-release.yaml +++ b/.github/workflows/post-release.yaml @@ -1,4 +1,4 @@ -# Toggle webhook to pull latest release onto dl.pelicanplatform.org +# Toggle webhook to pull latest release onto pelicanplatform.org and update the download offerings there name: post-release on: From c24dd44a12324c9128f71a6f45d211cbaee92eec Mon Sep 17 00:00:00 2001 From: Cannon Lock <49032265+CannonLock@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:54:30 -0600 Subject: [PATCH 77/86] Update post-release.yaml --- .github/workflows/post-release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/post-release.yaml b/.github/workflows/post-release.yaml index 1b9e6daa9..33aa2d6ec 100644 --- a/.github/workflows/post-release.yaml +++ b/.github/workflows/post-release.yaml @@ -1,4 +1,5 @@ # Toggle webhook to pull latest release onto pelicanplatform.org and update the download offerings there +# Post release this will result in the new release being available and the Major/Minor pointers being moved/created accordingly name: post-release on: From f89afeb06d01192cd152c1d1e9a9d22483a2e5a5 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Thu, 5 Dec 2024 17:14:01 +0000 Subject: [PATCH 78/86] add logging for sentinel check, update logging to reference 'objects' --- server_utils/origin.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/server_utils/origin.go b/server_utils/origin.go index 231478030..dc4898ee3 100644 --- a/server_utils/origin.go +++ b/server_utils/origin.go @@ -21,6 +21,7 @@ package server_utils import ( "fmt" "net/http" + "net/url" "path" "path/filepath" "reflect" @@ -753,11 +754,12 @@ from S3 service URL. In this configuration, objects can be accessed at /federati return originExports, nil } -// Generate a test auth token for checking the sentinel location -func generateFileTestScitoken(resourceScope string) (string, error) { +// Generate a minimally scoped auth token that allows the origin +// to query itself for its sentinel file +func generateSentinelCheckScitoken(resourceScope string) (string, error) { issuerUrl := param.Server_ExternalWebUrl.GetString() if issuerUrl == "" { // if both are empty, then error - return "", errors.New("failed to create token: invalid iss, Server_ExternalWebUrl is empty") + return "", errors.New("failed to create a sentinel check auth token because required configuration 'Server.ExternalWebUrl' is empty") } fTestTokenCfg := token.NewWLCGToken() fTestTokenCfg.Lifetime = time.Minute @@ -770,7 +772,7 @@ func generateFileTestScitoken(resourceScope string) (string, error) { // CreateToken also handles validation for us tok, err := fTestTokenCfg.CreateToken() if err != nil { - return "", errors.Wrap(err, "failed to create file test token") + return "", errors.Wrap(err, "failed to create sentinel check auth token") } return tok, nil @@ -780,32 +782,36 @@ func generateFileTestScitoken(resourceScope string) (string, error) { func CheckOriginSentinelLocations(exports []OriginExport) (ok bool, err error) { for _, export := range exports { if export.SentinelLocation != "" { + log.Infof("Checking that sentinel object %v is present for federation prefix %s", export.SentinelLocation, export.FederationPrefix) sentinelPath := path.Clean(export.SentinelLocation) if path.Base(sentinelPath) != sentinelPath { - return false, errors.Errorf("invalid SentinelLocation path for StoragePrefix %s, file must not contain a directory. Got %s", export.StoragePrefix, export.SentinelLocation) + return false, errors.Errorf("invalid SentinelLocation path for federation prefix %s, path must not contain a directory. Got %s", export.FederationPrefix, export.SentinelLocation) } fullPath := filepath.Join(export.FederationPrefix, sentinelPath) - tkn, err := generateFileTestScitoken(sentinelPath) + tkn, err := generateSentinelCheckScitoken(sentinelPath) if err != nil { - return false, errors.Wrap(err, "Failed to generate self-auth token for sentinel file check") + return false, errors.Wrap(err, "failed to generate self-auth token for sentinel object check") } - sentinelUrl := fmt.Sprintf("%v%v", param.Origin_Url.GetString(), fullPath) + sentinelUrl, err := url.JoinPath(param.Origin_Url.GetString(), fullPath) + if err != nil { + return false, errors.Wrapf(err, "unable fo form sentinel URL for Origin.Url %v, sentinel path %v", param.Origin_Url.GetString(), fullPath) + } req, err := http.NewRequest(http.MethodGet, sentinelUrl, nil) if err != nil { - return false, errors.Wrap(err, "Failed to create GET request for sentinel file check") + return false, errors.Wrap(err, "failed to create GET request for sentinel object check") } req.Header.Set("Authorization", "Bearer "+tkn) client := http.Client{Transport: config.GetTransport()} resp, err := client.Do(req) if err != nil { - return false, errors.Wrapf(err, "fail to open SentinelLocation %s for StoragePrefix %s. Collection check failed", export.SentinelLocation, export.StoragePrefix) + return false, errors.Wrapf(err, "fail to open sentinel object %s for federation prefix %s.", export.SentinelLocation, export.FederationPrefix) } if resp.StatusCode != 200 { - return false, errors.New(fmt.Sprintf("Got non-200 response code %v when checking SentinelLocation %s for StoragePrefix %s", resp.StatusCode, export.SentinelLocation, export.StoragePrefix)) + return false, errors.New(fmt.Sprintf("got non-200 response code %v when checking sentinel object %s for federation prefix %s", resp.StatusCode, export.SentinelLocation, export.FederationPrefix)) } } } From 1a47800dee9d20b9c3fd9ca454d405541a27aa39 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Thu, 5 Dec 2024 18:49:51 +0000 Subject: [PATCH 79/86] rename sentinel check token generator to reflect type of token generated --- server_utils/origin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server_utils/origin.go b/server_utils/origin.go index dc4898ee3..02963b1f1 100644 --- a/server_utils/origin.go +++ b/server_utils/origin.go @@ -756,7 +756,7 @@ from S3 service URL. In this configuration, objects can be accessed at /federati // Generate a minimally scoped auth token that allows the origin // to query itself for its sentinel file -func generateSentinelCheckScitoken(resourceScope string) (string, error) { +func generateSentinelReadToken(resourceScope string) (string, error) { issuerUrl := param.Server_ExternalWebUrl.GetString() if issuerUrl == "" { // if both are empty, then error return "", errors.New("failed to create a sentinel check auth token because required configuration 'Server.ExternalWebUrl' is empty") @@ -789,7 +789,7 @@ func CheckOriginSentinelLocations(exports []OriginExport) (ok bool, err error) { } fullPath := filepath.Join(export.FederationPrefix, sentinelPath) - tkn, err := generateSentinelCheckScitoken(sentinelPath) + tkn, err := generateSentinelReadToken(sentinelPath) if err != nil { return false, errors.Wrap(err, "failed to generate self-auth token for sentinel object check") } From 0f2c11223f3011a2be37a282f507fccba06882cb Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 6 Dec 2024 16:29:44 +0000 Subject: [PATCH 80/86] Removed the tsl defer functionality from the cache until we can fix it on the xrootd end --- xrootd/resources/xrootd-cache.cfg | 4 --- xrootd/xrootd_config_test.go | 46 ------------------------------- 2 files changed, 50 deletions(-) diff --git a/xrootd/resources/xrootd-cache.cfg b/xrootd/resources/xrootd-cache.cfg index 60883b15a..ae2aabb25 100644 --- a/xrootd/resources/xrootd-cache.cfg +++ b/xrootd/resources/xrootd-cache.cfg @@ -58,10 +58,6 @@ pfc.writequeue 16 4 pfc.ram 4g pfc.diskusage {{if .Cache.LowWatermark}}{{.Cache.LowWatermark}}{{else}}0.90{{end}} {{if .Cache.HighWaterMark}}{{.Cache.HighWaterMark}}{{else}}0.95{{end}} purgeinterval 300s xrootd.fslib ++ throttle # throttle plugin is needed to calculate server IO load -http.tlsclientauth defer -{{- range $Prefix := .Cache.X509ClientAuthenticationPrefixes}} -http.tlsrequiredprefix {{$Prefix}} -{{- end}} {{if .Cache.Concurrency}} throttle.throttle concurrency {{.Cache.Concurrency}} {{end}} diff --git a/xrootd/xrootd_config_test.go b/xrootd/xrootd_config_test.go index 235ad53a3..35d6f1e0e 100644 --- a/xrootd/xrootd_config_test.go +++ b/xrootd/xrootd_config_test.go @@ -582,52 +582,6 @@ func TestXrootDCacheConfig(t *testing.T) { assert.NotNil(t, configPath) }) - t.Run("TestCacheHTTPTLSRequiredPrefixCorrectConfig", func(t *testing.T) { - xrootd := xrootdTest{T: t} - xrootd.setup() - - // Set our config - viper.Set("Cache.X509ClientAuthenticationPrefixes", []string{"pref1", "pref2", "pref3"}) - - // Generate the xrootd config - configPath, err := ConfigXrootd(ctx, false) - require.NoError(t, err) - assert.NotNil(t, configPath) - - // Verify the output - file, err := os.Open(configPath) - assert.NoError(t, err) - defer file.Close() - - content, err := io.ReadAll(file) - assert.NoError(t, err) - assert.Contains(t, string(content), "http.tlsrequiredprefix pref1") - assert.Contains(t, string(content), "http.tlsrequiredprefix pref2") - assert.Contains(t, string(content), "http.tlsrequiredprefix pref3") - }) - - t.Run("TestCacheAuthenticationPrefixes", func(t *testing.T) { - xrootd := xrootdTest{T: t} - xrootd.setup() - - // Set our config - viper.Set("Cache.X509AuthenticationPrefixes", []string{}) - - // Generate the xrootd config - configPath, err := ConfigXrootd(ctx, false) - require.NoError(t, err) - assert.NotNil(t, configPath) - - // Verify the output - file, err := os.Open(configPath) - assert.NoError(t, err) - defer file.Close() - - content, err := io.ReadAll(file) - assert.NoError(t, err) - assert.NotContains(t, string(content), "http.tlsrequiredprefix") - }) - t.Run("TestNestedDataMetaNamespace", func(t *testing.T) { testDir := t.TempDir() viper.Set("Cache.StorageLocation", testDir) From 4d31f83a2b9a303b7a337c8d7a0d5b23dee23c00 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 6 Dec 2024 16:40:37 +0000 Subject: [PATCH 81/86] Removed calling the x509 test script until we can fix the functionality --- .github/workflows/test-template.yml | 2 -- .github/workflows/test.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 0109c967e..12e09f25f 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -95,7 +95,5 @@ jobs: run: ./github_scripts/get_put_test.sh - name: Run End-to-End Test for Director stat run: ./github_scripts/stat_test.sh - - name: Run End-to-End Test of x509 access - run: ./github_scripts/x509_test.sh - name: Run End-to-End Test for --version flag run: ./github_scripts/version_test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0804df90f..730019e5a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,4 +65,4 @@ jobs: with: tags: "lotman" coverprofile: "coverage-server.out" - binary_name: "pelican-server" + binary_name: "pelican-server" \ No newline at end of file From 598ef7fb6a7ae2c95e93b341be18c1f342c507e6 Mon Sep 17 00:00:00 2001 From: Emma Turetsky Date: Fri, 6 Dec 2024 17:04:09 +0000 Subject: [PATCH 82/86] Fixed missing newline --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 730019e5a..0804df90f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,4 +65,4 @@ jobs: with: tags: "lotman" coverprofile: "coverage-server.out" - binary_name: "pelican-server" \ No newline at end of file + binary_name: "pelican-server" From b1a13236a682181ab73a8194c9de7b6b8fd3cafd Mon Sep 17 00:00:00 2001 From: Brian Aydemir Date: Fri, 6 Dec 2024 20:12:16 -0600 Subject: [PATCH 83/86] Pre-process GeoIP overrides into CIDRs for more efficient testing (#1384) --- director/cache_ads_test.go | 4 +- director/sort.go | 86 +++++++++++++++++++++----------------- director/sort_test.go | 10 ++--- 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/director/cache_ads_test.go b/director/cache_ads_test.go index 15699834c..34944efdb 100644 --- a/director/cache_ads_test.go +++ b/director/cache_ads_test.go @@ -441,11 +441,11 @@ func TestRecordAd(t *testing.T) { statUtils = make(map[string]serverStatUtil) serverAds.DeleteAll() - geoIPOverrides = nil + geoNetOverrides = nil }) server_utils.ResetTestState() func() { - geoIPOverrides = nil + geoNetOverrides = nil healthTestUtilsMutex.Lock() statUtilsMutex.Lock() diff --git a/director/sort.go b/director/sort.go index cc7054cc1..62dca2bf8 100644 --- a/director/sort.go +++ b/director/sort.go @@ -61,6 +61,11 @@ type ( Coordinate Coordinate `mapstructure:"Coordinate"` } + GeoNetOverride struct { + IPNet net.IPNet + Coordinate Coordinate + } + geoIPError struct { labels prometheus.Labels errorMsg string @@ -72,8 +77,8 @@ func (e geoIPError) Error() string { } var ( - invalidOverrideLogOnce = map[string]bool{} - geoIPOverrides []GeoIPOverride + // Stores the unmarshalled GeoIP override config in a form that's efficient to test + geoNetOverrides []GeoNetOverride // Stores a mapping of client IPs that have been randomly assigned a coordinate clientIpCache = ttlcache.New(ttlcache.WithTTL[netip.Addr, Coordinate](20 * time.Minute)) @@ -108,53 +113,58 @@ func (me SwapMaps) Swap(left, right int) { me[left], me[right] = me[right], me[left] } +// Unmarshal any configured GeoIP overrides. +// Malformed IPs and CIDRs are logged but not returned as errors. +func unmarshalOverrides() error { + var geoIPOverrides []GeoIPOverride + + // Ensure that we're starting with an empty slice. + geoNetOverrides = nil + + err := param.GeoIPOverrides.Unmarshal(&geoIPOverrides) + if err != nil { + return err + } + + for _, override := range geoIPOverrides { + var configuredCIDR string + + if strings.Contains(override.IP, "/") { + configuredCIDR = override.IP + } else { + configuredCIDR = override.IP + "/32" + } + + _, ipNet, err := net.ParseCIDR(configuredCIDR) + if err != nil { + // Log the error, and continue looking for good configuration. + log.Warningf("Failed to parse configured GeoIPOverride address (%s). Unable to use for GeoIP resolution!", override.IP) + continue + } + geoNetOverrides = append(geoNetOverrides, GeoNetOverride{IPNet: *ipNet, Coordinate: override.Coordinate}) + } + return nil +} + // Check for any pre-configured IP-to-lat/long overrides. If the passed address // matches an override IP (either directly or via CIDR masking), then we use the // configured lat/long from the override instead of relying on MaxMind. // NOTE: We don't return an error because if checkOverrides encounters an issue, // we still have GeoIP to fall back on. func checkOverrides(addr net.IP) (coordinate *Coordinate) { - // Unmarshal the values, but only the first time we run through this block - if geoIPOverrides == nil { - err := param.GeoIPOverrides.Unmarshal(&geoIPOverrides) + // Unmarshal the GeoIP override config if we haven't already done so. + if geoNetOverrides == nil { + err := unmarshalOverrides() if err != nil { - log.Warningf("Error while unmarshaling GeoIP Overrides: %v", err) + log.Warningf("Error while unmarshalling GeoIP overrides: %v", err) + return nil } } - - for _, geoIPOverride := range geoIPOverrides { - // Check for regular IP addresses before CIDR - overrideIP := net.ParseIP(geoIPOverride.IP) - if overrideIP == nil { - // The IP is malformed - if !invalidOverrideLogOnce[geoIPOverride.IP] && !strings.Contains(geoIPOverride.IP, "/") { - // Don't return here, because we have more to check. - // Do provide a notice to the user, however. - log.Warningf("Failed to parse configured GeoIPOverride address (%s). Unable to use for GeoIP resolution!", geoIPOverride.IP) - invalidOverrideLogOnce[geoIPOverride.IP] = true - } - } - if overrideIP.Equal(addr) { - return &geoIPOverride.Coordinate - } - - // Alternatively, we can match by CIDR blocks - if strings.Contains(geoIPOverride.IP, "/") { - _, ipNet, err := net.ParseCIDR(geoIPOverride.IP) - if err != nil { - if !invalidOverrideLogOnce[geoIPOverride.IP] { - // Same reason as above for not returning. - log.Warningf("Failed to parse configured GeoIPOverride CIDR address (%s): %v. Unable to use for GeoIP resolution!", geoIPOverride.IP, err) - invalidOverrideLogOnce[geoIPOverride.IP] = true - } - continue - } - if ipNet.Contains(addr) { - return &geoIPOverride.Coordinate - } + for _, override := range geoNetOverrides { + if override.IPNet.Contains(addr) { + return &override.Coordinate } } - return nil } diff --git a/director/sort_test.go b/director/sort_test.go index 753ba752d..d20b6deb0 100644 --- a/director/sort_test.go +++ b/director/sort_test.go @@ -49,7 +49,7 @@ func TestCheckOverrides(t *testing.T) { server_utils.ResetTestState() t.Cleanup(func() { server_utils.ResetTestState() - geoIPOverrides = nil + geoNetOverrides = nil }) // We'll also check that our logging feature responsibly reports @@ -80,11 +80,9 @@ func TestCheckOverrides(t *testing.T) { t.Run("test-log-output", func(t *testing.T) { // Check that the log caught our malformed IP and CIDR. We only need to test this once, because it is only logged the very first time. require.Contains(t, logOutput.String(), "Failed to parse configured GeoIPOverride address (192.168.0). Unable to use for GeoIP resolution!") - require.Contains(t, logOutput.String(), "Failed to parse configured GeoIPOverride CIDR address (10.0.0./24): invalid CIDR address: 10.0.0./24."+ - " Unable to use for GeoIP resolution!") + require.Contains(t, logOutput.String(), "Failed to parse configured GeoIPOverride address (10.0.0./24). Unable to use for GeoIP resolution!") require.Contains(t, logOutput.String(), "Failed to parse configured GeoIPOverride address (FD00::000G). Unable to use for GeoIP resolution!") - require.Contains(t, logOutput.String(), "Failed to parse configured GeoIPOverride CIDR address (FD00::000F/11S): invalid CIDR address: FD00::000F/11S."+ - " Unable to use for GeoIP resolution!") + require.Contains(t, logOutput.String(), "Failed to parse configured GeoIPOverride address (FD00::000F/11S). Unable to use for GeoIP resolution!") }) t.Run("test-ipv4-match", func(t *testing.T) { @@ -196,7 +194,7 @@ func TestSortServerAds(t *testing.T) { server_utils.ResetTestState() t.Cleanup(func() { server_utils.ResetTestState() - geoIPOverrides = nil + geoNetOverrides = nil }) // A random IP that should geo-resolve to roughly the same location as the Madison server From 0d88c543c532441ea52c67a9f395fc60630a1c22 Mon Sep 17 00:00:00 2001 From: Brian Aydemir Date: Sat, 7 Dec 2024 08:40:00 -0600 Subject: [PATCH 84/86] Fix erroneous assumption that all IPs are IPv4 addresses --- director/sort.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/director/sort.go b/director/sort.go index 62dca2bf8..b0450f355 100644 --- a/director/sort.go +++ b/director/sort.go @@ -27,7 +27,6 @@ import ( "net/netip" "slices" "sort" - "strings" "time" "github.com/jellydator/ttlcache/v3" @@ -121,22 +120,24 @@ func unmarshalOverrides() error { // Ensure that we're starting with an empty slice. geoNetOverrides = nil - err := param.GeoIPOverrides.Unmarshal(&geoIPOverrides) - if err != nil { + if err := param.GeoIPOverrides.Unmarshal(&geoIPOverrides); err != nil { return err } for _, override := range geoIPOverrides { - var configuredCIDR string - - if strings.Contains(override.IP, "/") { - configuredCIDR = override.IP - } else { - configuredCIDR = override.IP + "/32" + var ipNet *net.IPNet + + if _, parsedNet, err := net.ParseCIDR(override.IP); err == nil { + ipNet = parsedNet + } else if ip := net.ParseIP(override.IP); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + ipNet = &net.IPNet{IP: ip4, Mask: net.CIDRMask(32, 32)} + } else if ip16 := ip.To16(); ip16 != nil { + ipNet = &net.IPNet{IP: ip16, Mask: net.CIDRMask(128, 128)} + } } - _, ipNet, err := net.ParseCIDR(configuredCIDR) - if err != nil { + if ipNet == nil { // Log the error, and continue looking for good configuration. log.Warningf("Failed to parse configured GeoIPOverride address (%s). Unable to use for GeoIP resolution!", override.IP) continue From 81e5bc64046eeb8a9ac7f47d82d596dc0cfab207 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Tue, 10 Dec 2024 19:55:57 +0000 Subject: [PATCH 85/86] Add `workflow_dispatch` as trigger for PR validation action Cannon pointed out this lets us re-run the action after linking an issue, since there's no trigger for "connected" and "disconnected" events. Best thing to do is still to link issues before the PR is officially opened, but that shouldn't be the only way to do things! --- .github/workflows/enforce-PR-labelling.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/enforce-PR-labelling.yml b/.github/workflows/enforce-PR-labelling.yml index 28fa6885f..e9d7f9ae7 100644 --- a/.github/workflows/enforce-PR-labelling.yml +++ b/.github/workflows/enforce-PR-labelling.yml @@ -4,6 +4,7 @@ on: pull_request: # one limitation here is that there's no trigger to re-run any time we "connect" or "disconnect" an issue types: [opened, edited, labeled, unlabeled, synchronize] + workflow_dispatch: jobs: validate-pr: From 5fe0cfc33351330804da2e2b07bc44f756a72d75 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 10 Dec 2024 16:44:14 -0600 Subject: [PATCH 86/86] Docker images: Put binaries into standard locations - pelican and osdf go into /usr/local/bin - pelican-server and osdf-server go into /usr/local/sbin Fixes #1744 --- images/Dockerfile | 22 +++++++++++----------- images/entrypoint.sh | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) mode change 100644 => 100755 images/entrypoint.sh diff --git a/images/Dockerfile b/images/Dockerfile index 30836869f..98d8573af 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -195,10 +195,10 @@ COPY --from=xrootd-plugin-builder /usr/lib64/libnlohmann_json_schema_validator.a COPY images/entrypoint.sh /entrypoint.sh # Copy here to reduce dependency on the pelican-build stage in the final-stage and x-base stage -COPY --from=pelican-build /pelican/dist/pelican_linux_amd64_v1/pelican /pelican/pelican -COPY --from=pelican-build /pelican/dist/pelican_linux_amd64_v1/pelican /pelican/osdf -RUN chmod +x /pelican/pelican \ - && chmod +x /pelican/osdf \ +COPY --from=pelican-build /pelican/dist/pelican_linux_amd64_v1/pelican /usr/local/bin/pelican +COPY --from=pelican-build /pelican/dist/pelican_linux_amd64_v1/pelican /usr/local/bin/osdf +RUN chmod +x /usr/local/bin/pelican \ + && chmod +x /usr/local/bin/osdf \ && chmod +x /entrypoint.sh ###################### @@ -206,23 +206,23 @@ RUN chmod +x /pelican/pelican \ ###################### FROM final-stage AS pelican-base -RUN rm -rf /pelican/osdf +RUN rm -f /usr/local/bin/osdf ###################### # OSDF base stage # ###################### FROM final-stage AS osdf-base -RUN rm -rf /pelican/pelican +RUN rm -f /usr/local/bin/pelican #################### # pelican/cache # #################### FROM pelican-base AS cache -RUN rm -rf /pelican/pelican -COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /pelican/pelican-server -RUN chmod +x /pelican/pelican-server +RUN rm -f /usr/local/bin/pelican +COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /usr/local/sbin/pelican-server +RUN chmod +x /usr/local/sbin/pelican-server # For now, we're only using pelican-server in the cache, but eventually we'll use it in all servers ENTRYPOINT [ "/entrypoint.sh", "pelican-server", "cache"] CMD [ "serve" ] @@ -264,8 +264,8 @@ CMD [ "serve" ] FROM osdf-base AS osdf-cache RUN rm -rf /pelican/osdf -COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /pelican/osdf-server -RUN chmod +x /pelican/osdf-server +COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /usr/local/sbin/osdf-server +RUN chmod +x /usr/local/sbin/osdf-server ENTRYPOINT [ "/entrypoint.sh" ,"osdf-server", "cache"] CMD [ "serve" ] diff --git a/images/entrypoint.sh b/images/entrypoint.sh old mode 100644 new mode 100755 index 4d8b24037..27763c1e4 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -114,7 +114,7 @@ if [ $# -ne 0 ]; then pelican) # Run pelican with the rest of the arguments echo "Running pelican with arguments: $@" - exec tini -- /pelican/pelican "$@" + exec tini -- /usr/local/bin/pelican "$@" # we shouldn't get here echo >&2 "Exec of tini failed!" exit 1 @@ -123,7 +123,7 @@ if [ $# -ne 0 ]; then # Our server-specific binary which may come with additional # features/system requirements (like Lotman) echo "Running pelican-server with arguments: $@" - exec tini -- /pelican/pelican-server "$@" + exec tini -- /usr/local/sbin/pelican-server "$@" # we shouldn't get here echo >&2 "Exec of tini failed!" exit 1 @@ -131,14 +131,14 @@ if [ $# -ne 0 ]; then osdf) # Run osdf with the rest of the arguments echo "Running osdf with arguments: $@" - exec tini -- /pelican/osdf "$@" + exec tini -- /usr/local/bin/osdf "$@" # we shouldn't get here echo >&2 "Exec of tini failed!" exit 1 ;; osdf-server) echo "Running osdf-server with arguments: $@" - exec tini -- /pelican/osdf-server "$@" + exec tini -- /usr/local/sbin/osdf-server "$@" # we shouldn't get here echo >&2 "Exec of tini failed!" exit 1