From fc892a5aa3c9aa79433d07211c87939fd648e0e9 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 17 Oct 2024 17:11:46 -0500 Subject: [PATCH 01/11] First pass at default config.d based configuration --- .goreleaser.yml | 20 ++++++++++++++++---- systemd/osdf-cache.yaml | 13 +++++++++++++ systemd/osdf-director.yaml | 13 +++++++++++++ systemd/osdf-origin.yaml | 13 +++++++++++++ systemd/osdf-registry.yaml | 13 +++++++++++++ systemd/pelican-cache.yaml | 13 +++++++++++++ systemd/pelican-director.yaml | 13 +++++++++++++ systemd/pelican-origin.yaml | 13 +++++++++++++ systemd/pelican-registry.yaml | 13 +++++++++++++ 9 files changed, 120 insertions(+), 4 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index b019704f3..10a825135 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -269,19 +269,31 @@ nfpms: dst: "/usr/lib/systemd/system/pelican-registry.service" - src: "systemd/pelican-cache.yaml" dst: "/etc/pelican/pelican-cache.yaml" - type: config|noreplace + type: config - src: "systemd/pelican-origin.yaml" dst: "/etc/pelican/pelican-origin.yaml" - type: config|noreplace + type: config - src: "systemd/pelican-director.yaml" dst: "/etc/pelican/pelican-director.yaml" - type: config|noreplace + type: config - src: "systemd/pelican-registry.yaml" dst: "/etc/pelican/pelican-registry.yaml" - type: config|noreplace + type: config - src: "systemd/pelican.logrotate" dst: "/etc/logrotate.d/pelican" type: config|noreplace + - dst: "/etc/pelican/config.d" + type: "dir" + file_info: + owner: root + group: root + mode: 0755 + - dst: "/usr/share/pelican/config.d" + type: "dir" + file_info: + owner: root + group: root + mode: 0755 - dst: "/var/log/pelican" type: "dir" file_info: diff --git a/systemd/osdf-cache.yaml b/systemd/osdf-cache.yaml index 0c9485ce1..a5169f6a5 100644 --- a/systemd/osdf-cache.yaml +++ b/systemd/osdf-cache.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# OSDF Pelican Cache configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs, including XRootD logs, to the log file # Debug: false Logging: diff --git a/systemd/osdf-director.yaml b/systemd/osdf-director.yaml index 940fbeb2a..37cb0b8c0 100644 --- a/systemd/osdf-director.yaml +++ b/systemd/osdf-director.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# OSDF Pelican Director configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs to the log file # Debug: false Logging: diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 4ea116cfe..f6cceab0f 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# OSDF Pelican Origin configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs, including XRootD logs, to the log file # Debug: false Logging: diff --git a/systemd/osdf-registry.yaml b/systemd/osdf-registry.yaml index c8c887bcb..3d2d44646 100644 --- a/systemd/osdf-registry.yaml +++ b/systemd/osdf-registry.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# OSDF Pelican Registry configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs to the log file # Debug: false Logging: diff --git a/systemd/pelican-cache.yaml b/systemd/pelican-cache.yaml index 5b48bc6d0..2bfcbcd7f 100644 --- a/systemd/pelican-cache.yaml +++ b/systemd/pelican-cache.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# Pelican Cache configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs, including XRootD logs, to the log file # Debug: false Logging: diff --git a/systemd/pelican-director.yaml b/systemd/pelican-director.yaml index 38b9d3f3a..27fc02025 100644 --- a/systemd/pelican-director.yaml +++ b/systemd/pelican-director.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# Pelican Director configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs, including XRootD logs, to the log file # Debug: false Logging: diff --git a/systemd/pelican-origin.yaml b/systemd/pelican-origin.yaml index 54fb4eb4a..e7f703cd2 100644 --- a/systemd/pelican-origin.yaml +++ b/systemd/pelican-origin.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# Pelican Origin configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs, including XRootD logs, to the log file # Debug: false Logging: diff --git a/systemd/pelican-registry.yaml b/systemd/pelican-registry.yaml index c06324a9d..df79dec6a 100644 --- a/systemd/pelican-registry.yaml +++ b/systemd/pelican-registry.yaml @@ -1,3 +1,16 @@ +--- +##################################################### +# Pelican Registry configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +ConfigLocations: + - "/usr/share/pelican/config.d" + - "/etc/pelican/config.d" + ## Enable Debug to send detailed logs, including XRootD logs, to the log file # Debug: false Logging: From 6f15231c87a8f9f0a0187287b5f7ba8c13194a4a Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 17 Oct 2024 17:14:30 -0500 Subject: [PATCH 02/11] Move common OSDF config to the federation-level config dir --- systemd/10-osdf-defaults.yaml | 16 ++++++++++++++++ systemd/osdf-cache.yaml | 5 ----- systemd/osdf-director.yaml | 4 ---- systemd/osdf-origin.yaml | 5 ----- systemd/osdf-registry.yaml | 4 ---- 5 files changed, 16 insertions(+), 18 deletions(-) create mode 100644 systemd/10-osdf-defaults.yaml diff --git a/systemd/10-osdf-defaults.yaml b/systemd/10-osdf-defaults.yaml new file mode 100644 index 000000000..cd3ee23ef --- /dev/null +++ b/systemd/10-osdf-defaults.yaml @@ -0,0 +1,16 @@ +--- +##################################################### +# Open Science Data Federation configuration +# +# DO NOT EDIT THIS FILE! It will be overwritten upon RPM upgrade. +# If you wish to make changes to the Pelican configuration, create files +# in /etc/pelican/config.d containing your changes. +##################################################### + +Federation: + DiscoveryUrl: "https://osg-htc.org" + +Server: + TLSCertificate: /etc/pki/tls/certs/pelican.crt + TLSKey: /etc/pki/tls/private/pelican.key + TLSCACertificateFile: /etc/pki/tls/certs/ca-bundle.crt diff --git a/systemd/osdf-cache.yaml b/systemd/osdf-cache.yaml index a5169f6a5..feaa2e880 100644 --- a/systemd/osdf-cache.yaml +++ b/systemd/osdf-cache.yaml @@ -18,11 +18,6 @@ Logging: ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. # Level: "Error" Server: - TLSCertificate: /etc/pki/tls/certs/pelican.crt - TLSKey: /etc/pki/tls/private/pelican.key - ## Use TLSCACertificateDirectory instead of TLSCACertificateFile to support auth from grid clients - TLSCACertificateFile: /etc/pki/tls/certs/ca-bundle.crt - # TLSCACertificateDirectory: /etc/grid-security/certificates EnableUI: false ## Set Hostname to the external DNS name this can be accessed over, if ## different than the current hostname. diff --git a/systemd/osdf-director.yaml b/systemd/osdf-director.yaml index 37cb0b8c0..00c875404 100644 --- a/systemd/osdf-director.yaml +++ b/systemd/osdf-director.yaml @@ -17,7 +17,3 @@ Logging: LogLocation: /var/log/pelican/osdf-director.log ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. # Level: "Error" -Server: - TLSCertificate: /etc/pki/tls/certs/pelican.crt - TLSKey: /etc/pki/tls/private/pelican.key - TLSCACertificateFile: /etc/pki/tls/certs/ca-bundle.crt diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index f6cceab0f..5d5eecff2 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -18,11 +18,6 @@ Logging: ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. # Level: "Error" Server: - TLSCertificate: /etc/pki/tls/certs/pelican.crt - TLSKey: /etc/pki/tls/private/pelican.key - ## Use TLSCACertificateDirectory instead of TLSCACertificateFile to support auth from grid clients - TLSCACertificateFile: /etc/pki/tls/certs/ca-bundle.crt - # TLSCACertificateDirectory: /etc/grid-security/certificates EnableUI: false ## Set Hostname to the external DNS name this can be accessed over, if ## different than the current hostname. diff --git a/systemd/osdf-registry.yaml b/systemd/osdf-registry.yaml index 3d2d44646..5ea2e32e9 100644 --- a/systemd/osdf-registry.yaml +++ b/systemd/osdf-registry.yaml @@ -17,7 +17,3 @@ Logging: LogLocation: /var/log/pelican/osdf-registry.log ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. # Level: "Error" -Server: - TLSCertificate: /etc/pki/tls/certs/pelican.crt - TLSKey: /etc/pki/tls/private/pelican.key - TLSCACertificateFile: /etc/pki/tls/certs/ca-bundle.crt From 318765a310fb8f51a6212f0ee7ed7fce5dc1812f Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 17 Oct 2024 17:27:53 -0500 Subject: [PATCH 03/11] Move common debugging configs to editable example file --- .goreleaser.yml | 3 +++ systemd/examples/90-debugging.yaml | 25 +++++++++++++++++++++++++ systemd/osdf-cache.yaml | 5 +---- systemd/osdf-director.yaml | 4 ---- systemd/osdf-origin.yaml | 5 +---- systemd/osdf-registry.yaml | 4 ---- systemd/pelican-cache.yaml | 5 +---- systemd/pelican-director.yaml | 5 +---- systemd/pelican-origin.yaml | 5 +---- systemd/pelican-registry.yaml | 5 +---- 10 files changed, 34 insertions(+), 32 deletions(-) create mode 100644 systemd/examples/90-debugging.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml index 10a825135..e1a099129 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -282,6 +282,9 @@ nfpms: - src: "systemd/pelican.logrotate" dst: "/etc/logrotate.d/pelican" type: config|noreplace + - src: "systemd/examples/90-debugging.yaml" + dst: "/etc/pelican/config.d/90-debugging.yaml" + type: config|noreplace - dst: "/etc/pelican/config.d" type: "dir" file_info: diff --git a/systemd/examples/90-debugging.yaml b/systemd/examples/90-debugging.yaml new file mode 100644 index 000000000..6cb3e2eba --- /dev/null +++ b/systemd/examples/90-debugging.yaml @@ -0,0 +1,25 @@ +--- +##################################################### +# Common Debugging Configuration +# +# Optional configuration for incr;easing log level verbosity +##################################################### + +## Set Debug: true to increase verbosity of logging in +## /var/log/pelican +# Debug: false + +## Uncomment the following sections to increase component-specific +## logging verbosity +# Logging: +# Level: "Trace" +# Cache: +# Http: "trace" +# Ofs: "trace" +# Pfc: "trace" +# Xrd: "trace" +# Origin: +# Http: "trace" +# Ofs: "trace" +# Scitokens: "trace" +# Xrootd: "trace" diff --git a/systemd/osdf-cache.yaml b/systemd/osdf-cache.yaml index feaa2e880..41ee06216 100644 --- a/systemd/osdf-cache.yaml +++ b/systemd/osdf-cache.yaml @@ -11,12 +11,9 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs, including XRootD logs, to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/osdf-cache.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" + Server: EnableUI: false ## Set Hostname to the external DNS name this can be accessed over, if diff --git a/systemd/osdf-director.yaml b/systemd/osdf-director.yaml index 00c875404..10481a50f 100644 --- a/systemd/osdf-director.yaml +++ b/systemd/osdf-director.yaml @@ -11,9 +11,5 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/osdf-director.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 5d5eecff2..2dbae9b43 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -11,12 +11,9 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs, including XRootD logs, to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/osdf-origin.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" + Server: EnableUI: false ## Set Hostname to the external DNS name this can be accessed over, if diff --git a/systemd/osdf-registry.yaml b/systemd/osdf-registry.yaml index 5ea2e32e9..b7012b898 100644 --- a/systemd/osdf-registry.yaml +++ b/systemd/osdf-registry.yaml @@ -11,9 +11,5 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/osdf-registry.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" diff --git a/systemd/pelican-cache.yaml b/systemd/pelican-cache.yaml index 2bfcbcd7f..f3f34699f 100644 --- a/systemd/pelican-cache.yaml +++ b/systemd/pelican-cache.yaml @@ -11,12 +11,9 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs, including XRootD logs, to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/pelican-cache.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" + Federation: ## The main URL for your federation. You must specify this. DiscoveryUrl: diff --git a/systemd/pelican-director.yaml b/systemd/pelican-director.yaml index 27fc02025..b59af179b 100644 --- a/systemd/pelican-director.yaml +++ b/systemd/pelican-director.yaml @@ -11,12 +11,9 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs, including XRootD logs, to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/pelican-director.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" + Federation: ## The main URL for your federation. You must specify this. DiscoveryUrl: diff --git a/systemd/pelican-origin.yaml b/systemd/pelican-origin.yaml index e7f703cd2..b91752bbe 100644 --- a/systemd/pelican-origin.yaml +++ b/systemd/pelican-origin.yaml @@ -11,12 +11,9 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs, including XRootD logs, to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/pelican-origin.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" + Federation: ## The main URL for your federation. You must specify this. DiscoveryUrl: diff --git a/systemd/pelican-registry.yaml b/systemd/pelican-registry.yaml index df79dec6a..07e50680b 100644 --- a/systemd/pelican-registry.yaml +++ b/systemd/pelican-registry.yaml @@ -11,12 +11,9 @@ ConfigLocations: - "/usr/share/pelican/config.d" - "/etc/pelican/config.d" -## Enable Debug to send detailed logs, including XRootD logs, to the log file -# Debug: false Logging: LogLocation: /var/log/pelican/pelican-registry.log - ## Valid Levels are Trace, Debug, Info, Warning, Error, Fatal and Panic. - # Level: "Error" + Federation: ## The main URL for your federation. You must specify this. DiscoveryUrl: From d0964df6bcd7ee172252aa9394eefdf759b34da8 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 11:39:00 -0500 Subject: [PATCH 04/11] Remove prescriptive cache / origin locations In the original XCache configurations, we forced oss.localroot to a location other than "/" for security reasons. We don't need to do this anymore because Pelican sets up the $(RunDirLocation)/export indirection --- systemd/osdf-cache.yaml | 2 +- systemd/osdf-origin.yaml | 1 - systemd/pelican-cache.yaml | 2 -- systemd/pelican-origin.yaml | 3 +-- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/systemd/osdf-cache.yaml b/systemd/osdf-cache.yaml index 41ee06216..a6a63a978 100644 --- a/systemd/osdf-cache.yaml +++ b/systemd/osdf-cache.yaml @@ -23,6 +23,6 @@ XRootD: ## Sitename is the resource name this cache is registered as under Topology. ## You must set this. Sitename: TOPOLOGY_RESOURCE_NAME + Cache: - DataLocation: "/mnt/osdf" RunLocation: /run/pelican/xrootd/osdf-cache diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 2dbae9b43..c799dd2a5 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -23,7 +23,6 @@ XRootD: ## Sitename is the resource name this origin is registered as under Topology. ## You must set this. Sitename: TOPOLOGY_RESOURCE_NAME - Mount: "/mnt/osdf" Port: 1095 Origin: NamespacePrefix: "/MY_NAMESPACE" diff --git a/systemd/pelican-cache.yaml b/systemd/pelican-cache.yaml index f3f34699f..771c72541 100644 --- a/systemd/pelican-cache.yaml +++ b/systemd/pelican-cache.yaml @@ -17,5 +17,3 @@ Logging: Federation: ## The main URL for your federation. You must specify this. DiscoveryUrl: -Cache: - DataLocation: "/mnt/pelican" diff --git a/systemd/pelican-origin.yaml b/systemd/pelican-origin.yaml index b91752bbe..87c2e773d 100644 --- a/systemd/pelican-origin.yaml +++ b/systemd/pelican-origin.yaml @@ -20,8 +20,7 @@ Federation: ## Set Hostname to the external DNS name this can be accessed over, if ## different than the current hostname. # Hostname: -XRootD: - Mount: "/mnt/pelican" + Origin: NamespacePrefix: "/MY_NAMESPACE" ## Enable multiuser to support Unix-based file ownership in the objects being served. From d1d6f6c30e5ff7bb7cb712e256fcaa6099e990e2 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 12:21:15 -0500 Subject: [PATCH 05/11] Drop old default origin port 1. For new origins, they should just use the default port 2. For proto-Pelican upgrades, we want them to use the default port so that admins can run the proto-Pelican and Pelican services side-by-side, validating the latter directly before turning off the former --- systemd/osdf-origin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index c799dd2a5..005bd95e5 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -23,7 +23,7 @@ XRootD: ## Sitename is the resource name this origin is registered as under Topology. ## You must set this. Sitename: TOPOLOGY_RESOURCE_NAME - Port: 1095 + Origin: NamespacePrefix: "/MY_NAMESPACE" Multiuser: false From f2a298903c6a1ba80c55498e362321dac11de793 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 12:43:23 -0500 Subject: [PATCH 06/11] Move multiuser origin config to an example file --- .goreleaser.yml | 3 +++ systemd/examples/60-origin-multiuser.yaml | 15 +++++++++++++++ systemd/osdf-origin.yaml | 1 - systemd/pelican-origin.yaml | 2 -- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 systemd/examples/60-origin-multiuser.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml index e1a099129..bb39b6cdb 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -282,6 +282,9 @@ nfpms: - src: "systemd/pelican.logrotate" dst: "/etc/logrotate.d/pelican" type: config|noreplace + - src: "systemd/examples/60-origin-multiuser.yaml" + dst: "/etc/pelican/config.d/60-origin-multiuser.yaml" + type: config|noreplace - src: "systemd/examples/90-debugging.yaml" dst: "/etc/pelican/config.d/90-debugging.yaml" type: config|noreplace diff --git a/systemd/examples/60-origin-multiuser.yaml b/systemd/examples/60-origin-multiuser.yaml new file mode 100644 index 000000000..25d9dd2e2 --- /dev/null +++ b/systemd/examples/60-origin-multiuser.yaml @@ -0,0 +1,15 @@ +--- +##################################################### +# Pelican Origin Multiuser Configuration (OPTIONAL) +# +# Optional configuration that allows a Pelican origin to switch users +# for reads/writes. Enable this configuration if your origin is +# serving POSIX-mounted data that is also accessible from another +# POSIX system (e.g., an HTCondor Access Point) +##################################################### + +## If you need Pelican multiuser behavior, uncomment the following +## lines and set both values to 'true' +# Origin: +# Multiuser: false +# ScitokensMapSubject: false diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 005bd95e5..76675a8ba 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -26,5 +26,4 @@ XRootD: Origin: NamespacePrefix: "/MY_NAMESPACE" - Multiuser: false RunLocation: /run/pelican/xrootd/osdf-origin diff --git a/systemd/pelican-origin.yaml b/systemd/pelican-origin.yaml index 87c2e773d..34c28e34f 100644 --- a/systemd/pelican-origin.yaml +++ b/systemd/pelican-origin.yaml @@ -23,5 +23,3 @@ Federation: Origin: NamespacePrefix: "/MY_NAMESPACE" - ## Enable multiuser to support Unix-based file ownership in the objects being served. - # Multiuser: false From 45068080e501d38308fa3f91982e3117c0c4938d Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 13:08:22 -0500 Subject: [PATCH 07/11] Move Pelican federation-facing config to an example file --- .goreleaser.yml | 3 +++ systemd/examples/10-federation.yaml | 15 +++++++++++++++ systemd/pelican-cache.yaml | 4 ---- systemd/pelican-director.yaml | 4 ---- systemd/pelican-origin.yaml | 7 ------- systemd/pelican-registry.yaml | 4 ---- 6 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 systemd/examples/10-federation.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml index bb39b6cdb..be7b0133e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -282,6 +282,9 @@ nfpms: - src: "systemd/pelican.logrotate" dst: "/etc/logrotate.d/pelican" type: config|noreplace + - src: "systemd/examples/10-federation.yaml" + dst: "/etc/pelican/config.d/10-federation.yaml" + type: config|noreplace - src: "systemd/examples/60-origin-multiuser.yaml" dst: "/etc/pelican/config.d/60-origin-multiuser.yaml" type: config|noreplace diff --git a/systemd/examples/10-federation.yaml b/systemd/examples/10-federation.yaml new file mode 100644 index 000000000..72b846111 --- /dev/null +++ b/systemd/examples/10-federation.yaml @@ -0,0 +1,15 @@ +--- +##################################################### +# Federation configuration +# +# Required configuration to integrate with your federation +##################################################### + +Federation: + ## The main URL for your federation. You must specify this. + DiscoveryUrl: + +## If the external DNS name of your host is different from the output +## of `hostname -f`, set Hostname to the external DNS name +# Server: +# Hostname: diff --git a/systemd/pelican-cache.yaml b/systemd/pelican-cache.yaml index 771c72541..fbe67dd57 100644 --- a/systemd/pelican-cache.yaml +++ b/systemd/pelican-cache.yaml @@ -13,7 +13,3 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/pelican-cache.log - -Federation: - ## The main URL for your federation. You must specify this. - DiscoveryUrl: diff --git a/systemd/pelican-director.yaml b/systemd/pelican-director.yaml index b59af179b..058be6b59 100644 --- a/systemd/pelican-director.yaml +++ b/systemd/pelican-director.yaml @@ -13,7 +13,3 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/pelican-director.log - -Federation: - ## The main URL for your federation. You must specify this. - DiscoveryUrl: diff --git a/systemd/pelican-origin.yaml b/systemd/pelican-origin.yaml index 34c28e34f..99e5bbd60 100644 --- a/systemd/pelican-origin.yaml +++ b/systemd/pelican-origin.yaml @@ -14,12 +14,5 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/pelican-origin.log -Federation: - ## The main URL for your federation. You must specify this. - DiscoveryUrl: - ## Set Hostname to the external DNS name this can be accessed over, if - ## different than the current hostname. - # Hostname: - Origin: NamespacePrefix: "/MY_NAMESPACE" diff --git a/systemd/pelican-registry.yaml b/systemd/pelican-registry.yaml index 07e50680b..a9056d70b 100644 --- a/systemd/pelican-registry.yaml +++ b/systemd/pelican-registry.yaml @@ -13,7 +13,3 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/pelican-registry.log - -Federation: - ## The main URL for your federation. You must specify this. - DiscoveryUrl: From decfa8890dcc792df58b047f6e3ed951e6955bbd Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 13:08:34 -0500 Subject: [PATCH 08/11] Move origin export config to an example file --- .goreleaser.yml | 3 ++ systemd/examples/20-origin-exports.yaml | 47 +++++++++++++++++++++++++ systemd/osdf-origin.yaml | 1 - systemd/pelican-origin.yaml | 3 -- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 systemd/examples/20-origin-exports.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml index be7b0133e..dc184d93a 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -285,6 +285,9 @@ nfpms: - src: "systemd/examples/10-federation.yaml" dst: "/etc/pelican/config.d/10-federation.yaml" type: config|noreplace + - src: "systemd/examples/20-origin-exports.yaml" + dst: "/etc/pelican/config.d/20-origin-exports.yaml" + type: config|noreplace - src: "systemd/examples/60-origin-multiuser.yaml" dst: "/etc/pelican/config.d/60-origin-multiuser.yaml" type: config|noreplace diff --git a/systemd/examples/20-origin-exports.yaml b/systemd/examples/20-origin-exports.yaml new file mode 100644 index 000000000..408902678 --- /dev/null +++ b/systemd/examples/20-origin-exports.yaml @@ -0,0 +1,47 @@ +--- +##################################################### +# Origin export configuration +# +# Required configuration for sharing data through your origin service +# +# See Origin config here: +# https://docs.pelicanplatform.org/federating-your-data +# +# If serving an S3 origin, see: +# https://docs.pelicanplatform.org/federating-your-data/s3-backend +##################################################### + +Origin: + # s3 or posix + StorageType: + + # If 'StorageType: s3', uncomment and fill out the following: + # S3ServiceUrl: + # S3Region: + # S3UrlStyle: + + # List of namespaces and backing filesystem or object store to serve + Exports: + # FederationPrefix is the namespace that your users will interact + # with to access data through your origin + - FederationPrefix: + # Possible capabilities: ["Reads", "PublicReads", "Writes", "DirectReads", "Listings"] + # https://docs.pelicanplatform.org/federating-your-data#origin-and-namespace-capabilities + Capabilities: + + # If 'StorageType: posix', uncomment and fill out the following + # with the path to the POSIX filesystem to serve + # StoragePrefix: + + # If 'StorageType: s3', uncomment and fill out the following: + # S3Bucket: + + # If 'StorageType: s3', uncomment and fill out the following if + # they are required by the underlying object store: + # S3AccessKeyfile: + # S3SecretKeyfile: + + # See multi-export configuration details here: + # https://docs.pelicanplatform.org/federating-your-data#multi-export-origins + # - FederationPrefix: + # Capabilities: diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 76675a8ba..71fecb2ee 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -25,5 +25,4 @@ XRootD: Sitename: TOPOLOGY_RESOURCE_NAME Origin: - NamespacePrefix: "/MY_NAMESPACE" RunLocation: /run/pelican/xrootd/osdf-origin diff --git a/systemd/pelican-origin.yaml b/systemd/pelican-origin.yaml index 99e5bbd60..898606cb3 100644 --- a/systemd/pelican-origin.yaml +++ b/systemd/pelican-origin.yaml @@ -13,6 +13,3 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/pelican-origin.log - -Origin: - NamespacePrefix: "/MY_NAMESPACE" From 3bc19341edacbf1bfbb0c88b6fe3042fe40f22e4 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 13:14:20 -0500 Subject: [PATCH 09/11] Move web UI config to example file --- systemd/examples/50-webui.yaml | 11 +++++++++++ systemd/osdf-cache.yaml | 5 ----- systemd/osdf-origin.yaml | 5 ----- 3 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 systemd/examples/50-webui.yaml diff --git a/systemd/examples/50-webui.yaml b/systemd/examples/50-webui.yaml new file mode 100644 index 000000000..5c9138467 --- /dev/null +++ b/systemd/examples/50-webui.yaml @@ -0,0 +1,11 @@ +--- +##################################################### +# Pelican admin web interface configuration +# +# Optional configuration. See: +# https://docs.pelicanplatform.org/federating-your-data#login-to-admin-website +##################################################### + +## Set to 'true' to enable the admin web interface +Server: + EnableUI: false diff --git a/systemd/osdf-cache.yaml b/systemd/osdf-cache.yaml index a6a63a978..ae4178b11 100644 --- a/systemd/osdf-cache.yaml +++ b/systemd/osdf-cache.yaml @@ -14,11 +14,6 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/osdf-cache.log -Server: - EnableUI: false - ## Set Hostname to the external DNS name this can be accessed over, if - ## different than the current hostname. - # Hostname: XRootD: ## Sitename is the resource name this cache is registered as under Topology. ## You must set this. diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 71fecb2ee..6c23d2f33 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -14,11 +14,6 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/osdf-origin.log -Server: - EnableUI: false - ## Set Hostname to the external DNS name this can be accessed over, if - ## different than the current hostname. - # Hostname: XRootD: ## Sitename is the resource name this origin is registered as under Topology. ## You must set this. From fc7b716d76884282237b88e2eb2499d30d805b07 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 13:15:55 -0500 Subject: [PATCH 10/11] Move Topology specific configuration to example file --- systemd/examples/15-osdf.yaml | 11 +++++++++++ systemd/osdf-cache.yaml | 5 ----- systemd/osdf-origin.yaml | 5 ----- 3 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 systemd/examples/15-osdf.yaml diff --git a/systemd/examples/15-osdf.yaml b/systemd/examples/15-osdf.yaml new file mode 100644 index 000000000..b4f77316b --- /dev/null +++ b/systemd/examples/15-osdf.yaml @@ -0,0 +1,11 @@ +--- +##################################################### +# OSDF configuration +# +# Required configuration to integrate your service with the OSDF +##################################################### + +XRootD: + ## Sitename is the resource name this cache is registered as under Topology. + ## You must set this. + Sitename: TOPOLOGY_RESOURCE_NAME diff --git a/systemd/osdf-cache.yaml b/systemd/osdf-cache.yaml index ae4178b11..85deb8db6 100644 --- a/systemd/osdf-cache.yaml +++ b/systemd/osdf-cache.yaml @@ -14,10 +14,5 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/osdf-cache.log -XRootD: - ## Sitename is the resource name this cache is registered as under Topology. - ## You must set this. - Sitename: TOPOLOGY_RESOURCE_NAME - Cache: RunLocation: /run/pelican/xrootd/osdf-cache diff --git a/systemd/osdf-origin.yaml b/systemd/osdf-origin.yaml index 6c23d2f33..6cc3c7431 100644 --- a/systemd/osdf-origin.yaml +++ b/systemd/osdf-origin.yaml @@ -14,10 +14,5 @@ ConfigLocations: Logging: LogLocation: /var/log/pelican/osdf-origin.log -XRootD: - ## Sitename is the resource name this origin is registered as under Topology. - ## You must set this. - Sitename: TOPOLOGY_RESOURCE_NAME - Origin: RunLocation: /run/pelican/xrootd/osdf-origin From bf2aae015040a6f1a1cc595f7eb1f0f176bdee6b Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Fri, 18 Oct 2024 13:34:06 -0500 Subject: [PATCH 11/11] Comment-out all origin export config Since we only have a single pelican-server RPM for the cache, director, origin, and registry we need to comment this incomplete origin-specific config out --- systemd/examples/20-origin-exports.yaml | 45 ++++++++++++++----------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/systemd/examples/20-origin-exports.yaml b/systemd/examples/20-origin-exports.yaml index 408902678..e1c30ae1f 100644 --- a/systemd/examples/20-origin-exports.yaml +++ b/systemd/examples/20-origin-exports.yaml @@ -2,7 +2,7 @@ ##################################################### # Origin export configuration # -# Required configuration for sharing data through your origin service +# The following sections describe required andoptional origin configurations # # See Origin config here: # https://docs.pelicanplatform.org/federating-your-data @@ -11,37 +11,42 @@ # https://docs.pelicanplatform.org/federating-your-data/s3-backend ##################################################### -Origin: - # s3 or posix - StorageType: +## If you are running an origin, uncomment this line +# Origin: + ## If you are running an origin, uncomment this line and set its + ## value to 's3' or 'posix' + # StorageType: - # If 'StorageType: s3', uncomment and fill out the following: + ## If 'StorageType: s3', uncomment and fill out the following: # S3ServiceUrl: # S3Region: # S3UrlStyle: - # List of namespaces and backing filesystem or object store to serve - Exports: - # FederationPrefix is the namespace that your users will interact - # with to access data through your origin - - FederationPrefix: - # Possible capabilities: ["Reads", "PublicReads", "Writes", "DirectReads", "Listings"] - # https://docs.pelicanplatform.org/federating-your-data#origin-and-namespace-capabilities - Capabilities: + ## If you are running an origin, uncomment this line to specify the + ## list of namespaces and backing filesystem or object store to serve + # Exports: + ## If you are running an origin, uncomment this line to specify + ## the namespace that you would like to serve with the federation + ## via your origin + # - FederationPrefix: + ## If you are running an origin, uncomment this line to specify + ## the capabilities associated with this namespace + ## https://docs.pelicanplatform.org/federating-your-data#origin-and-namespace-capabilities + # Capabilities: - # If 'StorageType: posix', uncomment and fill out the following - # with the path to the POSIX filesystem to serve + ## If 'StorageType: posix', uncomment and fill out the following + ## with the path to the POSIX filesystem to serve # StoragePrefix: - # If 'StorageType: s3', uncomment and fill out the following: + ## If 'StorageType: s3', uncomment and fill out the following: # S3Bucket: - # If 'StorageType: s3', uncomment and fill out the following if - # they are required by the underlying object store: + ## If 'StorageType: s3', uncomment and fill out the following if + ## they are required by the underlying object store: # S3AccessKeyfile: # S3SecretKeyfile: - # See multi-export configuration details here: - # https://docs.pelicanplatform.org/federating-your-data#multi-export-origins + ## OPTIONAL: See multi-export configuration details here: + ## https://docs.pelicanplatform.org/federating-your-data#multi-export-origins # - FederationPrefix: # Capabilities: