diff --git a/.github/scripts/validate-parameters/main.py b/.github/scripts/validate-parameters/main.py index d5447ac7f..bbd24dd2a 100644 --- a/.github/scripts/validate-parameters/main.py +++ b/.github/scripts/validate-parameters/main.py @@ -15,7 +15,8 @@ "OIDCAuthenticationRequirements", "AuthorizationTemplates", "IPMapping", - "Exports" + "Exports", + "Lots" ] diff --git a/client/director.go b/client/director.go index 9068d10be..94909d2f9 100644 --- a/client/director.go +++ b/client/director.go @@ -28,40 +28,18 @@ import ( "strconv" "strings" - "github.com/pelicanplatform/pelican/config" - namespaces "github.com/pelicanplatform/pelican/namespaces" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + + "github.com/pelicanplatform/pelican/config" + namespaces "github.com/pelicanplatform/pelican/namespaces" + "github.com/pelicanplatform/pelican/utils" ) type directorResponse struct { Error string `json:"error"` } -// Simple parser to that takes a "values" string from a header and turns it -// into a map of key/value pairs -func HeaderParser(values string) (retMap map[string]string) { - retMap = map[string]string{} - - // Some headers might not have values, such as the - // X-OSDF-Authorization header when the resource is public - if values == "" { - return - } - - mapPairs := strings.Split(values, ",") - for _, pair := range mapPairs { - // Remove any unwanted spaces - pair = strings.ReplaceAll(pair, " ", "") - - // Break out key/value pairs and put in the map - split := strings.Split(pair, "=") - retMap[split[0]] = split[1] - } - - return retMap -} - // Given the Director response, create the ordered list of caches // and store it as namespace.SortedDirectorCaches func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Namespace, err error) { @@ -70,7 +48,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name err = errors.New("Pelican director did not include mandatory X-Pelican-Namespace header in response") return } - xPelicanNamespace := HeaderParser(pelicanNamespaceHdr[0]) + xPelicanNamespace := utils.HeaderParser(pelicanNamespaceHdr[0]) namespace.Path = xPelicanNamespace["namespace"] namespace.UseTokenOnRead, _ = strconv.ParseBool(xPelicanNamespace["require-token"]) namespace.ReadHTTPS, _ = strconv.ParseBool(xPelicanNamespace["readhttps"]) @@ -82,7 +60,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name //So it's a map entry - HeaderParser returns a max entry //We want to appen the value for _, authEntry := range dirResp.Header.Values("X-Pelican-Authorization") { - parsedEntry := HeaderParser(authEntry) + parsedEntry := utils.HeaderParser(authEntry) xPelicanAuthorization = append(xPelicanAuthorization, parsedEntry["issuer"]) } namespace.Issuer = xPelicanAuthorization @@ -90,7 +68,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name var xPelicanTokenGeneration map[string]string if len(dirResp.Header.Values("X-Pelican-Token-Generation")) > 0 { - xPelicanTokenGeneration = HeaderParser(dirResp.Header.Values("X-Pelican-Token-Generation")[0]) + xPelicanTokenGeneration = utils.HeaderParser(dirResp.Header.Values("X-Pelican-Token-Generation")[0]) // Instantiate the cred gen struct namespace.CredentialGen = &namespaces.CredentialGeneration{} diff --git a/client/director_test.go b/client/director_test.go index b8f8d08c9..c580234ab 100644 --- a/client/director_test.go +++ b/client/director_test.go @@ -29,21 +29,9 @@ import ( "github.com/stretchr/testify/assert" namespaces "github.com/pelicanplatform/pelican/namespaces" + "github.com/pelicanplatform/pelican/utils" ) -func TestHeaderParser(t *testing.T) { - header1 := "namespace=/foo/bar, issuer = https://get-your-tokens.org, readhttps=False" - newMap1 := HeaderParser(header1) - - assert.Equal(t, "/foo/bar", newMap1["namespace"]) - assert.Equal(t, "https://get-your-tokens.org", newMap1["issuer"]) - assert.Equal(t, "False", newMap1["readhttps"]) - - header2 := "" - newMap2 := HeaderParser(header2) - assert.Equal(t, map[string]string{}, newMap2) -} - func TestGetCachesFromDirectorResponse(t *testing.T) { // Construct the Director's Response, comprising headers and a body directorHeaders := make(map[string][]string) @@ -125,7 +113,7 @@ func TestCreateNsFromDirectorResp(t *testing.T) { var xPelicanAuthorization map[string]string var issuer string if len(directorResponse.Header.Values("X-Pelican-Authorization")) > 0 { - xPelicanAuthorization = HeaderParser(directorResponse.Header.Values("X-Pelican-Authorization")[0]) + xPelicanAuthorization = utils.HeaderParser(directorResponse.Header.Values("X-Pelican-Authorization")[0]) issuer = xPelicanAuthorization["issuer"] } diff --git a/config/config.go b/config/config.go index 56e473d92..145f2e242 100644 --- a/config/config.go +++ b/config/config.go @@ -880,12 +880,16 @@ func InitServer(ctx context.Context, currentServers ServerType) error { viper.SetDefault("Origin.Multiuser", true) viper.SetDefault("Director.GeoIPLocation", "/var/cache/pelican/maxmind/GeoLite2-City.mmdb") viper.SetDefault("Registry.DbLocation", "/var/lib/pelican/registry.sqlite") + // The lotman db will actually take this path and create the lot at /path/.lot/lotman_cpp.sqlite + viper.SetDefault("Lotman.DbLocation", "/var/lib/pelican") viper.SetDefault("Monitoring.DataLocation", "/var/lib/pelican/monitoring/data") viper.SetDefault("Shoveler.QueueDirectory", "/var/spool/pelican/shoveler/queue") viper.SetDefault("Shoveler.AMQPTokenLocation", "/etc/pelican/shoveler-token") } else { viper.SetDefault("Director.GeoIPLocation", filepath.Join(configDir, "maxmind", "GeoLite2-City.mmdb")) viper.SetDefault("Registry.DbLocation", filepath.Join(configDir, "ns-registry.sqlite")) + // Lotdb will live at /.lot/lotman_cpp.sqlite + viper.SetDefault("Lotman.DbLocation", configDir) viper.SetDefault("Monitoring.DataLocation", filepath.Join(configDir, "monitoring/data")) viper.SetDefault("Shoveler.QueueDirectory", filepath.Join(configDir, "shoveler/queue")) viper.SetDefault("Shoveler.AMQPTokenLocation", filepath.Join(configDir, "shoveler-token")) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index ee689aa9e..2cd7c8db4 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -874,6 +874,12 @@ description: >- number of connections to this value. type: int default: none +--- +name: Cache.EnableLotman +description: >- + LotMan is a library that provides management of storage space in the cache. +type: bool +default: false components: ["cache"] --- name: Cache.PermittedNamespaces @@ -1916,3 +1922,65 @@ description: >- type: string default: none components: ["plugin"] +--- +############################ +# LotMan-level configs # +############################ +name: Lotman.DbLocation +description: >- + The prefix indicating where LotMan should store its lot database. For the provided path, the database + will be stored at /.lot/lotman_cpp.sqlite +type: filename +root_default: /var/run/pelican +default: $ConfigBase +components: ["cache"] +--- +name: Lotman.LibLocation +description: >- + The location of the system's installed LotMan library (libLotMan.so). When unset, the system will attempt to find Lotman + at these fallback paths: + - /usr/lib64/libLotMan.so + - /usr/local/lib64/libLotMan.so + - /opt/local/lib64/libLotMan.so +type: filename +default: none +components: ["cache"] +--- +name: Lotman.EnableAPI +description: >- + Whether Lotman should enable its CRUD web endpoints. If true, administrators with an appropriately-signed token can interface + with Lotman via HTTP. Otherwise, lots are only configurable via the Pelican configuration file at the cache. +type: bool +default: false +components: ["cache"] +--- +name: Lotman.Lots +description: >- + Declarative configuration for LotMan. This is a list of objects, each of which describes a "lot". Every lot + can be defined with the following: + - `LotName`: REQUIRED. The name of the lot. This is used to identify the lot in the LotMan database. + - `Owner`: REQUIRED. A string identifying the owner of the lot's data (as opposed to someone who can modify the lot itself). + The Owner field should generally be set to the issue for the lot's namespace path. For example, if the lot + tracks namespace `/foo/bar`, the owner might be set to `https://registry.com/api/v1.0/registry/foo/bar`. + - `Paths`: OPTIONAL. A list of path objects, each of which describes a path that should be managed by the lot. + - `Path`: REQUIRED. The path to be managed by the lot. + - `Recursive`: REQUIRED. A boolean indicating whether the path should be managed recursively. If true, the lot will + manage all files and directories under the specified path. + - `ManagementPolicyAttrs`: REQUIRED. The lot's management policy attributes object. This contains information about resources the lot should + be allocated, and how it should be managed. + - `DedicatedGB`: REQUIRED. The amount of storage, in GB, that should be dedicated to the lot. This means the lot can assume it + always has access to this quantity. + - `OpportunisticGB`: REQUIRED. The amount of opportunistic storage, in GB, the lot should have access to, when storage is available. + - `MaxNumObjects`: REQUIRED. The maximum number of objects a lot is allowed to store. + - `CreationTime`: REQUIRED. A unix timestamp indicating when the lot should begin being considered valid. Times in the future indicate + the lot should not be considered valid until that time. + - `ExpirationTime`: REQUIRED. A unix timestamp indicating when the lot expires. Lots may continue to function after expiration, but lot + data owners should recognize the storage is at-will and may be pre-empted at any time. + - `DeletionTime`: REQUIRED. A unix timestamp indicating when the lot and its associated data should be deleted. + + Note that example configurations can be found in lotman/resources/lots-config.yaml + For more information about LotMan configuration, see: + https://github.com/pelicanplatform/lotman +type: object +default: none +components: ["cache"] diff --git a/docs/scopes.yaml b/docs/scopes.yaml index 458427ec1..eaf647577 100644 --- a/docs/scopes.yaml +++ b/docs/scopes.yaml @@ -128,3 +128,30 @@ description: >- For granting object staging permissions to the bearer of the token. This scope must also posses a path to be valid, eg `storage.stage:/foo/bar` issuedBy: ["origin"] acceptedBy: ["origin", "cache"] +--- +############################ +# Lotman Scopes # +############################ +name: "lot.create" +description: >- + For creating a new lot +issuedBy: ["origin"] +acceptedBy: ["cache"] +--- +name: "lot.read" +description: >- + For getting/reading the contents of a lot from a cache +issuedBy: ["origin"] +acceptedBy: ["cache"] +--- +name: "lot.modify" +description: >- + For modifying the contents of a lot in a cache +issuedBy: ["origin"] +acceptedBy: ["cache"] +--- +name: "lot.delete" +description: >- + For deleting a lot from a cache +issuedBy: ["origin"] +acceptedBy: ["cache"] diff --git a/generate/scope_generator.go b/generate/scope_generator.go index 1e20d7f89..3125f75d0 100644 --- a/generate/scope_generator.go +++ b/generate/scope_generator.go @@ -87,6 +87,7 @@ func GenTokenScope() { scopes := make([]ScopeName, 0) storageScopes := make([]ScopeName, 0) + lotmanScopes := make([]ScopeName, 0) for i := 0; i < len(values); i++ { entry := values[i].(map[string]interface{}) @@ -109,6 +110,8 @@ func GenTokenScope() { if strings.HasPrefix(scopeName, "storage") { displayName = strings.TrimSuffix(displayName, ":") storageScopes = append(storageScopes, ScopeName{Raw: scopeName, Display: displayName}) + } else if strings.HasPrefix(scopeName, "lot") { + lotmanScopes = append(lotmanScopes, ScopeName{Raw: scopeName, Display: displayName}) } else { scopes = append(scopes, ScopeName{Raw: scopeName, Display: displayName}) } @@ -124,9 +127,11 @@ func GenTokenScope() { err = tokenTemplate.Execute(f, struct { Scopes []ScopeName StorageScopes []ScopeName + LotmanScopes []ScopeName }{ Scopes: scopes, StorageScopes: storageScopes, + LotmanScopes: lotmanScopes, }) if err != nil { @@ -134,7 +139,8 @@ func GenTokenScope() { } } -var tokenTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. +var tokenTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT THIS FILE. +// To make changes to source, see generate/scope_generator.go and docs/scopes.yaml /*************************************************************** * * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research @@ -170,6 +176,11 @@ const ( {{- range $idx, $scope := .StorageScopes}} {{$scope.Display}} TokenScope = "{{$scope.Raw}}" {{- end}} + + // Lotman Scopes + {{- range $idx, $scope := .LotmanScopes}} + {{$scope.Display}} TokenScope = "{{$scope.Raw}}" + {{- end}} ) func (s TokenScope) String() string { diff --git a/go.mod b/go.mod index c501d8644..bb56128ba 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ go 1.21 require ( github.com/JGLTechnologies/gin-rate-limit v1.5.4 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 + github.com/ebitengine/purego v0.6.0 github.com/gin-gonic/gin v1.9.1 github.com/glebarez/sqlite v1.10.0 github.com/go-ini/ini v1.67.0 diff --git a/go.sum b/go.sum index 266dc7a93..9f71ea6bb 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.6.0 h1:Yo9uBc1x+ETQbfEaf6wcBsjrQfCEnh/gaGUg7lguEJY= +github.com/ebitengine/purego v0.6.0/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/elastic/go-sysinfo v1.11.2 h1:mcm4OSYVMyws6+n2HIVMGkln5HOpo5Ie1ZmbbNn0jg4= diff --git a/launchers/cache_serve.go b/launchers/cache_serve.go index a3fbabda2..159fe5b69 100644 --- a/launchers/cache_serve.go +++ b/launchers/cache_serve.go @@ -26,17 +26,20 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "github.com/pelicanplatform/pelican/broker" "github.com/pelicanplatform/pelican/cache" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/daemon" "github.com/pelicanplatform/pelican/launcher_utils" + "github.com/pelicanplatform/pelican/lotman" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/xrootd" - log "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" ) func CacheServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, modules config.ServerType) (server_structs.XRootDServer, error) { @@ -58,6 +61,19 @@ func CacheServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, m return nil, err } + // Register Lotman + if param.Cache_EnableLotman.GetBool() { + // Register the web endpoints + if param.Lotman_EnableAPI.GetBool() { + log.Debugln("Registering Lotman API") + lotman.RegisterLotman(ctx, engine.Group("/")) + } + // Bind the c library funcs to Go + if success := lotman.InitLotman(); !success { + return nil, errors.New("Failed to initialize lotman") + } + } + broker.RegisterBrokerCallback(ctx, engine.Group("/")) broker.LaunchNamespaceKeyMaintenance(ctx, egrp) configPath, err := xrootd.ConfigXrootd(ctx, false) diff --git a/lotman/lotman.go b/lotman/lotman.go new file mode 100644 index 000000000..98f1a2bbb --- /dev/null +++ b/lotman/lotman.go @@ -0,0 +1,39 @@ +//go:build windows || darwin || (linux && ppc64le) + +/*************************************************************** +* +* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research +* +* Licensed under the Apache License, Version 2.0 (the "License"); you +* may not use this file except in compliance with the License. You may +* obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +***************************************************************/ + +// LotMan is only supported on Linux at the moment. This file is a placeholder for other platforms and is +// intended to export any functions that might be called outside of the package +package lotman + +import ( + "context" + + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +func RegisterLotman(ctx context.Context, router *gin.RouterGroup) { + log.Warningln("LotMan is not supported on this platform. Skipping...") +} + +func InitLotman() bool { + log.Warningln("LotMan is not supported on this platform. Skipping...") + return false +} diff --git a/lotman/lotman_linux.go b/lotman/lotman_linux.go new file mode 100644 index 000000000..fd09d1827 --- /dev/null +++ b/lotman/lotman_linux.go @@ -0,0 +1,688 @@ +//go:build linux && !ppc64le + +/*************************************************************** +* +* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research +* +* Licensed under the Apache License, Version 2.0 (the "License"); you +* may not use this file except in compliance with the License. You may +* obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +***************************************************************/ + +// The LotMan library is used for managing storage in Pelican caches. For more information, see: +// https://github.com/pelicanplatform/lotman +package lotman + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "runtime" + "sync" + "unsafe" + + "github.com/ebitengine/purego" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/pelicanplatform/pelican/param" +) + +var ( + // A mutex for the Lotman caller context -- make sure we're calling lotman functions with the appropriate caller + callerMutex = sync.RWMutex{} + + initializedLots []Lot + + // Lotman func signatures we'll bind to the underlying C headers + LotmanVersion func() string + // Strings in go are immutable, so they're actually passed to the underlying SO as `const`. To get dynamic + // output, we need to pass a pointer to a byte array + LotmanAddLot func(lotJSON string, errMsg *[]byte) int32 + LotmanGetLotJSON func(lotName string, recursive bool, output *[]byte, errMsg *[]byte) int32 + LotmanAddToLot func(additionsJSON string, errMsg *[]byte) int32 + LotmanUpdateLot func(updateJSON string, errMsg *[]byte) int32 + LotmanDeleteLotsRecursive func(lotName string, errMsg *[]byte) int32 + + // Auxilliary functions + LotmanLotExists func(lotName string, errMsg *[]byte) int32 + LotmanSetContextStr func(contextKey string, contextValue string, errMsg *[]byte) int32 + // Functions that would normally take a char *** as an argument take an *unsafe.Pointer instead because + // these functions are responsible for allocating and deallocating the memory for the char ***. The Go + // runtime will handle the memory management for the *unsafe.Pointer. + LotmanGetLotOwners func(lotName string, recursive bool, output *unsafe.Pointer, errMsg *[]byte) int32 + // Here, getSelf means get the lot proper if it's a self parent + LotmanGetLotParents func(lotName string, recursive bool, getSelf bool, output *unsafe.Pointer, errMsg *[]byte) int32 + LotmanGetLotsFromDir func(dir string, recursive bool, output *unsafe.Pointer, errMsg *[]byte) int32 +) + +type ( + Int64FromFloat struct { + Value int64 + } + + LotPaths struct { + Path string `json:"path" mapstructure:"Path"` + Recursive bool `json:"recursive" mapstructure:"Recursive"` + LotName string `json:"lot_name,omitempty"` + } + + LotValueMapInt struct { + LotName string `json:"lot_name"` + Value Int64FromFloat `json:"value"` + } + + LotValueMapFloat struct { + LotName string `json:"lot_name"` + Value float64 `json:"value"` + } + + MPA struct { + DedicatedGB *float64 `json:"dedicated_GB,omitempty" mapstructure:"DedicatedGB"` + OpportunisticGB *float64 `json:"opportunistic_GB,omitempty" mapstructure:"OpportunisticGB"` + MaxNumObjects *Int64FromFloat `json:"max_num_objects,omitempty" mapstructure:"MaxNumObjects"` + CreationTime *Int64FromFloat `json:"creation_time,omitempty" mapstructure:"CreationTime"` + ExpirationTime *Int64FromFloat `json:"expiration_time,omitempty" mapstructure:"ExpirationTime"` + DeletionTime *Int64FromFloat `json:"deletion_time,omitempty" mapstructure:"DeletionTime"` + } + + RestrictiveMPA struct { + DedicatedGB LotValueMapFloat `json:"dedicated_GB"` + OpportunisticGB LotValueMapFloat `json:"opportunistic_GB"` + MaxNumObjects LotValueMapInt `json:"max_num_objects"` + CreationTime LotValueMapInt `json:"creation_time"` + ExpirationTime LotValueMapInt `json:"expiration_time"` + DeletionTime LotValueMapInt `json:"deletion_time"` + } + + UsageMapFloat struct { + SelfContrib float64 `json:"self_contrib,omitempty"` + ChildrenContrib float64 `json:"children_contrib,omitempty"` + Total float64 `json:"total"` + } + + UsageMapInt struct { + SelfContrib Int64FromFloat `json:"self_contrib,omitempty"` + ChildrenContrib Int64FromFloat `json:"children_contrib,omitempty"` + Total Int64FromFloat `json:"total"` + } + + LotUsage struct { + GBBeingWritten UsageMapFloat `json:"GB_being_written,omitempty"` + ObjectsBeingWritten UsageMapInt `json:"objects_being_written,omitempty"` + DedicatedGB UsageMapFloat `json:"dedicated_GB,omitempty"` + OpportunisticGB UsageMapFloat `json:"opportunistic_GB,omitempty"` + NumObjects UsageMapInt `json:"num_objects,omitempty"` + TotalGB UsageMapFloat `json:"total_GB,omitempty"` + } + + Lot struct { + LotName string `json:"lot_name" mapstructure:"LotName"` + Owner string `json:"owner,omitempty" mapstructure:"Owner"` + // We don't expose Owners via map structure because that's not something we can configure. It's a derived value + Owners []string `json:"owners,omitempty"` + Parents []string `json:"parents" mapstructure:"Parents"` + // While we _could_ expose Children, that complicates things so for now we keep it hidden from the config + Children *[]string `json:"children,omitempty"` + Paths []LotPaths `json:"paths,omitempty" mapstructure:"Paths"` + MPA *MPA `json:"management_policy_attrs,omitempty" mapstructure:"ManagementPolicyAttrs"` + // Again, these are derived + RestrictiveMPA *RestrictiveMPA `json:"restrictive_management_policy_attrs,omitempty"` + Usage *LotUsage `json:"usage,omitempty"` + } + + ParentUpdate struct { + Current string `json:"current"` + New string `json:"new"` + } + + PathUpdate struct { + Current string `json:"current"` + New string `json:"new"` + Recursive bool `json:"recursive"` + } + + LotUpdate struct { + LotName string `json:"lot_name"` + Owner *string `json:"owner,omitempty"` + Parents *[]ParentUpdate `json:"parents,omitempty"` + Paths *[]PathUpdate `json:"paths,omitempty"` + MPA *MPA `json:"management_policy_attrs,omitempty"` + } +) + +// Lotman has a tendency to return an int as 123.0 instead of 123. This struct is used to unmarshal +// those values into an int64 +func (i *Int64FromFloat) UnmarshalJSON(b []byte) error { + var f float64 + if err := json.Unmarshal(b, &f); err != nil { + return err + } + i.Value = int64(f) + return nil +} + +func (i Int64FromFloat) MarshalJSON() ([]byte, error) { + return json.Marshal(i.Value) +} + +// Convert a cArray to a Go slice of strings. The cArray is a null-terminated +// array of null-terminated strings. +func cArrToGoArr(cArr *unsafe.Pointer) []string { + ptr := uintptr(*cArr) + var goArr []string + for { + // Read the uintptr at the current position. + strPtr := *(*uintptr)(unsafe.Pointer(ptr)) + + // Break if the uintptr is null. + if strPtr == 0 { + break + } + + // Create a Go string from the null-terminated string. + goStr := "" + for i := 0; ; i++ { + // Read the byte at the current position. + b := *(*byte)(unsafe.Pointer(strPtr + uintptr(i))) + + // Break if the byte is null. + if b == 0 { + break + } + + // Append the byte to the Go string. + goStr += string(b) + } + + // Append the Go string to the slice. + goArr = append(goArr, goStr) + + // Move to the next uintptr. + ptr += unsafe.Sizeof(uintptr(0)) + } + + return goArr +} + +// Trim any buffer we get back from LotMan to the first null char +func trimBuf(buf *[]byte) { + // Find the index of the first null character + nullIndex := bytes.IndexByte(*buf, 0) + + // Trim the slice after the first null character + if nullIndex != -1 { + *buf = (*buf)[:nullIndex] + } +} + +// Use the detected runtime to predict the location of the LotMan library. +func getLotmanLib() string { + fallbackPaths := []string{ + "/usr/lib64/libLotMan.so", + "/usr/local/lib64/libLotMan.so", + "/opt/local/lib64/libLotMan.so", + } + + switch runtime.GOOS { + case "linux": + configuredPath := param.Lotman_LibLocation.GetString() + if configuredPath != "" { + if _, err := os.Stat(configuredPath); err == nil { + return configuredPath + } + log.Errorln("libLotMan.so not found in configured path, attempting to find using known fallbacks") + } + + for _, path := range fallbackPaths { + if _, err := os.Stat(path); err == nil { + return path + } + } + panic("libLotMan.so not found in any of the known paths") + default: + panic(fmt.Errorf("GOOS=%s is not supported", runtime.GOOS)) + } +} + +func GetAuthorizedCallers(lotName string) (*[]string, error) { + // A caller is authorized if they own a parent of the lot. In the case of self-parenting lots, the owner is authorized. + errMsg := make([]byte, 2048) + cParents := unsafe.Pointer(nil) + + // Get immediate parents (including self to determine rootliness). We'll use them to determine owners + // who are allowed to manipulate, and thus delete, the lot + ret := LotmanGetLotParents(lotName, false, true, &cParents, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return nil, errors.Errorf("Failed to determine %s's parents: %s", lotName, string(errMsg)) + } + + parents := cArrToGoArr(&cParents) + + // Use a map to handle deduplication of owners list + ownersSet := make(map[string]struct{}) + for _, parent := range parents { + cOwners := unsafe.Pointer(nil) + LotmanGetLotOwners(parent, true, &cOwners, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return nil, errors.Errorf("Failed to determine appropriate owners of %s's parents: %s", lotName, string(errMsg)) + } + + for _, owner := range cArrToGoArr(&cOwners) { + ownersSet[owner] = struct{}{} + } + } + + // Convert the keys of the map to a slice + owners := make([]string, 0, len(ownersSet)) + for owner := range ownersSet { + owners = append(owners, owner) + } + + return &owners, nil +} + +// Under our model, we set owner to the issuer. Since this is owned by the federation, we set it in order of preference: +// 1. The federation's discovery url +// 2. The federation's director url +// TODO: Consider what happens to the lot if either of these values change in the future after the lot is created? +func getFederationIssuer() string { + federationIssuer := param.Federation_DiscoveryUrl.GetString() + if federationIssuer == "" { + federationIssuer = param.Federation_DirectorUrl.GetString() + } + + return federationIssuer +} + +// Initialize the LotMan library and bind its functions to the global vars +// We also perform a bit of extra setup such as setting the lotman db location +func InitLotman() bool { + log.Infof("Initializing LotMan...") + + // dlopen the LotMan library + lotmanLib, err := purego.Dlopen(getLotmanLib(), purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + log.Errorf("Error opening LotMan library: %v", err) + return false + } + + // Register LotMan funcs + purego.RegisterLibFunc(&LotmanVersion, lotmanLib, "lotman_version") + // C + purego.RegisterLibFunc(&LotmanAddLot, lotmanLib, "lotman_add_lot") + // R + purego.RegisterLibFunc(&LotmanGetLotJSON, lotmanLib, "lotman_get_lot_as_json") + // U + purego.RegisterLibFunc(&LotmanUpdateLot, lotmanLib, "lotman_update_lot") + // D + purego.RegisterLibFunc(&LotmanDeleteLotsRecursive, lotmanLib, "lotman_remove_lots_recursive") + + // Auxilliary functions + purego.RegisterLibFunc(&LotmanLotExists, lotmanLib, "lotman_lot_exists") + purego.RegisterLibFunc(&LotmanSetContextStr, lotmanLib, "lotman_set_context_str") + purego.RegisterLibFunc(&LotmanGetLotOwners, lotmanLib, "lotman_get_owners") + purego.RegisterLibFunc(&LotmanGetLotParents, lotmanLib, "lotman_get_parent_names") + purego.RegisterLibFunc(&LotmanGetLotsFromDir, lotmanLib, "lotman_get_lots_from_dir") + + // Set the lot_home context -- where the db lives + lotHome := param.Lotman_DbLocation.GetString() + + errMsg := make([]byte, 2048) + + log.Infof("Setting lot_home context to %s", lotHome) + ret := LotmanSetContextStr("lot_home", lotHome, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error setting lot_home context: %s", string(errMsg)) + return false + } + + defaultInitialized := false + rootInitialized := false + + err = param.Lotman_Lots.Unmarshal(&initializedLots) + if err != nil { + log.Warningf("Error while unmarshaling Lots from config: %v", err) + } + + federationIssuer := getFederationIssuer() + + callerMutex.Lock() + defer callerMutex.Unlock() + ret = LotmanSetContextStr("caller", federationIssuer, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error setting context for default lot: %s", string(errMsg)) + return false + } + + // Create the basic lots if they don't already exist. We'll make one for default + // and one for the root namespace + ret = LotmanLotExists("default", &errMsg) + if ret < 0 { + trimBuf(&errMsg) + log.Errorf("Error checking if default lot exists: %s", string(errMsg)) + return false + } else if ret == 0 { + // First we try to create the lots that might be configured via Pelican.yaml. If there are none, we'll use + // a few default values + for _, lot := range initializedLots { + if lot.LotName == "default" { + log.Debugf("Creating the default lot defined by %v", lot) + lotJSON, err := json.Marshal(lot) + if err != nil { + log.Errorf("Error marshalling default lot JSON: %v", err) + return false + } + + ret = LotmanAddLot(string(lotJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error creating default lot: %s", string(errMsg)) + return false + } + defaultInitialized = true + } + } + + if !defaultInitialized { + // Create the default lot + if federationIssuer == "" { + log.Errorf("your federation's issuer could not be deduced from your config's federation discovery URL or director URL") + return false + } + + initDedicatedGB := float64(0) + initOpportunisticGB := float64(0) + defaultLot := Lot{ + LotName: "default", + // Set the owner to the Federation's discovery url -- under this model, we can treat it like an issuer + Owner: federationIssuer, + // A self-parent lot indicates superuser status + Parents: []string{"default"}, + MPA: &MPA{ + DedicatedGB: &initDedicatedGB, + OpportunisticGB: &initOpportunisticGB, + MaxNumObjects: &Int64FromFloat{Value: 0}, + CreationTime: &Int64FromFloat{Value: 0}, + ExpirationTime: &Int64FromFloat{Value: 0}, + DeletionTime: &Int64FromFloat{Value: 0}, + }, + } + + log.Debugf("Creating the default lot defined by %v", defaultLot) + lotJSON, err := json.Marshal(defaultLot) + if err != nil { + log.Errorf("Error marshalling default lot JSON: %v", err) + return false + } + + ret = LotmanAddLot(string(lotJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error creating default lot: %s", string(errMsg)) + return false + } + } + + log.Infof("Created default lot") + } + + ret = LotmanLotExists("root", &errMsg) + if ret < 0 { + trimBuf(&errMsg) + log.Errorf("Error checking if root lot exists: %s", string(errMsg)) + return false + } else if ret == 0 { + // Try to create the root lot based on what we have in the config + for _, lot := range initializedLots { + if lot.LotName == "root" { + lotJSON, err := json.Marshal(lot) + if err != nil { + log.Errorf("Error marshalling root lot JSON: %v", err) + return false + } + + ret = LotmanAddLot(string(lotJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error creating root lot: %s", string(errMsg)) + return false + } + rootInitialized = true + } + } + + if !rootInitialized { + // Create the root lot based on predefined setup + if federationIssuer == "" { + log.Errorf("your federation's issuer could not be deduced from your config's federation discovery URL or director URL") + return false + } + + initDedicatedGB := float64(0) + initOpportunisticGB := float64(0) + rootLot := Lot{ + LotName: "root", + Owner: federationIssuer, + // A self-parent lot indicates superuser status + Parents: []string{"root"}, + Paths: []LotPaths{ + { + Path: "/", + Recursive: false, + }, + }, + MPA: &MPA{ + DedicatedGB: &initDedicatedGB, + OpportunisticGB: &initOpportunisticGB, + MaxNumObjects: &Int64FromFloat{Value: 0}, + CreationTime: &Int64FromFloat{Value: 0}, + ExpirationTime: &Int64FromFloat{Value: 0}, + DeletionTime: &Int64FromFloat{Value: 0}, + }, + } + + log.Debugf("Creating the root lot defined by %v", rootLot) + lotJSON, err := json.Marshal(rootLot) + if err != nil { + log.Errorf("Error marshalling root lot JSON: %v", err) + return false + } + + ret = LotmanAddLot(string(lotJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error creating root lot: %s", string(errMsg)) + return false + } + } + log.Infof("Created root lot") + } + + // Now instantiate any other lots that are in the config + for _, lot := range initializedLots { + if lot.LotName != "default" && lot.LotName != "root" { + lotJSON, err := json.Marshal(lot) + if err != nil { + log.Errorf("Error marshalling lot JSON for %s: %v", lot.LotName, err) + return false + } + + ret = LotmanAddLot(string(lotJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + log.Errorf("Error creating lot %s: %s", lot.LotName, string(errMsg)) + return false + } + } + } + + log.Infof("LotMan initialization complete") + return true +} + +// Create a lot in the lot database with the given lot struct. The caller is the entity that +// is creating the lot, and is used to determine whether we want to allow the creation to go through. +// Here, caller is used to determine whether this lot is allowed to create any sublots of an indicated +// parent lot. The lot struct has the form: +// +// { +// "lot_name": "lot_name", (REQUIRED) +// "owner": "owner", (REQUIRED) +// "parents": ["parent1", "parent2"], (REQUIRED) +// "paths": [ +// { +// "path": "path", +// "recursive": true/false +// } +// ], (OPTIONAL) +// "management_policy_attrs": { +// "dedicated_GB": 0.0, +// "opportunistic_GB": 0.0, +// "max_num_objects": 0, +// "creation_time": 0, +// "expiration_time": 0, +// "deletion_time": 0 +// } (REQUIRED) +// } +func CreateLot(newLot *Lot, caller string) error { + // Marshal the JSON into a string for the C function + lotJSON, err := json.Marshal(*newLot) + if err != nil { + return errors.Wrapf(err, "Error marshalling lot JSON: %v", err) + } + + // Set the context to the incoming lot's owner: + errMsg := make([]byte, 2048) + callerMutex.Lock() + defer callerMutex.Unlock() + ret := LotmanSetContextStr("caller", caller, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return fmt.Errorf(fmt.Sprintf("Error creating lot: %s", string(errMsg))) + } + + // Now finally add the lot + ret = LotmanAddLot(string(lotJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return fmt.Errorf(fmt.Sprintf("Error creating lot: %s", string(errMsg))) + } + + return nil +} + +// Given a lot name, get the lot from the lot database. If recursive is true, we'll also +// determine all hierarchical restrictions on the lot. For example, if the lot "foo" has +// dedicated_GB = 2.0 but its parent lot "bar" has dedicated_GB = 1.0, then calling this +// with recusrive = true will indicate the restricting value and the lot it comes from. +func GetLot(lotName string, recursive bool) (*Lot, error) { + // Haven't given much thought to these buff sizes yet + outputBuf := make([]byte, 4096) + errMsg := make([]byte, 2048) + + ret := LotmanGetLotJSON(lotName, recursive, &outputBuf, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return nil, errors.Errorf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&outputBuf) + var lot Lot + err := json.Unmarshal(outputBuf, &lot) + if err != nil { + return nil, errors.Wrapf(err, "Error unmarshalling lot JSON: %v", err) + } + return &lot, nil +} + +// Update a lot in the lot database with the given lotUpdate struct. The caller is the entity that +// is updating the lot, and is used to determine whether we want to allow the update to go through. +// In general, a valid caller is one that matches the owner of any of the lot's parents. +// The lot update struct has the form: +// +// { +// "lot_name": "lot_name", (REQUIRED) +// "owner": "new_owner", (OPTIONAL) +// "parents": [ +// { +// "current": "current_parent", +// "new": "new_parent" +// } +// ], (OPTIONAL) +// "paths": [ +// { +// "current": "current_path", +// "new": "new_path", +// "recursive": true/false +// } +// ], (OPTIONAL) +// "management_policy_attrs": { +// "dedicated_GB": 0.0, +// "opportunistic_GB": 0.0, +// "max_num_objects": 0, +// "creation_time": 0, +// "expiration_time": 0, +// "deletion_time": 0 +// } (OPTIONAL) +// } +func UpdateLot(lotUpdate *LotUpdate, caller string) error { + // Marshal the JSON into a string for the C function + updateJSON, err := json.Marshal(*lotUpdate) + if err != nil { + return errors.Wrapf(err, "Error marshalling lot JSON: %v", err) + } + + errMsg := make([]byte, 2048) + callerMutex.Lock() + defer callerMutex.Unlock() + ret := LotmanSetContextStr("caller", caller, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return fmt.Errorf(fmt.Sprintf("Error setting caller for lot update: %s", string(errMsg))) + } + + ret = LotmanUpdateLot(string(updateJSON), &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return fmt.Errorf(fmt.Sprintf("Error updating lot: %s", string(errMsg))) + } + + return nil +} + +// Delete a lot from the lot database. The caller is the entity that is deleting the lot, and is used to determine +// whether we want to allow the deletion to go through. In general, a valid caller is one that matches an owner from +// any of the lot's recursive parents. This function deletes the lot and all of its children. +func DeleteLotsRecursive(lotName string, caller string) error { + errMsg := make([]byte, 2048) + callerMutex.Lock() + defer callerMutex.Unlock() + ret := LotmanSetContextStr("caller", caller, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return fmt.Errorf(fmt.Sprintf("Error creating lot: %s", string(errMsg))) + } + + // We've set the caller, now try to delete the lots + ret = LotmanDeleteLotsRecursive(lotName, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return fmt.Errorf(fmt.Sprintf("Error deleting lots: %s", string(errMsg))) + } + + return nil +} diff --git a/lotman/lotman_test.go b/lotman/lotman_test.go new file mode 100644 index 000000000..da4e18866 --- /dev/null +++ b/lotman/lotman_test.go @@ -0,0 +1,317 @@ +//go:build linux && !ppc64le + +/*************************************************************** +* +* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research +* +* Licensed under the Apache License, Version 2.0 (the "License"); you +* may not use this file except in compliance with the License. You may +* obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +***************************************************************/ + +package lotman + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +//go:embed resources/lots-config.yaml + +var yamlMockup string + +func setupLotmanFromConf(t *testing.T, readConfig bool, name string) (bool, func()) { + // Load in our config + if readConfig { + viper.Set("Federation.DiscoveryUrl", "https://fake-federation.com") + viper.SetConfigType("yaml") + err := viper.ReadConfig(strings.NewReader(yamlMockup)) + if err != nil { + t.Fatalf("Error reading config: %v", err) + } + } + + tmpPathPattern := name + "*" + tmpPath, err := os.MkdirTemp("", tmpPathPattern) + require.NoError(t, err) + + viper.Set("Lotman.DbLocation", tmpPath) + success := InitLotman() + //reset func + return success, func() { + viper.Reset() + } +} + +// Test the library initializer. NOTE: this also tests CreateLot, which is a part of initialization. +func TestLotmanInit(t *testing.T) { + viper.Reset() + + t.Run("TestBadInit", func(t *testing.T) { + // We haven't set various bits needed to create the lots, like discovery URL + success, cleanup := setupLotmanFromConf(t, false, "LotmanBadInit") + defer cleanup() + require.False(t, success) + }) + + t.Run("TestGoodInit", func(t *testing.T) { + viper.Set("Log.Level", "debug") + viper.Set("Federation.DiscoveryUrl", "https://fake-federation.com") + success, cleanup := setupLotmanFromConf(t, false, "LotmanGoodInit") + defer cleanup() + require.True(t, success) + + // Now that we've initialized (without config) test that we have default/root + defaultOutput := make([]byte, 4096) + errMsg := make([]byte, 2048) + + ret := LotmanGetLotJSON("default", false, &defaultOutput, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + t.Fatalf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&defaultOutput) + var defaultLot Lot + err := json.Unmarshal(defaultOutput, &defaultLot) + require.NoError(t, err, fmt.Sprintf("Error unmarshalling default lot JSON: %s", string(defaultOutput))) + require.Equal(t, "default", defaultLot.LotName) + require.Equal(t, "https://fake-federation.com", defaultLot.Owner) + require.Equal(t, "default", defaultLot.Parents[0]) + require.Equal(t, 0.0, *(defaultLot.MPA.DedicatedGB)) + require.Equal(t, int64(0), (defaultLot.MPA.MaxNumObjects.Value)) + + rootOutput := make([]byte, 4096) + ret = LotmanGetLotJSON("root", false, &rootOutput, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + t.Fatalf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&rootOutput) + var rootLot Lot + err = json.Unmarshal(rootOutput, &rootLot) + require.NoError(t, err, fmt.Sprintf("Error unmarshalling root lot JSON: %s", string(rootOutput))) + require.Equal(t, "root", rootLot.LotName) + require.Equal(t, "https://fake-federation.com", rootLot.Owner) + require.Equal(t, "root", rootLot.Parents[0]) + require.Equal(t, 0.0, *(rootLot.MPA.DedicatedGB)) + require.Equal(t, int64(0), (rootLot.MPA.MaxNumObjects.Value)) + }) +} + +func TestLotmanInitFromConfig(t *testing.T) { + viper.Reset() + + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + defer cleanup() + require.True(t, success) + + // Lotman is initialized, let's check that it has the information it should based on the config + defaultOutput := make([]byte, 4096) + errMsg := make([]byte, 2048) + + // Check for default lot + ret := LotmanGetLotJSON("default", false, &defaultOutput, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + t.Fatalf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&defaultOutput) + var defaultLot Lot + err := json.Unmarshal(defaultOutput, &defaultLot) + require.NoError(t, err, fmt.Sprintf("Error unmarshalling default lot JSON: %s", string(defaultOutput))) + require.Equal(t, "default", defaultLot.LotName) + require.Equal(t, "https://fake-federation.com", defaultLot.Owner) + require.Equal(t, "default", defaultLot.Parents[0]) + require.Equal(t, 100.0, *(defaultLot.MPA.DedicatedGB)) + require.Equal(t, int64(1000), (defaultLot.MPA.MaxNumObjects.Value)) + + // Now root + rootOutput := make([]byte, 4096) + ret = LotmanGetLotJSON("root", false, &rootOutput, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + t.Fatalf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&rootOutput) + var rootLot Lot + err = json.Unmarshal(rootOutput, &rootLot) + require.NoError(t, err, fmt.Sprintf("Error unmarshalling root lot JSON: %s", string(rootOutput))) + require.Equal(t, "root", rootLot.LotName) + require.Equal(t, "https://fake-federation.com", rootLot.Owner) + require.Equal(t, "root", rootLot.Parents[0]) + require.Equal(t, 1.0, *(rootLot.MPA.DedicatedGB)) + require.Equal(t, int64(10), rootLot.MPA.MaxNumObjects.Value) + require.Equal(t, "/", rootLot.Paths[0].Path) + require.False(t, rootLot.Paths[0].Recursive) + + // Now test-1 + test1Output := make([]byte, 4096) + ret = LotmanGetLotJSON("test-1", false, &test1Output, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + t.Fatalf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&test1Output) + var test1Lot Lot + err = json.Unmarshal(test1Output, &test1Lot) + require.NoError(t, err, fmt.Sprintf("Error unmarshalling test-1 lot JSON: %s", string(test1Output))) + require.Equal(t, "test-1", test1Lot.LotName) + require.Equal(t, "https://different-fake-federation.com", test1Lot.Owner) + require.Equal(t, "root", test1Lot.Parents[0]) + require.Equal(t, 1.11, *(test1Lot.MPA.DedicatedGB)) + require.Equal(t, int64(42), test1Lot.MPA.MaxNumObjects.Value) + require.Equal(t, "/test-1", test1Lot.Paths[0].Path) + require.False(t, test1Lot.Paths[0].Recursive) + + // Finally test-2 + test2Output := make([]byte, 4096) + ret = LotmanGetLotJSON("test-2", false, &test2Output, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + t.Fatalf("Error getting lot JSON: %s", string(errMsg)) + } + trimBuf(&test2Output) + var test2Lot Lot + err = json.Unmarshal(test2Output, &test2Lot) + require.NoError(t, err, fmt.Sprintf("Error unmarshalling test-2 lot JSON: %s", string(test2Output))) + require.Equal(t, "test-2", test2Lot.LotName) + require.Equal(t, "https://another-fake-federation.com", test2Lot.Owner) + require.Equal(t, "test-1", test2Lot.Parents[0]) + require.Equal(t, 1.11, *(test2Lot.MPA.DedicatedGB)) + require.Equal(t, int64(42), test2Lot.MPA.MaxNumObjects.Value) + require.Equal(t, "/test-1/test-2", test2Lot.Paths[0].Path) + require.True(t, test2Lot.Paths[0].Recursive) +} + +func TestGetLotmanLib(t *testing.T) { + libLoc := getLotmanLib() + require.Equal(t, "/usr/lib64/libLotMan.so", libLoc) + + // Now try to fool it and see that we get the same value back. We can detect this by + // capturing the log output + logOutput := &(bytes.Buffer{}) + log.SetOutput(logOutput) + log.SetLevel(log.DebugLevel) + viper.Set("Lotman.LibLocation", "/not/a/pathlibLotMan.so") + libLoc = getLotmanLib() + require.Equal(t, "/usr/lib64/libLotMan.so", libLoc) + require.Contains(t, logOutput.String(), "libLotMan.so not found in configured path, attempting to find using known fallbacks") +} + +func TestGetAuthzCallers(t *testing.T) { + viper.Reset() + success, cleanup := setupLotmanFromConf(t, true, "LotmanGetAuthzCalleres") + defer cleanup() + require.True(t, success) + + // Lotman is initialized, let's check that it has the information it should based on the config + // test-2's authzed callers are the owners of root and test-1 + authzedCallers, err := GetAuthorizedCallers("test-2") + require.NoError(t, err, "Failed to get authorized callers") + require.Equal(t, 2, len(*authzedCallers)) + require.Contains(t, *authzedCallers, "https://fake-federation.com") + require.Contains(t, *authzedCallers, "https://different-fake-federation.com") + + // test with a non-existent lot + _, err = GetAuthorizedCallers("non-existent-lot") + require.Error(t, err, "Expected error for non-existent lot") +} + +func TestGetLot(t *testing.T) { + viper.Reset() + success, cleanup := setupLotmanFromConf(t, true, "LotmanGetLot") + defer cleanup() + require.True(t, success) + + lot, err := GetLot("test-2", true) + require.NoError(t, err, "Failed to get lot") + require.NotNil(t, lot) + require.Equal(t, "test-2", (lot).LotName) + require.Equal(t, 2, len(lot.Parents)) + require.Contains(t, lot.Parents, "root") + require.Contains(t, lot.Parents, "test-1") + require.Equal(t, 3, len(lot.Owners)) + require.Contains(t, lot.Owners, "https://fake-federation.com") + require.Contains(t, lot.Owners, "https://different-fake-federation.com") + require.Contains(t, lot.Owners, "https://another-fake-federation.com") + require.Equal(t, 1.11, *(lot.MPA.DedicatedGB)) + require.Equal(t, int64(42), lot.MPA.MaxNumObjects.Value) + require.Equal(t, "/test-1/test-2", lot.Paths[0].Path) + require.True(t, lot.Paths[0].Recursive) +} + +func TestUpdateLot(t *testing.T) { + viper.Reset() + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + defer cleanup() + require.True(t, success) + + // Update the test-1 lot + dedGB := float64(999.0) + lotUpdate := LotUpdate{ + LotName: "test-1", + MPA: &MPA{ + DedicatedGB: &dedGB, + MaxNumObjects: &Int64FromFloat{ + Value: 84, + }, + }, + Paths: &[]PathUpdate{ + { + Current: "/test-1", + New: "/test-1-updated", + Recursive: false, + }, + }, + } + + err := UpdateLot(&lotUpdate, "https://fake-federation.com") + require.NoError(t, err, "Failed to update lot") + + // Now check that the update was successful + lot, err := GetLot("test-1", true) + require.NoError(t, err, "Failed to get lot") + require.Equal(t, "test-1", lot.LotName) + require.Equal(t, dedGB, *(lot.MPA.DedicatedGB)) + require.Equal(t, int64(84), lot.MPA.MaxNumObjects.Value) + require.Equal(t, "/test-1-updated", lot.Paths[0].Path) + require.False(t, lot.Paths[0].Recursive) +} + +func TestDeleteLotsRec(t *testing.T) { + viper.Reset() + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + defer cleanup() + require.True(t, success) + + // Delete test-1, then verify both it and test-2 are gone + err := DeleteLotsRecursive("test-1", "https://fake-federation.com") + require.NoError(t, err, "Failed to delete lot") + + // Now check that the delete was successful + lot, err := GetLot("test-1", false) + require.Error(t, err, "Expected error for non-existent lot") + require.Nil(t, lot) + + lot, err = GetLot("test-2", false) + require.Error(t, err, "Expected error for non-existent lot") + require.Nil(t, lot) +} diff --git a/lotman/lotman_ui.go b/lotman/lotman_ui.go new file mode 100644 index 000000000..c30a2ecde --- /dev/null +++ b/lotman/lotman_ui.go @@ -0,0 +1,539 @@ +//go:build linux && !ppc64le + +/*************************************************************** +* +* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research +* +* Licensed under the Apache License, Version 2.0 (the "License"); you +* may not use this file except in compliance with the License. You may +* obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +***************************************************************/ + +package lotman + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "unsafe" + + "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" + "github.com/pelicanplatform/pelican/server_utils" + "github.com/pelicanplatform/pelican/token" + "github.com/pelicanplatform/pelican/token_scopes" + "github.com/pelicanplatform/pelican/utils" +) + +type LotAction string + +var ( + LotUpdateAction LotAction = "modify" + LotDeleteAction LotAction = "delete" +) + +// Given a token and a list of authorized callers, check that the token is signed by one of the authorized callers. Return +// a pointer to the parsed token. +func tokenSignedByAuthorizedCaller(strToken string, authorizedCallers *[]string) (bool, *jwt.Token, error) { + ownerFound := false + var tok jwt.Token + for _, owner := range *authorizedCallers { + kSet, err := server_utils.GetJWKSFromIssUrl(owner) + if err != nil { + log.Debugf("Error getting JWKS for owner %s: %v", owner, err) + continue + } + tok, err = jwt.Parse([]byte(strToken), jwt.WithKeySet(*kSet), jwt.WithValidate(true)) + if err != nil { + log.Debugf("Token verification failed with owner %s: %v -- skipping", owner, err) + continue + } + ownerFound = true + break + } + + if !ownerFound { + return false, nil, errors.New("token not signed by any of the owners of any parent lot") + } + + return true, &tok, nil +} + +// Verify that a token received is a valid token. Upon verification, we set the lot's parents/owner to the +// appropriate values. Returns true if the token is valid, false otherwise. +func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { + tokenApproved := false + + // Get the path associated with the lot (right now we assume/enforce a single path) and try + // to deduce a namespace prefix from that. We'll use that namespace prefix's issuer as the + // lot's owner field (which is the data owner). If there is no associated isuer, we assign + // ownership to the federation + + path := ((lot.Paths)[0]).Path + log.Debugf("Attempting to add lot for path: %s", path) + errMsg := make([]byte, 2048) + lots := unsafe.Pointer(nil) + // Pass a pointer to the first element of the slice to the C++ function. + ret := LotmanGetLotsFromDir(path, false, &lots, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return false, errors.Errorf("Error getting lot JSON: %s", string(errMsg)) + } + + goLots := cArrToGoArr(&lots) + + // Check if goOut[0] is "default". Lotman should ALWAYS put something in this array. If it's empty, we have a problem. + if goLots[0] == "default" { + // We'll assign to the root lot. However, be careful here because we make an assumption -- under the hood, + // Lotman will return "default" if there's no other lot, or if the actual path is in some way assigned to + // the default lot. When we assign to parent to root here, we're assuming the former. This implies we should stay + // away from assigning paths to the default lot. + lot.Parents = []string{"root"} + } else { + // We found a different logical parent + lot.Parents = goLots + } + + var tok jwt.Token + if len(lot.Parents) != 0 && lot.Parents[0] == "root" { + // We check that the token is signed by the federation + // First check for discovery URL and then for director URL, both of which should host the federation's pubkey + issuerUrl := getFederationIssuer() + + kSet, err := server_utils.GetJWKSFromIssUrl(issuerUrl) + if err != nil { + return false, errors.Wrap(err, "Error getting JWKS from issuer URL") + } + + tok, err = jwt.Parse([]byte(strToken), jwt.WithKeySet(*kSet), jwt.WithValidate(true)) + if err != nil { + return false, errors.Wrap(err, "Error parsing token") + } + } else { + + // Use a map to handle deduplication of owners list + ownersSet := make(map[string]struct{}) + for _, parent := range lot.Parents { + cOwners := unsafe.Pointer(nil) + LotmanGetLotOwners(parent, true, &cOwners, &errMsg) + if ret != 0 { + trimBuf(&errMsg) + return false, errors.Errorf("Error getting lot JSON: %s", string(errMsg)) + } + + for _, owner := range cArrToGoArr(&cOwners) { + ownersSet[owner] = struct{}{} + } + } + + ownerFound := false + for owner := range ownersSet { + kSet, err := server_utils.GetJWKSFromIssUrl(owner) + + // Print the kSet as a string for debugging + kSetStr, _ := json.Marshal(kSet) + log.Debugf("JWKS for owner %s: %s", owner, string(kSetStr)) + if err != nil { + log.Debugf("Error getting JWKS for owner %s: %v", owner, err) + continue + } + tok, err = jwt.Parse([]byte(strToken), jwt.WithKeySet(*kSet), jwt.WithValidate(true)) + if err != nil { + log.Debugf("Token verification failed with owner %s: %v -- skipping", owner, err) + continue + } + ownerFound = true + break + } + + if !ownerFound { + return false, errors.New("token not signed by any of the owners of any parent lot") + } + } + + // We've determined the token is signed by someone we like, now to check that it has the correct lot.create permission! + scope_any, present := tok.Get("scope") + if !present { + return false, errors.New("No scope claim in token") + } + scope, ok := scope_any.(string) + if !ok { + return false, errors.New("scope claim in token is not string-valued") + } + scopes := strings.Split(scope, " ") + for _, scope := range scopes { + if scope == token_scopes.Lot_Create.String() { + tokenApproved = true + break + } + } + + if !tokenApproved { + return false, errors.New("The token was correctly signed but did not possess the necessary lot.create scope") + } + + // At this point, we have a good token, now we need to get the appropriate owner for the lot. + // To do this, we get a namespace from the path and then get the issuer for that namespace. If no + // namespace exists, we'll assign ownership to the federation. + + // TODO: Once we have the new Director endpoint that returns a namespace for a given path, we'll use that + // and cut out a lot of this cruft + + // Get the namespace by querying the director and checking the headers + errMsgPrefix := "the provided token is acceptible, but no owner could be determined because " + directorUrlStr := param.Federation_DirectorUrl.GetString() + if directorUrlStr == "" { + return false, errors.New(errMsgPrefix + "the federation director URL is not set") + } + directorUrl, err := url.Parse(directorUrlStr) + if err != nil { + return false, errors.Wrap(err, errMsgPrefix+"the federation director URL is not a valid URL") + } + + directorUrl.Path, err = url.JoinPath("/api/v1.0/director/object", path) + if err != nil { + return false, errors.Wrap(err, errMsgPrefix+"the director URL could not be joined with the path") + } + + // Get the namespace by querying the director and checking the headers. The client should NOT + // follow the redirect + httpClient := &http.Client{ + Transport: config.GetTransport(), + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + req, err := http.NewRequest("GET", directorUrl.String(), nil) + if err != nil { + return false, errors.Wrap(err, errMsgPrefix+"the director request could not be created") + } + resp, err := httpClient.Do(req) + if err != nil { + return false, errors.Wrapf(err, errMsgPrefix+"the director couldn't be queried for path %s", path) + } + + // Check the response code, make sure it's not in the error ranges (400-500) + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return false, errors.Errorf(errMsgPrefix+"the director returned a bad status for path %s: %s", path, resp.Status) + } + + // Get the namespace from the X-Pelican-Namespace header + namespaceHeader := resp.Header.Values("X-Pelican-Namespace") + xPelicanNamespaceMap := utils.HeaderParser(namespaceHeader[0]) + namespace := xPelicanNamespaceMap["namespace"] + + // Get the issuer URL for that namespace + nsIssuerUrl, err := server_utils.GetNSIssuerURL(namespace) + if err != nil { + return false, errors.Wrapf(err, errMsgPrefix+"no issuer could be found for namespace %s", namespace) + } + lot.Owner = nsIssuerUrl + + return true, nil +} + +// Actions that modify lots themselves (be it delete or modify) require the same authorization. This function +// verifies the token against the requested action. +func VerifyLotModTokens(lotName string, strToken string, action LotAction) (bool, error) { + // Get all parents of the lot, which we use to determine owners who can modify the lot itself (as opposed + // to the data in the lot). + tokenApproved := false + + log.Debugf("Attempting to %s lot %s", action, lotName) + authzCallers, err := GetAuthorizedCallers(lotName) + if err != nil { + return false, errors.Wrap(err, "Error getting authorized callers") + } + + ownerSigned, tok, err := tokenSignedByAuthorizedCaller(strToken, authzCallers) + if err != nil { + return false, errors.Wrap(err, "Error verifying token is appropriately signed") + } + if !ownerSigned { + return false, errors.New("Token not signed by any of the owners of any parent lot") + } + + // We've determined the token is signed by someone we like, now to check that it has the correct scope! + scope_any, present := (*tok).Get("scope") + if !present { + return false, errors.New("no scope claim in token") + } + scope, ok := scope_any.(string) + if !ok { + return false, errors.New("scope claim in token is not string-valued") + } + scopes := strings.Split(scope, " ") + for _, scope := range scopes { + switch action { + case LotUpdateAction: + if scope == token_scopes.Lot_Modify.String() { + tokenApproved = true + break + } + case LotDeleteAction: + if scope == token_scopes.Lot_Delete.String() { + tokenApproved = true + break + } + default: + return false, errors.New(fmt.Sprintf("invalid lot action: %s", action)) + } + } + + if !tokenApproved { + return false, errors.New(fmt.Sprintf("The token was correctly signed, but does not have permission to %s the lot", action)) + } + + return true, nil + +} + +// The function Gin routes to when the CreateLot endpoint is hit. +func uiCreateLot(ctx *gin.Context) { + // Unmarshal the lot JSON from the incoming context + extraInfo := "" + var lot Lot + err := ctx.BindJSON(&lot) + if err != nil { + log.Errorf("Error binding lot JSON: %v", err) + ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error binding incoming lot JSON: %v", err)}) + return + } + + // Right now we assume a single path. The authorization scheme gets complicated quickly otherwise. + if len(lot.Paths) > 1 { + log.Errorf("error creating lot: Lot contains more than one path, which is not yet supported by Pelican") + ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Error creating lot: Lot contains more than one path, which is not yet supported by Pelican"}) + } + + if lot.Owner != "" { + extraInfo += "NOTE: New lot owner fields are ignored. The owner will be set to the issuer of the token used to create the lot. " + } + if len(lot.Parents) > 0 { + extraInfo += "NOTE: New lot parent fields are ignored. The parent will be determined by the namespace heirarchy associated with the lot's path. " + } + + // TODO: Figure out the best way to inform the user that we ignore any owner or parent they set, because we handle that internally. + + token := token.GetAuthzEscaped(ctx) + if token == "" { + log.Debugln("No token provided in request") + ctx.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "No token provided in request"}) + return + } + + // TODO: Since VerifyNewLotToken has a few side effects (like modifying the underlying lot obj), it might also + // need to handle a case where the lot for /foo/bar/baz is created before /foo/bar. Currently, if these are the + // first two lots we create with this endpoint, then both will be set to have "root" as a parent, and we'd like + // /foo/bar/baz to be modified to have /foo/bar as a parent. This gets complicated, so let's punt on it for now. + ok, err := VerifyNewLotToken(&lot, token) + + // TODO: Distinguish between true errors and unauthorized errors + if err != nil { + log.Debugln("Error verifying token: ", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error verifying token: %v", err)}) + return + } + if !ok { + log.Debugln("Token verification failed") + ctx.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Token does not appear to have necessary authorization to create a lot"}) + return + } + + // For creating lots, the Lotman caller must be set to an owner of a parent. Since the incoming token + // was presumably signed by someone with the necessary permissions, we can use the token's issuer as the + // Lotman caller. + tok, err := jwt.Parse([]byte(token), jwt.WithVerify(false), jwt.WithValidate(false)) + if err != nil { + log.Debugf("Failed to parse token while determining Lotman Caller: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Failed to parse token while determining Lotman Caller"}) + } + caller := tok.Issuer() + + err = CreateLot(&lot, caller) + if err != nil { + log.Errorf("Error creating lot: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error creating lot: %v", err)}) + return + } + + ctx.JSON(http.StatusOK, gin.H{"status": "success" + extraInfo}) +} + +// The function Gin routes to when the GetLotJSON endpoint is hit. +func uiGetLotJSON(ctx *gin.Context) { + lotName := ctx.Query("lotName") + recursiveStr := ctx.Query("recursive") + var recursive bool + var err error + if recursiveStr != "" { + recursive, err = strconv.ParseBool(recursiveStr) + if err != nil { + log.Errorf("Error parsing recursive query param: %v", err) + ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error parsing recursive query param: %v", err)}) + return + } + } + + lot, err := GetLot(lotName, recursive) + if err != nil { + log.Errorf("Error fetching lot: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error fetching lot: %v", err)}) + return + } + + ctx.JSON(http.StatusOK, lot) +} + +// NOTE: For now we'll allow updates to parents, paths and MPAs. We'll ignore owner fields, since Pelican figures out who +// owns things internally +func uiUpdateLot(ctx *gin.Context) { + // Unmarshal the lot JSON from the incoming context + extraInfo := " " + var lotUpdate LotUpdate + err := ctx.BindJSON(&lotUpdate) + if err != nil { + log.Errorf("Error binding lot JSON: %v", err) + ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error binding incoming lot JSON: %v", err)}) + return + } + + // Right now we assume a single path. The authorization scheme gets complicated quickly otherwise. + if len(*lotUpdate.Paths) > 1 { + log.Errorf("error updating lot: The update contains more than one path, which is not yet supported by Pelican") + ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Error updating lot: The update contains more than one path, which is not yet supported by Pelican"}) + } + + if lotUpdate.Owner != nil { + extraInfo += "NOTE: Lot owner fields are ignored. The owner will be set to the issuer of the token used to create the lot. " + lotUpdate.Owner = nil + } + + // TODO: I thought about this a bit, and wasn't sure how to handle the case where we have a path update. One of two scenarios + // might be true. + // 1: The new path is already tied to another lot. If this is the case, then we shouldn't be trying to add that path to this lot + // because LotMan assumes a path can be held by only one lot (hierarchical ownership not withstanding). + // 2: The new path is not tied to another lot. In this case, what we should really do is create a new lot for that namespace, assuming + // one exists. But then why not create a new lot? + // Thus, until we've sorted out multi-path lots, we should probably just error out in both of these cases. An adminstrator at the cache + // can get access to the system and update things however they choose anyway (albeit that means writing C to do it). + if lotUpdate.Paths != nil { + extraInfo += "NOTE: Lot update path fields are ignored. Pelican does not yet support this feature. " + lotUpdate.Paths = nil + } + + token := token.GetAuthzEscaped(ctx) + if token == "" { + log.Debugln("No token provided in request") + ctx.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "No token provided in request"}) + return + } + + ok, err := VerifyLotModTokens(lotUpdate.LotName, token, LotUpdateAction) + + // TODO: Distinguish between true errors and unauthorized errors + if err != nil { + log.Debugln("Error verifying token: ", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error verifying token: %v", err)}) + return + } + if !ok { + log.Debugln("Token verification failed") + ctx.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Token does not appear to have necessary authorization to delete a lot"}) + return + } + + // For updating lots, the Lotman caller must be set to an owner of a parent. Since the incoming token + // was presumably signed by someone with the necessary permissions, we can use the token's issuer as the + // Lotman caller. + tok, err := jwt.Parse([]byte(token), jwt.WithVerify(false), jwt.WithValidate(false)) + if err != nil { + log.Debugf("Failed to parse token while determining Lotman Caller: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Failed to parse token while determining Lotman Caller"}) + } + + caller := tok.Issuer() + err = UpdateLot(&lotUpdate, caller) + if err != nil { + log.Errorf("Error updating lot: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error updating lot: %v", err)}) + return + } + + ctx.JSON(http.StatusOK, gin.H{"status": "success" + extraInfo}) +} + +// The function Gin routes to when the DeleteLotsRecursive endpoint is hit. +func uiDeleteLot(ctx *gin.Context) { + lotName := ctx.Query("lotName") + if lotName == "" { + ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "No lot name provided in URL query param 'lotName'"}) + return + } + + token := token.GetAuthzEscaped(ctx) + if token == "" { + log.Debugln("No token provided in request") + ctx.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "No token provided in request"}) + return + } + + ok, err := VerifyLotModTokens(lotName, token, LotDeleteAction) + + // TODO: Distinguish between true errors and unauthorized errors + if err != nil { + log.Debugln("Error verifying token: ", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error verifying token: %v", err)}) + return + } + if !ok { + log.Debugln("Token verification failed") + ctx.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Token does not appear to have necessary authorization to delete the lot"}) + return + } + + // For creating lots, the Lotman caller must be set to an owner of a parent. Since the incoming token + // was presumably signed by someone with the necessary permissions, we can use the token's issuer as the + // Lotman caller. + tok, err := jwt.Parse([]byte(token), jwt.WithVerify(false), jwt.WithValidate(false)) + if err != nil { + log.Debugf("Failed to parse token while determining Lotman Caller: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Failed to parse token while determining Lotman Caller"}) + } + caller := tok.Issuer() + + err = DeleteLotsRecursive(lotName, caller) + if err != nil { + log.Errorf("Error deleting lot: %v", err) + ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error deleting lot: %v", err)}) + return + } + + ctx.JSON(http.StatusOK, gin.H{"status": "success"}) +} + +func RegisterLotman(ctx context.Context, router *gin.RouterGroup) { + router.GET("/api/v1.0/lotman/getLotJSON", uiGetLotJSON) + router.PUT("/api/v1.0/lotman/createLot", uiCreateLot) + router.DELETE("/api/v1.0/lotman/deleteLotsRecursive", uiDeleteLot) + router.PUT("/api/v1.0/lotman/updateLot", uiUpdateLot) +} diff --git a/lotman/resources/lots-config.yaml b/lotman/resources/lots-config.yaml new file mode 100644 index 000000000..874b0b7a3 --- /dev/null +++ b/lotman/resources/lots-config.yaml @@ -0,0 +1,100 @@ +# *************************************************************** +# +# Copyright (C) 2024, Pelican Project, Morgridge Institute for Research +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# *************************************************************** + +# Configuration options used to test Lot declarations +Lotman: + Lots: + - LotName: "default" + Owner: "https://fake-federation.com" + Parents: + - "default" + ManagementPolicyAttrs: + DedicatedGB: 100 + OpportunisticGB: 200 + # Wrapping these in a map is an unfortunate side effect of the + # way we need to handle the float-->int conversion. + MaxNumObjects: + Value: 1000 + CreationTime: + Value: 1234 + ExpirationTime: + Value: 12345 + DeletionTime: + Value: 123456 + + - LotName: "root" + Owner: "https://fake-federation.com" + Parents: + - "root" + Paths: + - Path: "/" + Recursive: false + ManagementPolicyAttrs: + DedicatedGB: 1 + OpportunisticGB: 2 + # Wrapping these in a map is an unfortunate side effect of the + # way we need to handle the float-->int conversion. + MaxNumObjects: + Value: 10 + CreationTime: + Value: 1234 + ExpirationTime: + Value: 12345 + DeletionTime: + Value: 123456 + + - LotName: "test-1" + Owner: "https://different-fake-federation.com" + Parents: + - "root" + Paths: + - Path: "/test-1" + Recursive: false + ManagementPolicyAttrs: + DedicatedGB: 1.11 + OpportunisticGB: 2.22 + # Wrapping these in a map is an unfortunate side effect of the + # way we need to handle the float-->int conversion. + MaxNumObjects: + Value: 42 + CreationTime: + Value: 1234 + ExpirationTime: + Value: 12345 + DeletionTime: + Value: 123456 + - LotName: "test-2" + Owner: "https://another-fake-federation.com" + Parents: + - "test-1" + Paths: + - Path: "/test-1/test-2" + Recursive: true + ManagementPolicyAttrs: + DedicatedGB: 1.11 + OpportunisticGB: 2.22 + # Wrapping these in a map is an unfortunate side effect of the + # way we need to handle the float-->int conversion. + MaxNumObjects: + Value: 42 + CreationTime: + Value: 1234 + ExpirationTime: + Value: 12345 + DeletionTime: + Value: 123456 diff --git a/param/parameters.go b/param/parameters.go index e0e858b2b..6d636e153 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -150,6 +150,8 @@ var ( Logging_Origin_Scitokens = StringParam{"Logging.Origin.Scitokens"} Logging_Origin_Xrd = StringParam{"Logging.Origin.Xrd"} Logging_Origin_Xrootd = StringParam{"Logging.Origin.Xrootd"} + Lotman_DbLocation = StringParam{"Lotman.DbLocation"} + Lotman_LibLocation = StringParam{"Lotman.LibLocation"} Monitoring_DataLocation = StringParam{"Monitoring.DataLocation"} OIDC_AuthorizationEndpoint = StringParam{"OIDC.AuthorizationEndpoint"} OIDC_ClientID = StringParam{"OIDC.ClientID"} @@ -265,6 +267,7 @@ var ( ) var ( + Cache_EnableLotman = BoolParam{"Cache.EnableLotman"} Cache_EnableVoms = BoolParam{"Cache.EnableVoms"} Cache_SelfTest = BoolParam{"Cache.SelfTest"} Client_DisableHttpProxy = BoolParam{"Client.DisableHttpProxy"} @@ -274,6 +277,7 @@ var ( DisableHttpProxy = BoolParam{"DisableHttpProxy"} DisableProxyFallback = BoolParam{"DisableProxyFallback"} Logging_DisableProgressBars = BoolParam{"Logging.DisableProgressBars"} + Lotman_EnableAPI = BoolParam{"Lotman.EnableAPI"} Monitoring_MetricAuthorization = BoolParam{"Monitoring.MetricAuthorization"} Monitoring_PromQLAuthorization = BoolParam{"Monitoring.PromQLAuthorization"} Origin_EnableBroker = BoolParam{"Origin.EnableBroker"} @@ -325,6 +329,7 @@ var ( GeoIPOverrides = ObjectParam{"GeoIPOverrides"} Issuer_AuthorizationTemplates = ObjectParam{"Issuer.AuthorizationTemplates"} Issuer_OIDCAuthenticationRequirements = ObjectParam{"Issuer.OIDCAuthenticationRequirements"} + Lotman_Lots = ObjectParam{"Lotman.Lots"} Origin_Exports = ObjectParam{"Origin.Exports"} Registry_CustomRegistrationFields = ObjectParam{"Registry.CustomRegistrationFields"} Registry_Institutions = ObjectParam{"Registry.Institutions"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index b739a75c6..1e6a1ca4b 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -27,6 +27,7 @@ type Config struct { Cache struct { Concurrency int DataLocation string + EnableLotman bool EnableVoms bool ExportLocation string PermittedNamespaces []string @@ -117,6 +118,12 @@ type Config struct { Xrootd string } } + Lotman struct { + DbLocation string + EnableAPI bool + LibLocation string + Lots interface{} + } MinimumDownloadSpeed int Monitoring struct { AggregatePrefixes []string @@ -273,6 +280,7 @@ type configWithType struct { Cache struct { Concurrency struct { Type string; Value int } DataLocation struct { Type string; Value string } + EnableLotman struct { Type string; Value bool } EnableVoms struct { Type string; Value bool } ExportLocation struct { Type string; Value string } PermittedNamespaces struct { Type string; Value []string } @@ -363,6 +371,12 @@ type configWithType struct { Xrootd struct { Type string; Value string } } } + Lotman struct { + DbLocation struct { Type string; Value string } + EnableAPI struct { Type string; Value bool } + LibLocation struct { Type string; Value string } + Lots struct { Type string; Value interface{} } + } MinimumDownloadSpeed struct { Type string; Value int } Monitoring struct { AggregatePrefixes struct { Type string; Value []string } diff --git a/server_utils/registry.go b/server_utils/registry.go index 03b0f7dc5..51589cb4a 100644 --- a/server_utils/registry.go +++ b/server_utils/registry.go @@ -26,9 +26,11 @@ import ( "net/url" "strings" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/pkg/errors" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" - "github.com/pkg/errors" ) // For a given prefix, get the prefix's issuer URL, where we consider that the openid endpoint @@ -101,3 +103,46 @@ func GetJWKSURLFromIssuerURL(issuerUrl string) (string, error) { return "", errors.New(fmt.Sprintf("no key found in openid-configuration for issuer %s", issuerUrl)) } } + +// Given an issuer URL, get the JWKS from the issuer's JWKS URL +func GetJWKSFromIssUrl(issuer string) (*jwk.Set, error) { + // Make sure our URL is solid + issuerUrl, err := url.Parse(issuer) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintln("Invalid issuer URL: ", issuerUrl)) + } + + // Discover the JWKS URL from the issuer + pubkeyUrlStr, err := GetJWKSURLFromIssuerURL(issuerUrl.String()) + if err != nil { + return nil, errors.Wrap(err, "Error getting JWKS URL from issuer URL") + } + + // Query the JWKS URL for the public keys + httpClient := &http.Client{Transport: config.GetTransport()} + req, err := http.NewRequest("GET", pubkeyUrlStr, nil) + if err != nil { + return nil, errors.Wrap(err, "Error creating request to issuer's JWKS URL") + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, errors.Wrapf(err, "Error querying issuer's key endpoint (%s)", pubkeyUrlStr) + } + defer resp.Body.Close() + // Check the response code, make sure it's not in the error ranges (400-500) + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return nil, errors.Errorf("The issuer's JWKS endpoint returned an unexpected status: %s", resp.Status) + } + + // Read the response body and parse the JWKs from it + jwksStr, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrapf(err, "Error reading response body from %s", pubkeyUrlStr) + } + kSet, err := jwk.ParseString(string(jwksStr)) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing JWKs from %s", pubkeyUrlStr) + } + + return &kSet, nil +} diff --git a/token/token_verify.go b/token/token_verify.go index e47c886e4..aa9a0bba5 100644 --- a/token/token_verify.go +++ b/token/token_verify.go @@ -20,19 +20,23 @@ package token import ( "context" + "encoding/json" "fmt" + "io" "net/http" + "net/url" "strings" "time" "github.com/gin-gonic/gin" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jwt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/token_scopes" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) type ( @@ -233,3 +237,132 @@ func Verify(ctx *gin.Context, authOption AuthOption) (status int, verified bool, log.Debug("Cannot verify token:\n", errMsg) return http.StatusForbidden, false, errors.New("Cannot verify token: " + errMsg) } + +// Given a request, try to get a token from its "authz" query parameter or "Authorization" header +func GetAuthzEscaped(ctx *gin.Context) (authzEscaped string) { + if authzQuery := ctx.Request.URL.Query()["authz"]; len(authzQuery) > 0 { + authzEscaped = authzQuery[0] + // if the authz URL query is coming from XRootD, it probably has a "Bearer " tacked in front + // even though it's coming via a URL + authzEscaped = strings.TrimPrefix(authzEscaped, "Bearer ") + } else if authzHeader := ctx.Request.Header["Authorization"]; len(authzHeader) > 0 { + authzEscaped = strings.TrimPrefix(authzHeader[0], "Bearer ") + authzEscaped = url.QueryEscape(authzEscaped) + } else if authzCookie, err := ctx.Cookie("login"); err == nil && len(authzCookie) > 0 { + authzEscaped = url.QueryEscape(authzCookie) + } + return +} + +// For a given prefix, get the prefix's issuer URL, where we consider that the openid endpoint +// we use to look up a key location. Note that this is NOT the same as the issuer key -- to +// find that, follow openid-style discovery using the issuer URL as a base. +func GetNSIssuerURL(prefix string) (string, error) { + if prefix == "" || !strings.HasPrefix(prefix, "/") { + return "", errors.New(fmt.Sprintf("the prefix \"%s\" is invalid", prefix)) + } + registryUrlStr := param.Federation_RegistryUrl.GetString() + if registryUrlStr == "" { + return "", errors.New("federation registry URL is not set and was not discovered") + } + registryUrl, err := url.Parse(registryUrlStr) + if err != nil { + return "", err + } + + registryUrl.Path, err = url.JoinPath(registryUrl.Path, "api", "v1.0", "registry", prefix) + + if err != nil { + return "", errors.Wrapf(err, "failed to construct openid-configuration lookup URL for prefix %s", prefix) + } + return registryUrl.String(), nil +} + +// Given an issuer url, lookup the JWKS URL from the openid-configuration +// For example, if the issuer URL is https://registry.com:8446/api/v1.0/registry/test-namespace, +// this function will return the key indicated by the openid-configuration JSON hosted at +// https://registry.com:8446/api/v1.0/registry/test-namespace/.well-known/openid-configuration. +func GetJWKSURLFromIssuerURL(issuerUrl string) (string, error) { + // Get/parse the openid-configuration JSON to lookup key location + issOpenIDUrl, err := url.Parse(issuerUrl) + if err != nil { + return "", errors.Wrap(err, "failed to parse issuer URL") + } + issOpenIDUrl.Path, _ = url.JoinPath(issOpenIDUrl.Path, ".well-known", "openid-configuration") + + client := &http.Client{Transport: config.GetTransport()} + openIDCfg, err := client.Get(issOpenIDUrl.String()) + if err != nil { + return "", errors.Wrapf(err, "failed to lookup openid-configuration for issuer %s", issuerUrl) + } + defer openIDCfg.Body.Close() + + // If we hit an old registry, it may not have the openid-configuration. In that case, we fallback to the old + // behavior of looking for the key directly at the issuer URL. + if openIDCfg.StatusCode == http.StatusNotFound { + oldKeyLoc, err := url.JoinPath(issuerUrl, ".well-known", "issuer.jwks") + if err != nil { + return "", errors.Wrapf(err, "failed to construct key lookup URL for issuer %s", issuerUrl) + } + return oldKeyLoc, nil + } + + body, err := io.ReadAll(openIDCfg.Body) + if err != nil { + return "", errors.Wrapf(err, "failed to read response body from %s", issuerUrl) + } + + var openIDCfgMap map[string]string + err = json.Unmarshal(body, &openIDCfgMap) + if err != nil { + return "", errors.Wrapf(err, "failed to unmarshal openid-configuration for issuer %s", issuerUrl) + } + + if keyLoc, ok := openIDCfgMap["jwks_uri"]; ok { + return keyLoc, nil + } else { + return "", errors.New(fmt.Sprintf("no key found in openid-configuration for issuer %s", issuerUrl)) + } +} + +func GetJWKSFromIssUrl(issuer string) (*jwk.Set, error) { + // Make sure our URL is solid + issuerUrl, err := url.Parse(issuer) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintln("Invalid issuer URL: ", issuerUrl)) + } + + // Discover the JWKS URL from the issuer + pubkeyUrlStr, err := GetJWKSURLFromIssuerURL(issuerUrl.String()) + if err != nil { + return nil, errors.Wrap(err, "Error getting JWKS URL from issuer URL") + } + + // Query the JWKS URL for the public keys + httpClient := &http.Client{Transport: config.GetTransport()} + req, err := http.NewRequest("GET", pubkeyUrlStr, nil) + if err != nil { + return nil, errors.Wrap(err, "Error creating request to issuer's JWKS URL") + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, errors.Wrapf(err, "Error querying issuer's key endpoint (%s)", pubkeyUrlStr) + } + defer resp.Body.Close() + // Check the response code, make sure it's not in the error ranges (400-500) + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return nil, errors.Errorf("The issuer's JWKS endpoint returned an unexpected status: %s", resp.Status) + } + + // Read the response body and parse the JWKs from it + jwksStr, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrapf(err, "Error reading response body from %s", pubkeyUrlStr) + } + kSet, err := jwk.ParseString(string(jwksStr)) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing JWKs from %s", pubkeyUrlStr) + } + + return &kSet, nil +} diff --git a/token/token_verify_test.go b/token/token_verify_test.go index ef73e9395..f027f34eb 100644 --- a/token/token_verify_test.go +++ b/token/token_verify_test.go @@ -19,6 +19,7 @@ package token import ( + "bytes" "fmt" "net/http" "net/http/httptest" @@ -295,3 +296,42 @@ func TestVerify(t *testing.T) { } } + +func TestGetAuthzEscaped(t *testing.T) { + // Test passing a token via header with no bearer prefix + req, err := http.NewRequest(http.MethodPost, "http://fake-server.com", bytes.NewBuffer([]byte("a body"))) + assert.NoError(t, err) + req.Header.Set("Authorization", "tokenstring") + ctx := &gin.Context{Request: req} + escapedToken := GetAuthzEscaped(ctx) + assert.Equal(t, escapedToken, "tokenstring") + + // Test passing a token via query with no bearer prefix + req, err = http.NewRequest(http.MethodPost, "http://fake-server.com/foo?authz=tokenstring", bytes.NewBuffer([]byte("a body"))) + assert.NoError(t, err) + ctx = &gin.Context{Request: req} + escapedToken = GetAuthzEscaped(ctx) + assert.Equal(t, escapedToken, "tokenstring") + + // Test passing the token via header with Bearer prefix + req, err = http.NewRequest(http.MethodPost, "http://fake-server.com", bytes.NewBuffer([]byte("a body"))) + assert.NoError(t, err) + ctx = &gin.Context{Request: req} + req.Header.Set("Authorization", "Bearer tokenstring") + escapedToken = GetAuthzEscaped(ctx) + assert.Equal(t, escapedToken, "tokenstring") + + // Test passing the token via URL with Bearer prefix and + encoded space + req, err = http.NewRequest(http.MethodPost, "http://fake-server.com/foo?authz=Bearer+tokenstring", bytes.NewBuffer([]byte("a body"))) + assert.NoError(t, err) + ctx = &gin.Context{Request: req} + escapedToken = GetAuthzEscaped(ctx) + assert.Equal(t, escapedToken, "tokenstring") + + // Finally, the same test as before, but test with %20 encoded space + req, err = http.NewRequest(http.MethodPost, "http://fake-server.com/foo?authz=Bearer%20tokenstring", bytes.NewBuffer([]byte("a body"))) + assert.NoError(t, err) + ctx = &gin.Context{Request: req} + escapedToken = GetAuthzEscaped(ctx) + assert.Equal(t, escapedToken, "tokenstring") +} diff --git a/token_scopes/token_scopes.go b/token_scopes/token_scopes.go index b20ef764f..54f2ea598 100644 --- a/token_scopes/token_scopes.go +++ b/token_scopes/token_scopes.go @@ -1,4 +1,5 @@ -// Code generated by go generate; DO NOT EDIT. +// Code generated by go generate; DO NOT EDIT THIS FILE. +// To make changes to source, see generate/scope_generator.go and docs/scopes.yaml /*************************************************************** * * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research @@ -43,6 +44,12 @@ const ( Storage_Create TokenScope = "storage.create" Storage_Modify TokenScope = "storage.modify" Storage_Stage TokenScope = "storage.stage" + + // Lotman Scopes + Lot_Create TokenScope = "lot.create" + Lot_Read TokenScope = "lot.read" + Lot_Modify TokenScope = "lot.modify" + Lot_Delete TokenScope = "lot.delete" ) func (s TokenScope) String() string { diff --git a/utils/web_utils.go b/utils/web_utils.go index 7e679f41f..01b6c90eb 100644 --- a/utils/web_utils.go +++ b/utils/web_utils.go @@ -25,6 +25,7 @@ import ( "fmt" "io" "net/http" + "strings" "github.com/pkg/errors" @@ -174,3 +175,27 @@ func CopyHeader(dst, src http.Header) { } } } + +// Simple parser to that takes a "values" string from a header and turns it +// into a map of key/value pairs +func HeaderParser(values string) (retMap map[string]string) { + retMap = map[string]string{} + + // Some headers might not have values, such as the + // X-OSDF-Authorization header when the resource is public + if values == "" { + return + } + + mapPairs := strings.Split(values, ",") + for _, pair := range mapPairs { + // Remove any unwanted spaces + pair = strings.ReplaceAll(pair, " ", "") + + // Break out key/value pairs and put in the map + split := strings.Split(pair, "=") + retMap[split[0]] = split[1] + } + + return retMap +} diff --git a/utils/web_utils_test.go b/utils/web_utils_test.go new file mode 100644 index 000000000..ae47b8316 --- /dev/null +++ b/utils/web_utils_test.go @@ -0,0 +1,38 @@ +/*************************************************************** +* +* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research +* +* Licensed under the Apache License, Version 2.0 (the "License"); you +* may not use this file except in compliance with the License. You may +* obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +***************************************************************/ + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHeaderParser(t *testing.T) { + header1 := "namespace=/foo/bar, issuer = https://get-your-tokens.org, readhttps=False" + newMap1 := HeaderParser(header1) + + assert.Equal(t, "/foo/bar", newMap1["namespace"]) + assert.Equal(t, "https://get-your-tokens.org", newMap1["issuer"]) + assert.Equal(t, "False", newMap1["readhttps"]) + + header2 := "" + newMap2 := HeaderParser(header2) + assert.Equal(t, map[string]string{}, newMap2) +} diff --git a/web_ui/frontend/app/config/page.tsx b/web_ui/frontend/app/config/page.tsx index 1607ade90..b9f021efc 100644 --- a/web_ui/frontend/app/config/page.tsx +++ b/web_ui/frontend/app/config/page.tsx @@ -368,7 +368,7 @@ function Config() { const filteredConfig: Config = structuredClone(config) Object.entries(configMetadata).forEach(([key, value]) => { - if([...enabledServers, "*"].filter(i => value.components.includes(i)).length === 0) { + if([...enabledServers, "*", "cache"].filter(i => value.components.includes(i)).length === 0) { deleteKey(filteredConfig, key.split(".")) } else { updateValue(filteredConfig, key.split("."), configMetadata[key]) diff --git a/web_ui/frontend/components/Config/DateTimeField.tsx b/web_ui/frontend/components/Config/DateTimeField.tsx new file mode 100644 index 000000000..80864ad45 --- /dev/null +++ b/web_ui/frontend/components/Config/DateTimeField.tsx @@ -0,0 +1,45 @@ +import {DateTimePicker, LocalizationProvider} from "@mui/x-date-pickers"; +import React, {useMemo, useCallback, SetStateAction, ChangeEvent} from "react"; + +import { ParameterInputProps } from "@/components/Config/index.d"; +import { createId, buildPatch } from "./util"; +import {DateTime} from "luxon"; +import 'chartjs-adapter-luxon'; +import {AdapterLuxon} from "@mui/x-date-pickers/AdapterLuxon"; + +export type DateTimePickerFieldProps = { + name: string; + value: number; + onChange: (a: number) => void; + verify?: (a: DateTime | null) => string; +} + +const DateTimeField = ({onChange, name, value, verify}: DateTimePickerFieldProps) => { + + const id = useMemo(() => createId(name), [name]) + + const [localValue, setLocalValue] = React.useState(value ? DateTime.fromSeconds(value) : null); + + const handleOnChange = useCallback((value: DateTime | null) => { + + if(value === null) { + return + } + + setLocalValue(value) + + onChange(value.toSeconds()) + }, [onChange]) + + return ( + + + + ) +} + +export default DateTimeField; diff --git a/web_ui/frontend/components/Config/Field.tsx b/web_ui/frontend/components/Config/Field.tsx index 14cadc872..3d7c06539 100644 --- a/web_ui/frontend/components/Config/Field.tsx +++ b/web_ui/frontend/components/Config/Field.tsx @@ -2,7 +2,7 @@ import { AuthorizationTemplate, CustomRegistrationField, Duration, Export, GeoIPOverride, - Institution, IPMapping, + Institution, IPMapping, Lot, OIDCAuthenticationRequirement, Parameter, ParameterInputProps @@ -15,7 +15,8 @@ import { ObjectField, GeoIPOverrideForm, InstitutionForm, - OIDCAuthenticationRequirementForm + OIDCAuthenticationRequirementForm, + LotForm } from "./ObjectField"; import {buildPatch} from "@/components/Config/util"; import IPMappingForm from "@/components/Config/ObjectField/IPMappingForm"; @@ -59,6 +60,8 @@ const Field = ({ onChange, ...props}: ParameterInputProps) => { return } name={props.name} value={props.Value as CustomRegistrationField[]} Form={CustomRegistrationFieldForm} keyGetter={v => v.name}/> case "Exports": return } name={props.name} value={props.Value as Export[]} Form={ExportForm} keyGetter={v => v.federationprefix}/> + case "Lots": + return } name={props.name} value={props.Value as Lot[]} Form={LotForm} keyGetter={v => v.lotname}/> default: console.log("Unknown type: " + props.type) } diff --git a/web_ui/frontend/components/Config/ObjectField/LotForm.tsx b/web_ui/frontend/components/Config/ObjectField/LotForm.tsx new file mode 100644 index 000000000..0658f44f1 --- /dev/null +++ b/web_ui/frontend/components/Config/ObjectField/LotForm.tsx @@ -0,0 +1,97 @@ +import {Action, Lot, Path} from "@/components/Config/index.d"; +import React from "react"; +import {Box, Button} from "@mui/material"; + +import {FormProps} from "@/components/Config/ObjectField/ObjectField"; +import {StringField, IntegerField, DateTimeField} from "@/components/Config"; +import {ObjectField} from "@/components/Config/ObjectField"; +import PathForm from "@/components/Config/ObjectField/PathForm"; + +const verifyForm = (x: Lot) => { + return ( + x.lotname != "" && + x.owner != "" && + x.managementpolicyattrs.creationtime.value != 0 && + x.managementpolicyattrs.expirationtime.value != 0 && + x.managementpolicyattrs.deletiontime.value != 0 + ) +} + +const LotForm = ({ onSubmit, value }: FormProps) => { + + const [lotName, setLotName] = React.useState(value?.lotname || "") + const [owner, setOwner] = React.useState(value?.owner || "") + const [paths, setPaths] = React.useState(value?.paths || []) + const [dedicatedGB, setDedicatedGB] = React.useState(value?.managementpolicyattrs?.dedicatedgb || 0) + const [opportunisticGB, setOpportunisticGB] = React.useState(value?.managementpolicyattrs?.opportunisticgb || 0) + const [maxNumberObjects, setMaxNumberObjects] = React.useState(value?.managementpolicyattrs?.maxnumberobjects.value || 0) + const [creationTime, setCreationTime] = React.useState(value?.managementpolicyattrs?.creationtime?.value || 0) + const [expirationTime, setExpirationTime] = React.useState(value?.managementpolicyattrs?.expirationtime?.value || 0) + const [deletionTime, setDeletionTime] = React.useState(value?.managementpolicyattrs?.deletiontime?.value || 0) + + const submitHandler = (event: React.FormEvent) => { + event.preventDefault(); + const value = { + lotname: lotName, + owner: owner, + paths: paths, + managementpolicyattrs: { + dedicatedgb: dedicatedGB, + opportunisticgb: opportunisticGB, + maxnumberobjects: { + value: maxNumberObjects + }, + creationtime: { + value: creationTime + }, + expirationtime: { + value: expirationTime + }, + deletiontime: { + value: deletionTime + } + } + } + + if(!verifyForm(value)) { + return + } + + onSubmit(value); + } + + return ( +
+ + + + + + + + x.path}/> + + + + + + + + + + + + + + + + + + + + +
+ ) +} + +export default LotForm; diff --git a/web_ui/frontend/components/Config/ObjectField/PathForm.tsx b/web_ui/frontend/components/Config/ObjectField/PathForm.tsx new file mode 100644 index 000000000..5f0a506d6 --- /dev/null +++ b/web_ui/frontend/components/Config/ObjectField/PathForm.tsx @@ -0,0 +1,43 @@ +import {Path} from "@/components/Config/index.d"; +import React, {MouseEventHandler} from "react"; +import {Box, Button, TextField} from "@mui/material"; + +import {FormProps, ModalProps} from "@/components/Config/ObjectField/ObjectField"; +import {StringField, BooleanField} from "@/components/Config"; + +const verifyForm = (x: Path) => { + return x.path != "" +} + +const PathForm = ({ onSubmit, value }: FormProps) => { + + const [path, setPath] = React.useState(value?.path || "") + const [recursive, setRecursive] = React.useState(value?.recursive || false) + + const submitHandler = () => { + const pathObject = { + path: path, + recursive: recursive + } + + if(!verifyForm(pathObject)) { + return + } + + onSubmit(pathObject); + } + + return ( + <> + + + + + + + + + ) +} + +export default PathForm; diff --git a/web_ui/frontend/components/Config/ObjectField/index.tsx b/web_ui/frontend/components/Config/ObjectField/index.tsx index a5077780a..a80c9ed30 100644 --- a/web_ui/frontend/components/Config/ObjectField/index.tsx +++ b/web_ui/frontend/components/Config/ObjectField/index.tsx @@ -6,5 +6,6 @@ import IPMappingForm from "./IPMappingForm"; import CustomRegistrationFieldForm from "./CustomRegistrationFieldForm"; import OptionForm from "./OptionForm"; import ExportForm from "./ExportForm"; +import LotForm from "./LotForm"; -export {ObjectField, InstitutionForm, GeoIPOverrideForm, OIDCAuthenticationRequirementForm, IPMappingForm, OptionForm, CustomRegistrationFieldForm, ExportForm} +export {ObjectField, InstitutionForm, GeoIPOverrideForm, OIDCAuthenticationRequirementForm, IPMappingForm, OptionForm, CustomRegistrationFieldForm, ExportForm, LotForm} diff --git a/web_ui/frontend/components/Config/index.d.tsx b/web_ui/frontend/components/Config/index.d.tsx index 0110b35f1..4f44a2a04 100644 --- a/web_ui/frontend/components/Config/index.d.tsx +++ b/web_ui/frontend/components/Config/index.d.tsx @@ -10,7 +10,7 @@ export interface ParameterMetadata { export interface ParameterValue { Type: string; - Value: string | number | boolean | string[] | undefined | Coordinate[] | Institution[] | CustomRegistrationField[] | OIDCAuthenticationRequirement[] | AuthorizationTemplate[] | IPMapping[] | GeoIPOverride[] | Export[]; + Value: string | number | boolean | string[] | undefined | Coordinate[] | Institution[] | CustomRegistrationField[] | OIDCAuthenticationRequirement[] | AuthorizationTemplate[] | IPMapping[] | GeoIPOverride[] | Export[] | Lot[]; } export type ParameterInputProps = Parameter & { @@ -84,3 +84,32 @@ export interface Export { federationprefix: string; capabilities: Capability[]; } + +export interface Path { + path: string; + recursive: boolean; +} + +export interface ManagementPolicyAttrs { + dedicatedgb: number; + opportunisticgb: number; + maxnumberobjects: { + value: number; + }; + creationtime: { + value: number; + }; + expirationtime: { + value: number; + }; + deletiontime: { + value: number; + }; +} + +export interface Lot { + lotname: string; + owner: string; + paths: Path[]; + managementpolicyattrs: ManagementPolicyAttrs; +} diff --git a/web_ui/frontend/components/Config/index.tsx b/web_ui/frontend/components/Config/index.tsx index 53807aa36..5d4e6273f 100644 --- a/web_ui/frontend/components/Config/index.tsx +++ b/web_ui/frontend/components/Config/index.tsx @@ -5,6 +5,7 @@ import StringSliceField from "./StringSliceField"; import IntegerField from "./IntegerField"; import MultiSelectField from "./MultiSelectField"; import SelectField from "./SelectField"; +import DateTimeField from "./DateTimeField"; import Field from "./Field"; -export { BooleanField, StringField, DurationField, StringSliceField, IntegerField, MultiSelectField, SelectField, Field } +export { BooleanField, StringField, DurationField, StringSliceField, IntegerField, MultiSelectField, SelectField, DateTimeField, Field }