Merge pull request #889 from jhiemstrawisc/issue-817

Add LotMan configuration to the cache

.github/scripts/validate-parameters/main.py

@@ -15,7 +15,8 @@
     "OIDCAuthenticationRequirements",
     "AuthorizationTemplates",
     "IPMapping",
-    "Exports"
+    "Exports",
+    "Lots"
 ]
 
 

client/director.go

@@ -28,40 +28,18 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pelicanplatform/pelican/config"
-	namespaces "github.com/pelicanplatform/pelican/namespaces"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/pelicanplatform/pelican/config"
+	namespaces "github.com/pelicanplatform/pelican/namespaces"
+	"github.com/pelicanplatform/pelican/utils"
 )
 
 type directorResponse struct {
 	Error string `json:"error"`
 }
 
-// Simple parser to that takes a "values" string from a header and turns it
-// into a map of key/value pairs
-func HeaderParser(values string) (retMap map[string]string) {
-	retMap = map[string]string{}
-
-	// Some headers might not have values, such as the
-	// X-OSDF-Authorization header when the resource is public
-	if values == "" {
-		return
-	}
-
-	mapPairs := strings.Split(values, ",")
-	for _, pair := range mapPairs {
-		// Remove any unwanted spaces
-		pair = strings.ReplaceAll(pair, " ", "")
-
-		// Break out key/value pairs and put in the map
-		split := strings.Split(pair, "=")
-		retMap[split[0]] = split[1]
-	}
-
-	return retMap
-}
-
 // Given the Director response, create the ordered list of caches
 // and store it as namespace.SortedDirectorCaches
 func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Namespace, err error) {
@@ -70,7 +48,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name
 		err = errors.New("Pelican director did not include mandatory X-Pelican-Namespace header in response")
 		return
 	}
-	xPelicanNamespace := HeaderParser(pelicanNamespaceHdr[0])
+	xPelicanNamespace := utils.HeaderParser(pelicanNamespaceHdr[0])
 	namespace.Path = xPelicanNamespace["namespace"]
 	namespace.UseTokenOnRead, _ = strconv.ParseBool(xPelicanNamespace["require-token"])
 	namespace.ReadHTTPS, _ = strconv.ParseBool(xPelicanNamespace["readhttps"])
@@ -82,15 +60,15 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name
 		//So it's a map entry - HeaderParser returns a max entry
 		//We want to appen the value
 		for _, authEntry := range dirResp.Header.Values("X-Pelican-Authorization") {
-			parsedEntry := HeaderParser(authEntry)
+			parsedEntry := utils.HeaderParser(authEntry)
 			xPelicanAuthorization = append(xPelicanAuthorization, parsedEntry["issuer"])
 		}
 		namespace.Issuer = xPelicanAuthorization
 	}
 
 	var xPelicanTokenGeneration map[string]string
 	if len(dirResp.Header.Values("X-Pelican-Token-Generation")) > 0 {
-		xPelicanTokenGeneration = HeaderParser(dirResp.Header.Values("X-Pelican-Token-Generation")[0])
+		xPelicanTokenGeneration = utils.HeaderParser(dirResp.Header.Values("X-Pelican-Token-Generation")[0])
 
 		// Instantiate the cred gen struct
 		namespace.CredentialGen = &namespaces.CredentialGeneration{}

client/director_test.go

@@ -29,21 +29,9 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	namespaces "github.com/pelicanplatform/pelican/namespaces"
+	"github.com/pelicanplatform/pelican/utils"
 )
 
-func TestHeaderParser(t *testing.T) {
-	header1 := "namespace=/foo/bar, issuer = https://get-your-tokens.org, readhttps=False"
-	newMap1 := HeaderParser(header1)
-
-	assert.Equal(t, "/foo/bar", newMap1["namespace"])
-	assert.Equal(t, "https://get-your-tokens.org", newMap1["issuer"])
-	assert.Equal(t, "False", newMap1["readhttps"])
-
-	header2 := ""
-	newMap2 := HeaderParser(header2)
-	assert.Equal(t, map[string]string{}, newMap2)
-}
-
 func TestGetCachesFromDirectorResponse(t *testing.T) {
 	// Construct the Director's Response, comprising headers and a body
 	directorHeaders := make(map[string][]string)
@@ -125,7 +113,7 @@ func TestCreateNsFromDirectorResp(t *testing.T) {
 	var xPelicanAuthorization map[string]string
 	var issuer string
 	if len(directorResponse.Header.Values("X-Pelican-Authorization")) > 0 {
-		xPelicanAuthorization = HeaderParser(directorResponse.Header.Values("X-Pelican-Authorization")[0])
+		xPelicanAuthorization = utils.HeaderParser(directorResponse.Header.Values("X-Pelican-Authorization")[0])
 		issuer = xPelicanAuthorization["issuer"]
 	}
 

config/config.go

@@ -880,12 +880,16 @@ func InitServer(ctx context.Context, currentServers ServerType) error {
 		viper.SetDefault("Origin.Multiuser", true)
 		viper.SetDefault("Director.GeoIPLocation", "/var/cache/pelican/maxmind/GeoLite2-City.mmdb")
 		viper.SetDefault("Registry.DbLocation", "/var/lib/pelican/registry.sqlite")
+		// The lotman db will actually take this path and create the lot at /path/.lot/lotman_cpp.sqlite
+		viper.SetDefault("Lotman.DbLocation", "/var/lib/pelican")
 		viper.SetDefault("Monitoring.DataLocation", "/var/lib/pelican/monitoring/data")
 		viper.SetDefault("Shoveler.QueueDirectory", "/var/spool/pelican/shoveler/queue")
 		viper.SetDefault("Shoveler.AMQPTokenLocation", "/etc/pelican/shoveler-token")
 	} else {
 		viper.SetDefault("Director.GeoIPLocation", filepath.Join(configDir, "maxmind", "GeoLite2-City.mmdb"))
 		viper.SetDefault("Registry.DbLocation", filepath.Join(configDir, "ns-registry.sqlite"))
+		// Lotdb will live at <configDir>/.lot/lotman_cpp.sqlite
+		viper.SetDefault("Lotman.DbLocation", configDir)
 		viper.SetDefault("Monitoring.DataLocation", filepath.Join(configDir, "monitoring/data"))
 		viper.SetDefault("Shoveler.QueueDirectory", filepath.Join(configDir, "shoveler/queue"))
 		viper.SetDefault("Shoveler.AMQPTokenLocation", filepath.Join(configDir, "shoveler-token"))

docs/parameters.yaml

@@ -874,6 +874,12 @@ description: >-
   number of connections to this value.
 type: int
 default: none
+---
+name: Cache.EnableLotman
+description: >-
+  LotMan is a library that provides management of storage space in the cache.
+type: bool
+default: false
 components: ["cache"]
 ---
 name: Cache.PermittedNamespaces
@@ -1916,3 +1922,65 @@ description: >-
 type: string
 default: none
 components: ["plugin"]
+---
+############################
+#   LotMan-level configs   #
+############################
+name: Lotman.DbLocation
+description: >-
+  The prefix indicating where LotMan should store its lot database. For the provided path, the database
+  will be stored at <path>/.lot/lotman_cpp.sqlite
+type: filename
+root_default: /var/run/pelican
+default: $ConfigBase
+components: ["cache"]
+---
+name: Lotman.LibLocation
+description: >-
+  The location of the system's installed LotMan library (libLotMan.so). When unset, the system will attempt to find Lotman
+  at these fallback paths:
+  - /usr/lib64/libLotMan.so
+  - /usr/local/lib64/libLotMan.so
+  - /opt/local/lib64/libLotMan.so
+type: filename
+default: none
+components: ["cache"]
+---
+name: Lotman.EnableAPI
+description: >-
+  Whether Lotman should enable its CRUD web endpoints. If true, administrators with an appropriately-signed token can interface
+  with Lotman via HTTP. Otherwise, lots are only configurable via the Pelican configuration file at the cache.
+type: bool
+default: false
+components: ["cache"]
+---
+name: Lotman.Lots
+description: >-
+  Declarative configuration for LotMan.  This is a list of objects, each of which describes a "lot". Every lot
+  can be defined with the following:
+  - `LotName`: REQUIRED. The name of the lot.  This is used to identify the lot in the LotMan database.
+  - `Owner`: REQUIRED. A string identifying the owner of the lot's data (as opposed to someone who can modify the lot itself).
+    The Owner field should generally be set to the issue for the lot's namespace path. For example, if the lot
+    tracks namespace `/foo/bar`, the owner might be set to `https://registry.com/api/v1.0/registry/foo/bar`.
+  - `Paths`: OPTIONAL. A list of path objects, each of which describes a path that should be managed by the lot.
+      - `Path`: REQUIRED. The path to be managed by the lot.
+      - `Recursive`: REQUIRED. A boolean indicating whether the path should be managed recursively. If true, the lot will
+        manage all files and directories under the specified path.
+  - `ManagementPolicyAttrs`: REQUIRED. The lot's management policy attributes object. This contains information about resources the lot should
+    be allocated, and how it should be managed.
+      - `DedicatedGB`: REQUIRED. The amount of storage, in GB, that should be dedicated to the lot. This means the lot can assume it
+        always has access to this quantity.
+      - `OpportunisticGB`: REQUIRED. The amount of opportunistic storage, in GB, the lot should have access to, when storage is available.
+      - `MaxNumObjects`: REQUIRED. The maximum number of objects a lot is allowed to store.
+      - `CreationTime`: REQUIRED. A unix timestamp indicating when the lot should begin being considered valid. Times in the future indicate
+        the lot should not be considered valid until that time.
+      - `ExpirationTime`: REQUIRED. A unix timestamp indicating when the lot expires. Lots may continue to function after expiration, but lot
+        data owners should recognize the storage is at-will and may be pre-empted at any time.
+      - `DeletionTime`: REQUIRED. A unix timestamp indicating when the lot and its associated data should be deleted.
+
+  Note that example configurations can be found in lotman/resources/lots-config.yaml
+  For more information about LotMan configuration, see:
+  https://github.com/pelicanplatform/lotman
+type: object
+default: none
+components: ["cache"]

docs/scopes.yaml

@@ -128,3 +128,30 @@ description: >-
   For granting object staging permissions to the bearer of the token. This scope must also posses a path to be valid, eg `storage.stage:/foo/bar`
 issuedBy: ["origin"]
 acceptedBy: ["origin", "cache"]
+---
+############################
+#      Lotman Scopes       #
+############################
+name: "lot.create"
+description: >-
+  For creating a new lot
+issuedBy: ["origin"]
+acceptedBy: ["cache"]
+---
+name: "lot.read"
+description: >-
+  For getting/reading the contents of a lot from a cache
+issuedBy: ["origin"]
+acceptedBy: ["cache"]
+---
+name: "lot.modify"
+description: >-
+  For modifying the contents of a lot in a cache
+issuedBy: ["origin"]
+acceptedBy: ["cache"]
+---
+name: "lot.delete"
+description: >-
+  For deleting a lot from a cache
+issuedBy: ["origin"]
+acceptedBy: ["cache"]

generate/scope_generator.go

@@ -87,6 +87,7 @@ func GenTokenScope() {
 
 	scopes := make([]ScopeName, 0)
 	storageScopes := make([]ScopeName, 0)
+	lotmanScopes := make([]ScopeName, 0)
 
 	for i := 0; i < len(values); i++ {
 		entry := values[i].(map[string]interface{})
@@ -109,6 +110,8 @@ func GenTokenScope() {
 		if strings.HasPrefix(scopeName, "storage") {
 			displayName = strings.TrimSuffix(displayName, ":")
 			storageScopes = append(storageScopes, ScopeName{Raw: scopeName, Display: displayName})
+		} else if strings.HasPrefix(scopeName, "lot") {
+			lotmanScopes = append(lotmanScopes, ScopeName{Raw: scopeName, Display: displayName})
 		} else {
 			scopes = append(scopes, ScopeName{Raw: scopeName, Display: displayName})
 		}
@@ -124,17 +127,20 @@ func GenTokenScope() {
 	err = tokenTemplate.Execute(f, struct {
 		Scopes        []ScopeName
 		StorageScopes []ScopeName
+		LotmanScopes  []ScopeName
 	}{
 		Scopes:        scopes,
 		StorageScopes: storageScopes,
+		LotmanScopes:  lotmanScopes,
 	})
 
 	if err != nil {
 		panic(err)
 	}
 }
 
-var tokenTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
+var tokenTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT THIS FILE.
+// To make changes to source, see generate/scope_generator.go and docs/scopes.yaml
 /***************************************************************
  *
  * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
@@ -170,6 +176,11 @@ const (
 	{{- range $idx, $scope := .StorageScopes}}
 	{{$scope.Display}} TokenScope = "{{$scope.Raw}}"
 	{{- end}}
+
+	// Lotman Scopes
+	{{- range $idx, $scope := .LotmanScopes}}
+	{{$scope.Display}} TokenScope = "{{$scope.Raw}}"
+	{{- end}}
 )
 
 func (s TokenScope) String() string {

go.mod

@@ -8,6 +8,7 @@ go 1.21
 require (
 	github.com/JGLTechnologies/gin-rate-limit v1.5.4
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
+	github.com/ebitengine/purego v0.6.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/glebarez/sqlite v1.10.0
 	github.com/go-ini/ini v1.67.0

go.sum

@@ -144,6 +144,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/ebitengine/purego v0.6.0 h1:Yo9uBc1x+ETQbfEaf6wcBsjrQfCEnh/gaGUg7lguEJY=
+github.com/ebitengine/purego v0.6.0/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/elastic/go-sysinfo v1.11.2 h1:mcm4OSYVMyws6+n2HIVMGkln5HOpo5Ie1ZmbbNn0jg4=

launchers/cache_serve.go

@@ -26,17 +26,20 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+
 	"github.com/pelicanplatform/pelican/broker"
 	"github.com/pelicanplatform/pelican/cache"
 	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/daemon"
 	"github.com/pelicanplatform/pelican/launcher_utils"
+	"github.com/pelicanplatform/pelican/lotman"
 	"github.com/pelicanplatform/pelican/param"
 	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/pelicanplatform/pelican/server_utils"
 	"github.com/pelicanplatform/pelican/xrootd"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
 )
 
 func CacheServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, modules config.ServerType) (server_structs.XRootDServer, error) {
@@ -58,6 +61,19 @@ func CacheServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, m
 		return nil, err
 	}
 
+	// Register Lotman
+	if param.Cache_EnableLotman.GetBool() {
+		// Register the web endpoints
+		if param.Lotman_EnableAPI.GetBool() {
+			log.Debugln("Registering Lotman API")
+			lotman.RegisterLotman(ctx, engine.Group("/"))
+		}
+		// Bind the c library funcs to Go
+		if success := lotman.InitLotman(); !success {
+			return nil, errors.New("Failed to initialize lotman")
+		}
+	}
+
 	broker.RegisterBrokerCallback(ctx, engine.Group("/"))
 	broker.LaunchNamespaceKeyMaintenance(ctx, egrp)
 	configPath, err := xrootd.ConfigXrootd(ctx, false)

lotman/lotman.go

@@ -0,0 +1,39 @@
+//go:build windows || darwin || (linux && ppc64le)
+
+/***************************************************************
+*
+* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+*
+* Licensed under the Apache License, Version 2.0 (the "License"); you
+* may not use this file except in compliance with the License.  You may
+* obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+***************************************************************/
+
+// LotMan is only supported on Linux at the moment. This file is a placeholder for other platforms and is
+// intended to export any functions that might be called outside of the package
+package lotman
+
+import (
+	"context"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func RegisterLotman(ctx context.Context, router *gin.RouterGroup) {
+	log.Warningln("LotMan is not supported on this platform. Skipping...")
+}
+
+func InitLotman() bool {
+	log.Warningln("LotMan is not supported on this platform. Skipping...")
+	return false
+}