Merge pull request PelicanPlatform#1291 from haoming29/add-http-backe…

…nd-to-export

Add http backend to origin data export API

docs/parameters.yaml

@@ -475,6 +475,12 @@ description: |+
         Capabilities: ["Reads", "PublicReads", "Writes", "Listings", "DirectReads"]
         SentinelLocation: demoproject_origin_A
     ```
+
+  There are additional configuration fields if Origin.StorageType == "s3".
+  - S3Bucket: [OPTIONAL] See `Origin.S3Bucket` for details
+  - S3AccessKeyfile: [OPTIONAL] See `Origin.S3AccessKeyfile` for details
+  - S3SecretKeyfile: [OPTIONAL] See `Origin.S3SecretKeyfile` for details
+
 type: object
 default: none
 components: ["origin"]
@@ -770,7 +776,7 @@ components: ["origin"]
 name: Origin.S3ServiceName
 description: |+
   [Deprecated] Origin.S3ServiceName was previously used in part to determine an export's FederationPrefix, but
-  upstream changes no longer rely on this value. As of Pelican 7.7.0, setting this value no longer has any effect.
+  upstream changes no longer rely on this value. As of Pelican `7.7.0`, setting this value no longer has any effect.
   AWSv4 signatures used by S3 servers to handle authentication now hardcode "s3" as their service name.
 
   When constructing signed URLs for S3, this value is used as a part of the signature. It is almost always "s3". For more
@@ -796,6 +802,8 @@ components: ["origin"]
 ---
 name: Origin.S3Bucket
 description: |+
+  **Note**: This value is only for setting up an origin that exports **one** storage prefix. For multiple exports, use `Origin.Exports`
+
   Objects in S3 are stored in "buckets", which have unique names at each S3 service URL (ie the URL that provides access to your objects).
   Setting a bucket restricts the origin to only serving objects from that bucket.
 
@@ -817,6 +825,8 @@ components: ["origin"]
 ---
 name: Origin.S3AccessKeyfile
 description: |+
+  **Note**: This value is only for setting up an origin that exports **one** storage prefix. For multiple exports, use `Origin.Exports`
+
   A path to a file containing an S3 access keyfile (also sometimes called an API key) for authenticated buckets when an origin
   is run in S3 mode.
 
@@ -828,6 +838,8 @@ components: ["origin"]
 ---
 name: Origin.S3SecretKeyfile
 description: |+
+  **Note**: This value is only for setting up an origin that exports **one** storage prefix. For multiple exports, use `Origin.Exports`
+
   A path to a file containing an S3 secret keyfile for authenticated buckets when an origin is run in S3 mode.
 
   This value is OPTIONAL for S3 origins, and only applies when an exported bucket requires authentication. It should not be used
@@ -838,8 +850,8 @@ components: ["origin"]
 ---
 name: Origin.S3UrlStyle
 description: |+
-  The style of S3 urls used by the service URL host. This can be either "path" if objects are fetched at <service-url>/<bucket>/<object>
-  or "virtual" if objects are fetched at <bucket>.<service-url>/<object>.
+  The style of S3 urls used by the service URL host. This can be either "path" if objects are fetched at `<service-url>/<bucket>/<object>`
+  or "virtual" if objects are fetched at `<bucket>.<service-url>/<object>`.
 
   This value is REQUIRED for S3 origins, but defaults to "path" if not set.
 type: string

origin/origin_ui.go

@@ -37,13 +37,28 @@ import (
 
 type (
 	exportsRes struct {
-		Type    string             `json:"type"` // either "posix" or "s3"
+		Type string `json:"type"` // "posix" | "s3" | "https"
+
+		// For S3 backend
+		S3Region     string `json:"s3Region,omitempty"`
+		S3ServiceUrl string `json:"s3ServiceUrl,omitempty"`
+		S3UrlStyle   string `json:"s3UrlStyle,omitempty"`
+
+		// For https backend
+		HttpServiceUrl string `json:"httpServiceUrl,omitempty"`
+
 		Exports []exportWithStatus `json:"exports"`
 	}
 )
 
 func handleExports(ctx *gin.Context) {
-	storageType := param.Origin_StorageType.GetString()
+	st := param.Origin_StorageType.GetString()
+	storageType, err := server_utils.ParseOriginStorageType(st)
+	if err != nil {
+		log.Errorf("Failed to parse origin storage type: %v", err)
+		ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{Status: server_structs.RespFailed, Msg: "Server encountered error when parsing the storage type of the origin: " + err.Error()})
+	}
+
 	exports, err := server_utils.GetOriginExports()
 	if err != nil {
 		log.Errorf("Failed to get the origin exports: %v", err)
@@ -99,7 +114,16 @@ func handleExports(ctx *gin.Context) {
 		}
 	}
 
-	ctx.JSON(http.StatusOK, exportsRes{Type: storageType, Exports: wrappedExports})
+	res := exportsRes{Type: string(storageType), Exports: wrappedExports}
+	switch storageType {
+	case server_utils.OriginStorageS3:
+		res.S3Region = param.Origin_S3Region.GetString()
+		res.S3ServiceUrl = param.Origin_S3ServiceUrl.GetString()
+		res.S3UrlStyle = param.Origin_S3UrlStyle.GetString()
+	case server_utils.OriginStorageHTTPS:
+		res.HttpServiceUrl = param.Origin_HttpServiceUrl.GetString()
+	}
+	ctx.JSON(http.StatusOK, res)
 }
 
 func RegisterOriginWebAPI(engine *gin.Engine) {

server_utils/origin.go

@@ -41,11 +41,11 @@ type (
 		StoragePrefix    string `json:"storage_prefix"`
 		FederationPrefix string `json:"federation_prefix"`
 
-		// Export fields specific to S3. Other things like
+		// Export fields specific to S3 backend. Other things like
 		// S3ServiceUrl, S3Region, etc are kept top-level in the config
-		S3Bucket        string `json:"s3_bucket"`
-		S3AccessKeyfile string `json:"s3_access_keyfile"`
-		S3SecretKeyfile string `json:"s3_secret_keyfile"`
+		S3Bucket        string `json:"s3_bucket,omitempty"`
+		S3AccessKeyfile string `json:"s3_access_keyfile,omitempty"`
+		S3SecretKeyfile string `json:"s3_secret_keyfile,omitempty"`
 
 		// Capabilities for the export
 		Capabilities     server_structs.Capabilities `json:"capabilities"`

swagger/pelican-swagger.yaml

@@ -476,7 +476,21 @@ definitions:
       type:
         type: string
         example: "posix"
-        description: "The storage type of the origin server. Is either \"posix\" or \"s3\""
+        description: "The storage type of the origin server. Values can be posix, s3, and https"
+      s3Region:
+        type: string
+        description: "The server region for S3 storage. Only available when type == s3"
+      s3ServiceUrl:
+        type: string
+        description: "The URL that provides API access the the S3 objects. Only available when type == s3"
+      s3UrlStyle:
+        type: string
+        description:
+          The style of S3 urls used by the service URL host. Can be either \"path\" or \"virtual\". Only available when type == s3
+          Refer to [Pelican Documentation](https://docs.pelicanplatform.org/parameters#Origin-S3UrlStyle) for details
+      httpServiceUrl:
+        type: string
+        description: "The URL used as the base for requests to the backend. Only available when type == https"
       exports:
         type: array
         items: