diff --git a/config/resources/osdf.yaml b/config/resources/osdf.yaml index 32bd8a6e0..ab9b33205 100644 --- a/config/resources/osdf.yaml +++ b/config/resources/osdf.yaml @@ -20,8 +20,9 @@ Xrootd: DetailedMonitoringHost: xrd-mon.osgstorage.org Federation: DiscoveryUrl: osg-htc.org - TopologyURL: https://topology.opensciencegrid.org - TopologyNamespaceURL: https://topology.opensciencegrid.org/osdf/namespaces?production=1 + TopologyUrl: https://topology.opensciencegrid.org + TopologyNamespaceUrl: https://topology.opensciencegrid.org/osdf/namespaces?production=1 + TopologyDowntimeUrl: https://topology.opensciencegrid.org/rgdowntime/xml TopologyReloadInterval: 4.5m Registry: RequireCacheApproval: true diff --git a/director/advertise.go b/director/advertise.go index 80b9f6a35..2bf532b51 100644 --- a/director/advertise.go +++ b/director/advertise.go @@ -20,6 +20,8 @@ package director import ( "context" + "encoding/xml" + "net/http" "net/url" "strings" "time" @@ -27,9 +29,11 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" + "github.com/pelicanplatform/pelican/utils" ) // Consolite two ServerAds that share the same ServerAd.URL. For all but the capability fields, @@ -54,7 +58,7 @@ func consolidateDupServerAd(newAd, existingAd server_structs.ServerAd) server_st // Takes in server information from topology and handles converting the necessary bits into a new Pelican // ServerAd. -func parseServerAdFromTopology(server server_utils.Server, serverType server_structs.ServerType, caps server_structs.Capabilities) server_structs.ServerAd { +func parseServerAdFromTopology(server server_structs.TopoServer, serverType server_structs.ServerType, caps server_structs.Capabilities) server_structs.ServerAd { serverAd := server_structs.ServerAd{} serverAd.Type = serverType.String() serverAd.Name = server.Resource @@ -118,30 +122,26 @@ func parseServerAdFromTopology(server server_utils.Server, serverType server_str return serverAd } -// Do a subtraction of excludeDowned set from the includeDowned set to find cache servers -// that are in downtime -// -// The excludeDowned is a list of running OSDF topology servers -// The includeDowned is a list of running and downed OSDF topology servers -func findDownedTopologyCache(excludeDowned, includeDowned []server_utils.Server) (caches []server_utils.Server) { - for _, included := range includeDowned { - found := false - for _, excluded := range excludeDowned { - if included == excluded { - found = true - break - } - } - if !found { - caches = append(caches, included) - } +// Use the topology downtime endpoint to create the list of downed servers. Servers are tracked using their +// resource name, NOT their FQDN. +func updateDowntimeFromTopology(ctx context.Context) error { + dtUrlStr := param.Federation_TopologyDowntimeUrl.GetString() + _, err := url.Parse(dtUrlStr) + if err != nil { + return errors.Wrapf(err, "encountered an invalid URL %s when parsing configured topology downtime URL", dtUrlStr) + } + tr := config.GetTransport() + resp, err := utils.MakeRequest(ctx, tr, dtUrlStr, http.MethodGet, nil, nil) + if err != nil { + return errors.Wrapf(err, "failed to fetch topology downtime from %s", dtUrlStr) } - return -} -// Update filteredServers based on topology downtime -func updateDowntimeFromTopology(excludedNss, includedNss *server_utils.TopologyNamespacesJSON) { - downedCaches := findDownedTopologyCache(excludedNss.Caches, includedNss.Caches) + // Parse the big blurb of XML into a struct. + var downtimeInfo server_structs.TopoDowntimeInfo + err = xml.Unmarshal(resp, &downtimeInfo) + if err != nil { + return errors.Wrap(err, "failed to unmarshal topology downtime XML") + } filteredServersMutex.Lock() defer filteredServersMutex.Unlock() @@ -151,33 +151,44 @@ func updateDowntimeFromTopology(excludedNss, includedNss *server_utils.TopologyN delete(filteredServers, key) } } - for _, dc := range downedCaches { - if sAd := serverAds.Get(dc.Endpoint); sAd == nil { - // The downed cache is not in the director yet - filteredServers[dc.Resource] = topoFiltered - } else { - // If we have the cache in the director, use it's name as the key - filteredServers[sAd.Value().Name] = topoFiltered + + const timeLayout = "Jan 2, 2006 15:04 PM MST" // see https://pkg.go.dev/time#pkg-constants + for _, downtime := range downtimeInfo.CurrentDowntimes.Downtimes { + parsedStartDT, err := time.Parse(timeLayout, downtime.StartTime) + if err != nil { + log.Warningf("Could not put %s into downtime because its start time '%s' could not be parsed: %s", downtime.ResourceName, downtime.StartTime, err) + continue + } + + parsedEndDT, err := time.Parse(timeLayout, downtime.EndTime) + if err != nil { + log.Warningf("Could not put %s into downtime because its end time '%s' could not be parsed: %s", downtime.ResourceName, downtime.EndTime, err) + continue + } + + currentTime := time.Now() + if parsedStartDT.Before(currentTime) && parsedEndDT.After(currentTime) { + filteredServers[downtime.ResourceName] = topoFiltered } } - log.Infof("The following servers are put in downtime: %#v", filteredServers) + + log.Infof("The following servers are currently configured in downtime: %#v", filteredServers) + return nil } // Populate internal cache with origin/cache ads func AdvertiseOSDF(ctx context.Context) error { - namespaces, err := server_utils.GetTopologyJSON(ctx, false) + namespaces, err := server_utils.GetTopologyJSON(ctx) if err != nil { return errors.Wrapf(err, "Failed to get topology JSON") } - // Second call to fetch all servers (including servers in downtime) - includedNss, err := server_utils.GetTopologyJSON(ctx, true) + err = updateDowntimeFromTopology(ctx) if err != nil { - return errors.Wrapf(err, "Failed to get topology JSON with server in downtime included (include_downed)") + // Don't treat this as a fatal error, but log it in a loud way. + log.Errorf("Unable to generate downtime list for servers from topology: %v", err) } - updateDowntimeFromTopology(namespaces, includedNss) - cacheAdMap := make(map[string]*server_structs.Advertisement) // key is serverAd.URL.String() originAdMap := make(map[string]*server_structs.Advertisement) // key is serverAd.URL.String() tGen := server_structs.TokenGen{} diff --git a/director/advertise_test.go b/director/advertise_test.go index 3ec334d92..5a2e05c3a 100644 --- a/director/advertise_test.go +++ b/director/advertise_test.go @@ -19,12 +19,15 @@ package director import ( + "bytes" "context" _ "embed" "net/http" "net/http/httptest" "net/url" "testing" + "text/template" + "time" "github.com/sirupsen/logrus" logrustest "github.com/sirupsen/logrus/hooks/test" @@ -33,7 +36,6 @@ import ( "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/server_structs" - "github.com/pelicanplatform/pelican/server_utils" ) var ( @@ -98,7 +100,7 @@ func TestConsolidateDupServerAd(t *testing.T) { func TestParseServerAdFromTopology(t *testing.T) { - server := server_utils.Server{ + server := server_structs.TopoServer{ Endpoint: "http://my-endpoint.com", AuthEndpoint: "https://my-auth-endpoint.com", Resource: "MY_SERVER", @@ -301,149 +303,85 @@ func TestAdvertiseOSDF(t *testing.T) { }) } -func TestFindDownedTopologyCache(t *testing.T) { - mockTopoCacheA := server_utils.Server{AuthEndpoint: "cacheA.org:8443", Endpoint: "cacheA.org:8000", Resource: "CACHE_A"} - mockTopoCacheB := server_utils.Server{AuthEndpoint: "cacheB.org:8443", Endpoint: "cacheB.org:8000", Resource: "CACHE_B"} - mockTopoCacheC := server_utils.Server{AuthEndpoint: "cacheC.org:8443", Endpoint: "cacheC.org:8000", Resource: "CACHE_C"} - mockTopoCacheD := server_utils.Server{AuthEndpoint: "cacheD.org:8443", Endpoint: "cacheD.org:8000", Resource: "CACHE_D"} - t.Run("empty-response", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{}, - []server_utils.Server{}, - ) - assert.Empty(t, get) - }) - - t.Run("no-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - assert.Empty(t, get) - }) - - t.Run("one-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - require.Len(t, get, 1) - assert.EqualValues(t, mockTopoCacheD, get[0]) - }) +func mockTopoDowntimeXMLHandler(w http.ResponseWriter, r *http.Request) { + downtimeInfo := server_structs.TopoCurrentDowntimes{ + Downtimes: []server_structs.TopoServerDowntime{ + { + // Current time falls in start-end window. SHould be filtered + ResourceName: "BOISE_INTERNET2_OSDF_CACHE", + ResourceFQDN: "dtn-pas.bois.nrp.internet2.edu", + StartTime: time.Now().Add(-24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + EndTime: time.Now().Add(24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + { + // start time is after current time. Should NOT be filtered + ResourceName: "DENVER_INTERNET2_OSDF_CACHE", + ResourceFQDN: "dtn-pas.denv.nrp.internet2.edu", + StartTime: time.Now().Add(24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + EndTime: time.Now().Add(25 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + { + // end time is before current time. Should NOT be filtered + ResourceName: "HOW_MUCH_CASH_COULD_A_STASHCACHE_STASH", + ResourceFQDN: "stash-cache.cache.osdf.biz", + StartTime: time.Now().Add(-24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + EndTime: time.Now().Add(-1 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + { + // Invalid time should cause updateDowntimeFromTopology to log an error but not return one + ResourceName: "FOOBAR", + ResourceFQDN: "foo.bar", + StartTime: "The second of January, 2006 03:04 PM MST", + EndTime: time.Now().Add(1 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + }, + } - t.Run("two-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{mockTopoCacheB, mockTopoCacheC}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - require.Len(t, get, 2) - assert.EqualValues(t, mockTopoCacheA, get[0]) - assert.EqualValues(t, mockTopoCacheD, get[1]) - }) + tmpl, err := template.ParseFiles("resources/mock_topology_downtime_template.xml") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } - t.Run("all-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - assert.EqualValues(t, []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, get) - }) + w.Header().Set("Content-Type", "application/xml") + err = tmpl.Execute(w, downtimeInfo) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } func TestUpdateDowntimeFromTopology(t *testing.T) { - mockTopoCacheA := server_utils.Server{AuthEndpoint: "cacheA.org:8443", Endpoint: "cacheA.org:8000", Resource: "CACHE_A"} - mockTopoCacheB := server_utils.Server{AuthEndpoint: "cacheB.org:8443", Endpoint: "cacheB.org:8000", Resource: "CACHE_B"} - mockTopoCacheC := server_utils.Server{AuthEndpoint: "cacheC.org:8443", Endpoint: "cacheC.org:8000", Resource: "CACHE_C"} - - t.Run("no-change-with-same-downtime", func(t *testing.T) { - filteredServers = map[string]filterType{} - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - checkResult := func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 2) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - } - checkResult() - - // second round of updates - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - // Same result - checkResult() + // Create a buffer to capture log output + var logBuffer bytes.Buffer + originalOutput := logrus.StandardLogger().Out + logrus.SetOutput(&logBuffer) + t.Cleanup(func() { + logrus.SetOutput(originalOutput) }) - t.Run("one-server-back-online", func(t *testing.T) { - filteredServers = map[string]filterType{} - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 2) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - }() + server := httptest.NewServer(http.HandlerFunc(mockTopoDowntimeXMLHandler)) + t.Cleanup(func() { + server.Close() + }) + viper.Set("Federation.TopologyDowntimeUrl", server.URL) - // second round of updates - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA}}, // A is back online - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 1) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(func() { + cancel() }) - t.Run("one-more-server-in-downtime", func(t *testing.T) { - filteredServers = map[string]filterType{} - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 2) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - }() + err := updateDowntimeFromTopology(ctx) + if err != nil { + t.Fatalf("updateDowntimeFromTopology() error = %v", err) + } - // second round of updates - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{}}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC}}, - ) - - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 3) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheC.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheC.Resource]) - }() - }) + // There should be a logged warning about the invalid time + logOutput := logBuffer.String() + assert.Contains(t, logOutput, "Could not put FOOBAR into downtime because its start time") + + assert.True(t, filteredServers["BOISE_INTERNET2_OSDF_CACHE"] == topoFiltered) + _, keyExists := filteredServers["DENVER_INTERNET2_OSDF_CACHE"] + assert.False(t, keyExists, "DENVER_INTERNET2_OSDF_CACHE should not be in filteredServers") + _, keyExists = filteredServers["HOW_MUCH_CASH_COULD_A_STASHCACHE_STASH"] + assert.False(t, keyExists, "HOW_MUCH_CASH_COULD_A_STASHCACHE_STASH should not be in filteredServers") } diff --git a/director/resources/mock_topology_downtime_template.xml b/director/resources/mock_topology_downtime_template.xml new file mode 100644 index 000000000..679de818e --- /dev/null +++ b/director/resources/mock_topology_downtime_template.xml @@ -0,0 +1,30 @@ + + +{{- range .Downtimes }} + + 1890864242 + 1405 + + I2BoiseInfrastructure + 1338 + + {{ .ResourceName }} + {{ .ResourceFQDN }} + {{ .StartTime }} + {{ .EndTime }} + UNSCHEDULED + Outage + Aug 19, 2024 16:53 PM UTC + Not Available + + + 156 + XRootD cache server + Internet2 Boise Cache + + + HW issues + +{{- end }} + + diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 37c87417a..12bececec 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -348,6 +348,14 @@ osdf_default: https://topology.opensciencegrid.org/osdf/namespaces default: none components: ["director", "registry"] --- +name: Federation.TopologyDowntimeUrl +description: |+ + A URL for determining OSG topology server downtime information. The result of querying this URL is an XML file containing downtime information. +type: url +osdf_default: https://topology.opensciencegrid.org/rgdowntime/xml +default: none +components: ["director"] +--- name: Federation.TopologyReloadInterval description: |+ The frequency, in minutes, that topology should be reloaded. @@ -1366,8 +1374,8 @@ components: ["director"] --- name: Director.FilteredServers description: |+ - A list of server host names to not to redirect client requests to. This is for admins to put a list of - servers in the federation into downtime. + A list of server resource names that the Director should consider in downtime, preventing the Director from issuing redirects to them. + Additional downtimes are aggregated from Topology (when the Director is served in OSDF mode), and the Web UI. type: stringSlice default: none components: ["director"] diff --git a/param/parameters.go b/param/parameters.go index 82f8c6adb..3afc09f9d 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -154,6 +154,7 @@ var ( Director_SupportContactEmail = StringParam{"Director.SupportContactEmail"} Director_SupportContactUrl = StringParam{"Director.SupportContactUrl"} Federation_DiscoveryUrl = StringParam{"Federation.DiscoveryUrl"} + Federation_TopologyDowntimeUrl = StringParam{"Federation.TopologyDowntimeUrl"} Federation_TopologyNamespaceUrl = StringParam{"Federation.TopologyNamespaceUrl"} Federation_TopologyUrl = StringParam{"Federation.TopologyUrl"} IssuerKey = StringParam{"IssuerKey"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index ac4d51a58..32cc73682 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -88,6 +88,7 @@ type Config struct { DiscoveryUrl string `mapstructure:"discoveryurl"` JwkUrl string `mapstructure:"jwkurl"` RegistryUrl string `mapstructure:"registryurl"` + TopologyDowntimeUrl string `mapstructure:"topologydowntimeurl"` TopologyNamespaceUrl string `mapstructure:"topologynamespaceurl"` TopologyReloadInterval time.Duration `mapstructure:"topologyreloadinterval"` TopologyUrl string `mapstructure:"topologyurl"` @@ -383,6 +384,7 @@ type configWithType struct { DiscoveryUrl struct { Type string; Value string } JwkUrl struct { Type string; Value string } RegistryUrl struct { Type string; Value string } + TopologyDowntimeUrl struct { Type string; Value string } TopologyNamespaceUrl struct { Type string; Value string } TopologyReloadInterval struct { Type string; Value time.Duration } TopologyUrl struct { Type string; Value string } diff --git a/registry/registry_db.go b/registry/registry_db.go index 2296f23ee..3d8afb9bc 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -520,7 +520,7 @@ func PopulateTopology(ctx context.Context) error { } // Next, get the values from topology - namespaces, err := server_utils.GetTopologyJSON(ctx, false) + namespaces, err := server_utils.GetTopologyJSON(ctx) if err != nil { return errors.Wrapf(err, "Failed to get topology JSON") } diff --git a/server_structs/topology.go b/server_structs/topology.go new file mode 100644 index 000000000..bb1d4c9b5 --- /dev/null +++ b/server_structs/topology.go @@ -0,0 +1,84 @@ +package server_structs + +import ( + "encoding/xml" +) + +type ( + TopoServer struct { + AuthEndpoint string `json:"auth_endpoint"` + Endpoint string `json:"endpoint"` + Resource string `json:"resource"` + } + + TopoScitokens struct { + BasePath []string `json:"base_path"` + Issuer string `json:"issuer"` + Restricted []string `json:"restricted_path"` + } + + TopoCredentialGeneration struct { + BasePath string `json:"base_path"` + Issuer string `json:"issuer"` + MaxScopeDepth int `json:"max_scope_depth"` + Strategy string `json:"strategy"` + VaultIssuer string `json:"vault_issuer"` + VaultServer string `json:"vault_server"` + } + + TopoNamespace struct { + Caches []TopoServer `json:"caches"` + Origins []TopoServer `json:"origins"` + CredentialGeneration TopoCredentialGeneration `json:"credential_generation"` + DirlistHost string `json:"dirlisthost"` + Path string `json:"path"` + ReadHTTPS bool `json:"readhttps"` + Scitokens []TopoScitokens `json:"scitokens"` + UseTokenOnRead bool `json:"usetokenonread"` + WritebackHost string `json:"writebackhost"` + } + + TopologyNamespacesJSON struct { + Caches []TopoServer `json:"caches"` + Namespaces []TopoNamespace `json:"namespaces"` + } + + // Structs for encoding downtimes + + TopoResourceGroup struct { + GroupName string `xml:"GroupName"` + GroupID int `xml:"GroupID"` + } + + TopoServices struct { + Service []TopoService `xml:"Service"` + } + + TopoService struct { + ID int `xml:"ID"` + Name string `xml:"Name"` + Description string `xml:"Description"` + } + + TopoDowntimeInfo struct { + XMLName xml.Name `xml:"Downtimes"` + CurrentDowntimes TopoCurrentDowntimes `xml:"CurrentDowntimes"` + } + + TopoCurrentDowntimes struct { + Downtimes []TopoServerDowntime `xml:"Downtime"` + } + + TopoServerDowntime struct { + ID int `xml:"ID"` + ResourceGroup TopoResourceGroup `xml:"ResourceGroup"` + ResourceName string `xml:"ResourceName"` + ResourceFQDN string `xml:"ResourceFQDN"` + StartTime string `xml:"StartTime"` + EndTime string `xml:"EndTime"` + CreatedTime string `xml:"CreatedTime"` + UpdateTime string `xml:"UpdateTime"` + Services TopoServices `xml:"Services"` + Description string `xml:"Description"` + } +) diff --git a/server_utils/server_utils.go b/server_utils/server_utils.go index e13e91f5f..45447d4c9 100644 --- a/server_utils/server_utils.go +++ b/server_utils/server_utils.go @@ -41,50 +41,11 @@ import ( "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" -) - -type ( - Server struct { - AuthEndpoint string `json:"auth_endpoint"` - Endpoint string `json:"endpoint"` - Resource string `json:"resource"` - } - - Scitokens struct { - BasePath []string `json:"base_path"` - Issuer string `json:"issuer"` - Restricted []string `json:"restricted_path"` - } - - CredentialGeneration struct { - BasePath string `json:"base_path"` - Issuer string `json:"issuer"` - MaxScopeDepth int `json:"max_scope_depth"` - Strategy string `json:"strategy"` - VaultIssuer string `json:"vault_issuer"` - VaultServer string `json:"vault_server"` - } - - Namespace struct { - Caches []Server `json:"caches"` - Origins []Server `json:"origins"` - CredentialGeneration CredentialGeneration `json:"credential_generation"` - DirlistHost string `json:"dirlisthost"` - Path string `json:"path"` - ReadHTTPS bool `json:"readhttps"` - Scitokens []Scitokens `json:"scitokens"` - UseTokenOnRead bool `json:"usetokenonread"` - WritebackHost string `json:"writebackhost"` - } - - TopologyNamespacesJSON struct { - Caches []Server `json:"caches"` - Namespaces []Namespace `json:"namespaces"` - } + "github.com/pelicanplatform/pelican/server_structs" ) // GetTopologyJSON returns the namespaces and caches from OSDF topology -func GetTopologyJSON(ctx context.Context, includeDowned bool) (*TopologyNamespacesJSON, error) { +func GetTopologyJSON(ctx context.Context) (*server_structs.TopologyNamespacesJSON, error) { topoNamespaceUrl := param.Federation_TopologyNamespaceUrl.GetString() if topoNamespaceUrl == "" { metrics.SetComponentHealthStatus(metrics.DirectorRegistry_Topology, metrics.StatusCritical, "Topology namespaces.json configuration option (`Federation.TopologyNamespaceURL`) not set") @@ -100,9 +61,6 @@ func GetTopologyJSON(ctx context.Context, includeDowned bool) (*TopologyNamespac req.Header.Set("Accept", "application/json") q := req.URL.Query() - if includeDowned { - q.Add("include_downed", "1") - } req.URL.RawQuery = q.Encode() // Use the transport to include timeouts @@ -125,7 +83,7 @@ func GetTopologyJSON(ctx context.Context, includeDowned bool) (*TopologyNamespac return nil, errors.Wrap(err, "Failure when reading OSDF namespace response") } - var namespaces TopologyNamespacesJSON + var namespaces server_structs.TopologyNamespacesJSON if err = json.Unmarshal(respBytes, &namespaces); err != nil { metrics.SetComponentHealthStatus(metrics.DirectorRegistry_Topology, metrics.StatusCritical, fmt.Sprintf("Failure when parsing JSON response from topology URL %v", topoNamespaceUrl)) return nil, errors.Wrapf(err, "Failure when parsing JSON response from topology URL %v", topoNamespaceUrl)