Merge pull request #3026 from rbeuque74/ovhcloud-cache

feat(ovh): add cache based on DNS zone SOA value

docs/tutorials/ovh.md

@@ -18,14 +18,15 @@ instructions for creating a zone.
 
 You first need to create an OVH application.
 
-Using the [OVH documentation](https://docs.ovh.com/gb/en/customer/first-steps-with-ovh-api/#creation-of-your-application-keys) you will have your `Application key` and `Application secret`
+Using the [OVH documentation](https://docs.ovh.com/gb/en/api/first-steps-with-ovh-api/#advanced-usage-pair-ovhcloud-apis-with-an-application_2) you will have your `Application key` and `Application secret`
 
 And you will need to generate your consumer key, here the permissions needed :
 - GET on `/domain/zone`
 - GET on `/domain/zone/*/record`
 - GET on `/domain/zone/*/record/*`
 - POST on `/domain/zone/*/record`
 - DELETE on `/domain/zone/*/record/*`
+- GET on `/domain/zone/*/soa`
 - POST on `/domain/zone/*/refresh`
 
 You can use the following `curl` request to generate & validated your `Consumer key`
@@ -37,6 +38,10 @@ curl -XPOST -H "X-Ovh-Application: <ApplicationKey>" -H "Content-type: applicati
       "method": "GET",
       "path": "/domain/zone"
     },
+    {
+      "method": "GET",
+      "path": "/domain/zone/*/soa"
+    },
     {
       "method": "GET",
       "path": "/domain/zone/*/record"

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/openshift/client-go v0.0.0-20230607134213-3cd0021bbee3
 	github.com/oracle/oci-go-sdk/v65 v65.45.0
 	github.com/ovh/go-ovh v1.4.1
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pluralsh/gqlclient v1.6.0
 	github.com/projectcontour/contour v1.25.2
@@ -157,7 +158,6 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/openshift/gssapi v0.0.0-20161010215902-5fb4217df13b // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/peterhellberg/link v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

provider/ovh/ovh.go

@@ -21,12 +21,16 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
+	"github.com/miekg/dns"
 	"github.com/ovh/go-ovh/ovh"
+	"github.com/patrickmn/go-cache"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
 	"sigs.k8s.io/external-dns/endpoint"
+	"sigs.k8s.io/external-dns/pkg/apis/externaldns"
 	"sigs.k8s.io/external-dns/plan"
 	"sigs.k8s.io/external-dns/provider"
 
@@ -56,6 +60,17 @@ type OVHProvider struct {
 
 	domainFilter endpoint.DomainFilter
 	DryRun       bool
+
+	// UseCache controls if the OVHProvider will cache records in memory, and serve them
+	// without recontacting the OVHcloud API if the SOA of the domain zone hasn't changed.
+	// Note that, when disabling cache, OVHcloud API has rate-limiting that will hit if
+	// your refresh rate/number of records is too big, which might cause issue with the
+	// provider.
+	// Default value: true
+	UseCache bool
+
+	cacheInstance *cache.Cache
+	dnsClient     dnsClient
 }
 
 type ovhClient interface {
@@ -64,6 +79,10 @@ type ovhClient interface {
 	Delete(string, interface{}) error
 }
 
+type dnsClient interface {
+	ExchangeContext(ctx context.Context, m *dns.Msg, a string) (*dns.Msg, time.Duration, error)
+}
+
 type ovhRecordFields struct {
 	FieldType string `json:"fieldType"`
 	SubDomain string `json:"subDomain"`
@@ -88,6 +107,9 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
 	if err != nil {
 		return nil, err
 	}
+
+	client.UserAgent = externaldns.Version
+
 	// TODO: Add Dry Run support
 	if dryRun {
 		return nil, ErrNoDryRun
@@ -97,6 +119,9 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
 		domainFilter:   domainFilter,
 		apiRateLimiter: ratelimit.New(apiRateLimit),
 		DryRun:         dryRun,
+		cacheInstance:  cache.New(cache.NoExpiration, cache.NoExpiration),
+		dnsClient:      new(dns.Client),
+		UseCache:       true,
 	}, nil
 }
 
@@ -217,14 +242,48 @@ func (p *OVHProvider) zones() ([]string, error) {
 	return filteredZones, nil
 }
 
+type ovhSoa struct {
+	Server  string `json:"server"`
+	Serial  uint32 `json:"serial"`
+	records []ovhRecord
+}
+
 func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<- []ovhRecord) error {
 	var recordsIds []uint64
 	ovhRecords := make([]ovhRecord, len(recordsIds))
 	eg, _ := errgroup.WithContext(*ctx)
 
+	if p.UseCache {
+		if cachedSoaItf, ok := p.cacheInstance.Get(*zone + "#soa"); ok {
+			cachedSoa := cachedSoaItf.(ovhSoa)
+
+			m := new(dns.Msg)
+			m.SetQuestion(dns.Fqdn(*zone), dns.TypeSOA)
+			in, _, err := p.dnsClient.ExchangeContext(*ctx, m, strings.TrimSuffix(cachedSoa.Server, ".")+":53")
+			if err == nil {
+				if s, ok := in.Answer[0].(*dns.SOA); ok {
+					// do something with t.Txt
+					if s.Serial == cachedSoa.Serial {
+						records <- cachedSoa.records
+						return nil
+					}
+				}
+			}
+
+			p.cacheInstance.Delete(*zone + "#soa")
+		}
+	}
+
 	log.Debugf("OVH: Getting records for %s", *zone)
 
 	p.apiRateLimiter.Take()
+	var soa ovhSoa
+	if p.UseCache {
+		if err := p.client.Get("/domain/zone/"+*zone+"/soa", &soa); err != nil {
+			return err
+		}
+	}
+
 	if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record", *zone), &recordsIds); err != nil {
 		return err
 	}
@@ -240,6 +299,12 @@ func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<-
 	for record := range chRecords {
 		ovhRecords = append(ovhRecords, record)
 	}
+
+	if p.UseCache {
+		soa.records = ovhRecords
+		_ = p.cacheInstance.Add(*zone+"#soa", soa, time.Hour)
+	}
+
 	records <- ovhRecords
 	return nil
 }