Skip to content

Commit

Permalink
feat(ovh): add cache based on DNS zone SOA value
Browse files Browse the repository at this point in the history
OVHcloud customers using 'ovh' provider for large DNS zone will spend a
lot of time waiting for all records to be retrieved from the API. As
records can't change without a modification of the SOA Serial of the DNS
zone, this commit adds a client-side cache that stores the value of the
DNS records for each zone, based of the current SOA Serial value of the
zone. In case the SOA Serial of the zone is identical to the one in
cache, we can serve all the records directly from cache. Otherwise,
fallback on refreshing the cache from the API.

Signed-off-by: Romain Beuque <[email protected]>
  • Loading branch information
rbeuque74 committed Jul 17, 2023
1 parent 22da9f2 commit 1a48cb8
Show file tree
Hide file tree
Showing 4 changed files with 214 additions and 7 deletions.
7 changes: 6 additions & 1 deletion docs/tutorials/ovh.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,15 @@ instructions for creating a zone.

You first need to create an OVH application.

Using the [OVH documentation](https://docs.ovh.com/gb/en/customer/first-steps-with-ovh-api/#creation-of-your-application-keys) you will have your `Application key` and `Application secret`
Using the [OVH documentation](https://docs.ovh.com/gb/en/api/first-steps-with-ovh-api/#advanced-usage-pair-ovhcloud-apis-with-an-application_2) you will have your `Application key` and `Application secret`

And you will need to generate your consumer key, here the permissions needed :
- GET on `/domain/zone`
- GET on `/domain/zone/*/record`
- GET on `/domain/zone/*/record/*`
- POST on `/domain/zone/*/record`
- DELETE on `/domain/zone/*/record/*`
- GET on `/domain/zone/*/soa`
- POST on `/domain/zone/*/refresh`

You can use the following `curl` request to generate & validated your `Consumer key`
Expand All @@ -37,6 +38,10 @@ curl -XPOST -H "X-Ovh-Application: <ApplicationKey>" -H "Content-type: applicati
"method": "GET",
"path": "/domain/zone"
},
{
"method": "GET",
"path": "/domain/zone/*/soa"
},
{
"method": "GET",
"path": "/domain/zone/*/record"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ require (
github.com/openshift/client-go v0.0.0-20230607134213-3cd0021bbee3
github.com/oracle/oci-go-sdk/v65 v65.41.0
github.com/ovh/go-ovh v1.4.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/pluralsh/gqlclient v1.5.1
github.com/projectcontour/contour v1.25.0
Expand Down Expand Up @@ -157,7 +158,6 @@ require (
github.com/oklog/ulid v1.3.1 // indirect
github.com/openshift/gssapi v0.0.0-20161010215902-5fb4217df13b // indirect
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/peterhellberg/link v1.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand Down
65 changes: 65 additions & 0 deletions provider/ovh/ovh.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@ import (
"errors"
"fmt"
"strings"
"time"

"github.com/miekg/dns"
"github.com/ovh/go-ovh/ovh"
"github.com/patrickmn/go-cache"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"

"sigs.k8s.io/external-dns/endpoint"
"sigs.k8s.io/external-dns/pkg/apis/externaldns"
"sigs.k8s.io/external-dns/plan"
"sigs.k8s.io/external-dns/provider"

Expand Down Expand Up @@ -56,6 +60,17 @@ type OVHProvider struct {

domainFilter endpoint.DomainFilter
DryRun bool

// UseCache controls if the OVHProvider will cache records in memory, and serve them
// without recontacting the OVHcloud API if the SOA of the domain zone hasn't changed.
// Note that, when disabling cache, OVHcloud API has rate-limiting that will hit if
// your refresh rate/number of records is too big, which might cause issue with the
// provider.
// Default value: true
UseCache bool

cacheInstance *cache.Cache
dnsClient dnsClient
}

type ovhClient interface {
Expand All @@ -64,6 +79,10 @@ type ovhClient interface {
Delete(string, interface{}) error
}

type dnsClient interface {
ExchangeContext(ctx context.Context, m *dns.Msg, a string) (*dns.Msg, time.Duration, error)
}

type ovhRecordFields struct {
FieldType string `json:"fieldType"`
SubDomain string `json:"subDomain"`
Expand All @@ -88,6 +107,9 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
if err != nil {
return nil, err
}

client.UserAgent = externaldns.Version

// TODO: Add Dry Run support
if dryRun {
return nil, ErrNoDryRun
Expand All @@ -97,6 +119,9 @@ func NewOVHProvider(ctx context.Context, domainFilter endpoint.DomainFilter, end
domainFilter: domainFilter,
apiRateLimiter: ratelimit.New(apiRateLimit),
DryRun: dryRun,
cacheInstance: cache.New(cache.NoExpiration, cache.NoExpiration),
dnsClient: new(dns.Client),
UseCache: true,
}, nil
}

Expand Down Expand Up @@ -217,14 +242,48 @@ func (p *OVHProvider) zones() ([]string, error) {
return filteredZones, nil
}

type ovhSoa struct {
Server string `json:"server"`
Serial uint32 `json:"serial"`
records []ovhRecord
}

func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<- []ovhRecord) error {
var recordsIds []uint64
ovhRecords := make([]ovhRecord, len(recordsIds))
eg, _ := errgroup.WithContext(*ctx)

if p.UseCache {
if cachedSoaItf, ok := p.cacheInstance.Get(*zone + "#soa"); ok {
cachedSoa := cachedSoaItf.(ovhSoa)

m := new(dns.Msg)
m.SetQuestion(dns.Fqdn(*zone), dns.TypeSOA)
in, _, err := p.dnsClient.ExchangeContext(*ctx, m, strings.TrimSuffix(cachedSoa.Server, ".")+":53")
if err == nil {
if s, ok := in.Answer[0].(*dns.SOA); ok {
// do something with t.Txt
if s.Serial == cachedSoa.Serial {
records <- cachedSoa.records
return nil
}
}
}

p.cacheInstance.Delete(*zone + "#soa")
}
}

log.Debugf("OVH: Getting records for %s", *zone)

p.apiRateLimiter.Take()
var soa ovhSoa
if p.UseCache {
if err := p.client.Get("/domain/zone/"+*zone+"/soa", &soa); err != nil {
return err
}
}

if err := p.client.Get(fmt.Sprintf("/domain/zone/%s/record", *zone), &recordsIds); err != nil {
return err
}
Expand All @@ -240,6 +299,12 @@ func (p *OVHProvider) records(ctx *context.Context, zone *string, records chan<-
for record := range chRecords {
ovhRecords = append(ovhRecords, record)
}

if p.UseCache {
soa.records = ovhRecords
_ = p.cacheInstance.Add(*zone+"#soa", soa, time.Hour)
}

records <- ovhRecords
return nil
}
Expand Down
Loading

0 comments on commit 1a48cb8

Please sign in to comment.