Skip to content

Commit

Permalink
Merge branch 'release/1.11.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
Marianasilvago committed Jul 5, 2021
2 parents 88587dd + 1482c0a commit d36f51d
Show file tree
Hide file tree
Showing 18 changed files with 647 additions and 187 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@
*.iml
*.iws

/build
/build
vendor/
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ If on go 1.12 then ensure the project either resides outside of your `GOPATH` or
| HealthCheckInterval | Interval between health checks | 30 seconds |
| HealthCheckCriticalTimeout | Amount of time to pass since last healthy health check to be deemed a critical failure | 90 seconds |
| RendererURL | URl for `dp-frontend-renderer`
| CacheUpdateInterval | Duration for homepage cache updation
| IsPublishingMode | Mode in which service is running
| Languages | Languages which are supported separated by comma

### Contributing

Expand Down
6 changes: 6 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ type Config struct {
HealthCheckCriticalTimeout time.Duration `envconfig:"HEALTHCHECK_CRITICAL_TIMEOUT"`
RendererURL string `envconfig:"RENDERER_URL"`
BabbageURL string `envconfig:"BABBAGE_URL"`
CacheUpdateInterval time.Duration `envconfig:"CACHE_UPDATE_INTERVAL"`
IsPublishingMode bool `envconfig:"IS_PUBLISHING_MODE"`
Languages string `envconfig:"LANGUAGES"`
}

var cfg *Config
Expand All @@ -34,6 +37,9 @@ func Get() (*Config, error) {
HealthCheckCriticalTimeout: 90 * time.Second,
RendererURL: "http://localhost:20010",
BabbageURL: "http://localhost:8080",
CacheUpdateInterval: 10 * time.Second,
IsPublishingMode: false,
Languages: "en,cy",
}

return cfg, envconfig.Process("", cfg)
Expand Down
4 changes: 3 additions & 1 deletion config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ func TestSpec(t *testing.T) {
So(cfg.HealthCheckInterval, ShouldEqual, 30*time.Second)
So(cfg.HealthCheckCriticalTimeout, ShouldEqual, 90*time.Second)
So(cfg.RendererURL, ShouldEqual, "http://localhost:20010")
So(cfg.BabbageURL, ShouldEqual, "http://localhost:8080")
So(cfg.IsPublishingMode, ShouldEqual, false)
So(cfg.CacheUpdateInterval, ShouldEqual, 10*time.Second)
So(cfg.Languages, ShouldEqual, "en,cy")
})
})
})
Expand Down
6 changes: 6 additions & 0 deletions dpcache/constants.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package dpcache

const (
// HomepageCacheKey is used to cache the rendered homepage
HomepageCacheKey = "homepage-cache"
)
93 changes: 93 additions & 0 deletions dpcache/dpcache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package dpcache

import (
"context"
"fmt"
"github.com/ONSdigital/log.go/log"
"sync"
"time"
)

type DpCacher interface {
Close()
Get(key string) (interface{}, bool)
Set(key string, data interface{})
AddUpdateFunc(key string, updateFunc func() (string, error))
StartUpdates(ctx context.Context, channel chan error)
}

type DpCache struct {
cache sync.Map
updateInterval time.Duration
close chan struct{}
updateFuncs map[string]func() (string, error)
}

func (dc *DpCache) Get(key string) (interface{}, bool) {
return dc.cache.Load(key)
}

func (dc *DpCache) Set(key string, data interface{}) {
dc.cache.Store(key, data)
}

func (dc *DpCache) Close() {
dc.close <- struct{}{}
for key, _ := range dc.updateFuncs {
dc.cache.Store(key, "")
}
dc.updateFuncs = make(map[string]func() (string, error))
}

func NewDpCache(updateInterval time.Duration) DpCacher {
return &DpCache{
cache: sync.Map{},
updateInterval: updateInterval,
close: make(chan struct{}),
updateFuncs: make(map[string]func() (string, error)),
}
}

func (dc *DpCache) AddUpdateFunc(key string, updateFunc func() (string, error)) {
dc.updateFuncs[key] = updateFunc
}

func (dc *DpCache) UpdateContent(ctx context.Context) error {
for key, updateFunc := range dc.updateFuncs {
updatedContent, err := updateFunc()
if err != nil {
return fmt.Errorf("HOMEPAGE_CACHE_UPDATE_FAILED. failed to update homepage cache for %s. error: %v", key, err)
}
dc.Set(key, updatedContent)
}
return nil
}

func (dc *DpCache) StartUpdates(ctx context.Context, errorChannel chan error) {
ticker := time.NewTicker(dc.updateInterval)
if len(dc.updateFuncs) == 0 {
return
}

err := dc.UpdateContent(ctx)
if err != nil {
errorChannel <- err
dc.Close()
return
}

for {
select {
case <-ticker.C:
err := dc.UpdateContent(ctx)
if err != nil {
log.Event(ctx, err.Error(), log.Error(err), log.ERROR)
}

case <-dc.close:
return
case <-ctx.Done():
return
}
}
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/ONSdigital/dp-healthcheck v1.0.5
github.com/ONSdigital/dp-net v1.0.11
github.com/ONSdigital/log.go v1.0.1
github.com/ReneKroon/ttlcache v1.7.0
github.com/fatih/color v1.10.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 // indirect
Expand Down
16 changes: 12 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
github.com/ONSdigital/dp-api-clients-go v1.28.0 h1:ExIUlHC6uBdBlFwt/gAI0ApSzpyigy0NWJFK3XCwSVc=
github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4=
github.com/ONSdigital/dp-api-clients-go v1.33.0 h1:VVWJZSpmHOJvXUETwgAcFaizW6voxRU8RbsmPHma4GU=
github.com/ONSdigital/dp-api-clients-go v1.33.0/go.mod h1:0pUK3MN1v7DTjq0JSAD+DqbsZ8AVTodrXSXgJecg9Pw=
Expand All @@ -21,9 +20,13 @@ github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQ
github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc=
github.com/ONSdigital/log.go v1.0.1 h1:SZ5wRZAwlt2jQUZ9AUzBB/PL+iG15KapfQpJUdA18/4=
github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M=
github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk=
github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
Expand All @@ -32,7 +35,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NTyxftZHSnFaJmWE5oCCrj4DyEXkAiMa1iZJM=
github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s=
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
Expand All @@ -53,18 +55,24 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4=
go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
Expand Down
16 changes: 13 additions & 3 deletions homepage/clients.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,41 @@ package homepage

import (
"context"

"github.com/ONSdigital/dp-api-clients-go/image"
"github.com/ONSdigital/dp-api-clients-go/zebedee"
"github.com/ONSdigital/dp-frontend-homepage-controller/clients/release_calendar"
health "github.com/ONSdigital/dp-healthcheck/healthcheck"
)

//go:generate moq -out mocks_test.go -pkg homepage . ZebedeeClient RenderClient BabbageClient ImageClient

// ZebedeeClient is an interface with methods required for a zebedee client
type ZebedeeClient interface {
GetTimeseriesMainFigure(ctx context.Context, userAuthToken, collectionID, lang, uri string) (m zebedee.TimeseriesMainFigure, err error)
GetHomepageContent(ctx context.Context, userAccessToken, collectionID, lang, path string) (m zebedee.HomepageContent, err error)
Checker(ctx context.Context, check *health.CheckState) error
}

// BabbageClient is an interface with methods required for a babbage client
type BabbageClient interface {
GetReleaseCalendar(ctx context.Context, userAccessToken, fromDay, fromMonth, fromYear string) (m release_calendar.ReleaseCalendar, err error)
Checker(ctx context.Context, check *health.CheckState) error
}

// ImageClient is an interface with methods required for the Image API service client
type ImageClient interface {
GetDownloadVariant(ctx context.Context, userAuthToken, serviceAuthToken, collectionID, imageID, variant string) (m image.ImageDownload, err error)
Checker(ctx context.Context, check *health.CheckState) error
}

// RenderClient is an interface with methods required for rendering a template
type RenderClient interface {
Do(string, []byte) ([]byte, error)
Checker(ctx context.Context, check *health.CheckState) error
}

// Clients contains all the required Clients for frontend homepage controller
type Clients struct {
Renderer RenderClient
Zebedee ZebedeeClient
Babbage BabbageClient
ImageAPI ImageClient
}
93 changes: 8 additions & 85 deletions homepage/homepage.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,11 @@ package homepage

import (
"context"
"encoding/json"
"net/http"
"sync"
"time"

"github.com/ONSdigital/dp-api-clients-go/image"
"github.com/ONSdigital/dp-api-clients-go/zebedee"
"github.com/ONSdigital/dp-frontend-homepage-controller/mapper"
model "github.com/ONSdigital/dp-frontend-models/model/homepage"
dphandlers "github.com/ONSdigital/dp-net/handlers"
"github.com/ONSdigital/log.go/log"
)
Expand All @@ -34,100 +30,27 @@ type MainFigure struct {
var mainFigureMap map[string]MainFigure

// Handler handles requests to homepage endpoint
func Handler(rend RenderClient, zcli ZebedeeClient, bcli BabbageClient, icli ImageClient) http.HandlerFunc {
func Handler(homepageClient HomepageClienter) http.HandlerFunc {
return dphandlers.ControllerHandler(func(w http.ResponseWriter, r *http.Request, lang, collectionID, accessToken string) {
handle(w, r, rend, zcli, bcli, icli, accessToken, collectionID, lang)
handle(w, r, accessToken, collectionID, lang, homepageClient)
})

}

func handle(w http.ResponseWriter, req *http.Request, rend RenderClient, zcli ZebedeeClient, bcli BabbageClient, icli ImageClient, userAccessToken, collectionID, lang string) {
func handle(w http.ResponseWriter, req *http.Request, userAccessToken, collectionID, lang string, homepageClient HomepageClienter) {
ctx := req.Context()

mappedMainFigures := make(map[string]*model.MainFigure)
var wg sync.WaitGroup
responses := make(chan *model.MainFigure, len(mainFigureMap))
for id, figure := range mainFigureMap {
wg.Add(1)
go func(ctx context.Context, zcli ZebedeeClient, id string, figure MainFigure) {
defer wg.Done()
zebResponses := []zebedee.TimeseriesMainFigure{}
for _, uri := range figure.uris {
zebResponse, err := zcli.GetTimeseriesMainFigure(ctx, userAccessToken, collectionID, lang, uri)
if err != nil {
log.Event(ctx, "error getting timeseries data", log.ERROR, log.Error(err), log.Data{"timeseries-data": uri})
mappedErrorFigure := &model.MainFigure{ID: id}
responses <- mappedErrorFigure
return
}
zebResponses = append(zebResponses, zebResponse)
}
trendInfo := getTrendInfo(ctx, userAccessToken, collectionID, lang, zcli, figure)
latestMainFigure := getLatestTimeSeriesData(ctx, zebResponses)
mappedMainFigure := mapper.MainFigure(ctx, id, figure.datePeriod, figure.differenceInterval, latestMainFigure, trendInfo)
responses <- mappedMainFigure
return
}(ctx, zcli, id, figure)
}
wg.Wait()
close(responses)

for response := range responses {
log.Event(ctx, "the response of this request was", log.ERROR, log.Data{"response": response})
mappedMainFigures[response.ID] = response
}

weekAgoTime := time.Now().AddDate(0, 0, -7)
dateFromDay := weekAgoTime.Format("02")
dateFromMonth := weekAgoTime.Format("01")
dateFromYear := weekAgoTime.Format("2006")
releaseCalResp, err := bcli.GetReleaseCalendar(ctx, userAccessToken, dateFromDay, dateFromMonth, dateFromYear)
homepageHTML, err := homepageClient.GetHomePage(ctx, userAccessToken, collectionID, lang)
if err != nil {
log.Event(ctx, "error failed to get release calendar data from babbage ", log.ERROR, log.Error(err))
}
releaseCalModelData := mapper.ReleaseCalendar(releaseCalResp)

// Get homepage data from Zebedee
homepageContent, err := zcli.GetHomepageContent(ctx, userAccessToken, collectionID, lang, HomepagePath)
if err != nil {
log.Event(ctx, "error getting homepage data from client", log.ERROR, log.Error(err), log.Data{"content-path": HomepagePath})
}

var mappedFeaturedContent []model.Feature
if len(homepageContent.FeaturedContent) > 0 {
imageObjects := map[string]image.ImageDownload{}
for _, fc := range homepageContent.FeaturedContent {
if fc.ImageID != "" {
image, err := icli.GetDownloadVariant(ctx, userAccessToken, "", "", fc.ImageID, ImageVariant)
if err != nil {
log.Event(ctx, "error getting image download variant", log.ERROR, log.Error(err), log.Data{"featured-content-entry": fc.Title})
}
imageObjects[fc.ImageID] = image
}
}
mappedFeaturedContent = mapper.FeaturedContent(homepageContent, imageObjects)
}

m := mapper.Homepage(lang, mappedMainFigures, releaseCalModelData, &mappedFeaturedContent, homepageContent.ServiceMessage)

b, err := json.Marshal(m)
if err != nil {
log.Event(ctx, "error marshalling body data to json", log.ERROR, log.Error(err))
http.Error(w, "error marshalling body data to json", http.StatusInternalServerError)
log.Event(ctx, "HOMEPAGE_RESPONSE_FAILED. failed to get homepage html", log.ERROR, log.Error(err))
w.WriteHeader(http.StatusInternalServerError)
return
}

templateHTML, err := rend.Do("homepage", b)
if err != nil {
log.Event(ctx, "error rendering page", log.ERROR, log.Error(err))
http.Error(w, "error rendering page", http.StatusInternalServerError)
return
}
if _, err := w.Write(templateHTML); err != nil {
if _, err := w.Write([]byte(homepageHTML)); err != nil {
log.Event(ctx, "failed to write response for homepage", log.ERROR, log.Error(err))
w.WriteHeader(http.StatusInternalServerError)
return
}

return
}

Expand Down
Loading

0 comments on commit d36f51d

Please sign in to comment.