Skip to content

Commit

Permalink
fix(publisher): control nightly interval from configuration file (#186)
Browse files Browse the repository at this point in the history 
Signed-off-by: wuhuizuo <[email protected]>

---------

Signed-off-by: wuhuizuo <[email protected]>
  • Loading branch information
@wuhuizuo
wuhuizuo authored Oct 28, 2024
1 parent e877e83 commit 2e1e85d
Show file tree
Hide file tree
Showing 15 changed files with 118 additions and 84 deletions.
25 changes: 25 additions & 0 deletions publisher/cmd/publisher/init.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package main

import (
"fmt"
"os"

"gopkg.in/yaml.v3"

"github.com/PingCAP-QE/ee-apps/publisher/pkg/config"
)

// Load and parse configuration
func loadConfig(configFile string) (config.Service, error) {
var config config.Service
{
configData, err := os.ReadFile(configFile)
if err != nil {
return config, fmt.Errorf("error reading config file: %v", err)
}
if err := yaml.Unmarshal(configData, &config); err != nil {
return config, fmt.Errorf("error parsing config file: %v", err)
}
}
return config, nil
}
67 changes: 28 additions & 39 deletions publisher/cmd/publisher/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ import (
"github.com/segmentio/kafka-go"
"goa.design/clue/debug"
"goa.design/clue/log"
"gopkg.in/yaml.v3"

gentiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup"
"github.com/PingCAP-QE/ee-apps/publisher/pkg/config"
"github.com/PingCAP-QE/ee-apps/publisher/pkg/impl/tiup"
)

Expand Down Expand Up @@ -54,11 +52,36 @@ func main() {
logLevel = zerolog.DebugLevel
}
zerolog.SetGlobalLevel(logLevel)
logger := zerolog.New(os.Stderr).With().Timestamp().Str("service", gentiup.ServiceName).Logger()

// Initialize the services.
tiupSvc, err := initTiupService(*configFile)
// Load and parse configuration
config, err := loadConfig(*configFile)
if err != nil {
log.Fatalf(ctx, err, "failed to initialize service")
log.Fatalf(ctx, err, "failed to load configuration")
}

// Initialize the services.
var (
tiupSvc gentiup.Service
)
{
// Configure Kafka kafkaWriter
kafkaWriter := kafka.NewWriter(kafka.WriterConfig{
Brokers: config.Kafka.Brokers,
Topic: config.Kafka.Topic,
Balancer: &kafka.LeastBytes{},
Logger: kafka.LoggerFunc(logger.Printf),
})

// Configure Redis client
redisClient := redis.NewClient(&redis.Options{
Addr: config.Redis.Addr,
Password: config.Redis.Password,
Username: config.Redis.Username,
DB: config.Redis.DB,
})

tiupSvc = tiup.NewService(&logger, kafkaWriter, redisClient, config.EventSource)
}

// Wrap the services in endpoints that can be invoked from other services
Expand Down Expand Up @@ -127,37 +150,3 @@ func main() {
wg.Wait()
log.Printf(ctx, "exited")
}

func initTiupService(configFile string) (gentiup.Service, error) {
// Load and parse configuration
var config config.Service
{
configData, err := os.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("error reading config file: %v", err)
}
if err := yaml.Unmarshal(configData, &config); err != nil {
return nil, fmt.Errorf("error parsing config file: %v", err)
}
}

logger := zerolog.New(os.Stderr).With().Timestamp().Str("service", gentiup.ServiceName).Logger()

// Configure Kafka kafkaWriter
kafkaWriter := kafka.NewWriter(kafka.WriterConfig{
Brokers: config.Kafka.Brokers,
Topic: config.Kafka.Topic,
Balancer: &kafka.LeastBytes{},
Logger: kafka.LoggerFunc(logger.Printf),
})

// Configure Redis client
redisClient := redis.NewClient(&redis.Options{
Addr: config.Redis.Addr,
Password: config.Redis.Password,
Username: config.Redis.Username,
DB: config.Redis.DB,
})

return tiup.NewTiup(&logger, kafkaWriter, redisClient, config.EventSource), nil
}
11 changes: 10 additions & 1 deletion publisher/cmd/worker/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os/signal"
"sync"
"syscall"
"time"

"github.com/cloudevents/sdk-go/v2/event"
"github.com/go-redis/redis/v8"
Expand Down Expand Up @@ -62,7 +63,15 @@ func main() {
var handler *tiup.Publisher
{
var err error
handler, err = tiup.NewPublisher(config.MirrorUrl, config.LarkWebhookURL, &log.Logger, redisClient)
nigthlyInterval, err := time.ParseDuration(config.Options.NightlyInterval)
if err != nil {
log.Fatal().Err(err).Msg("Error parsing nightly interval")
}
handler, err = tiup.NewPublisher(&log.Logger, redisClient, tiup.PublisherOptions{
MirrorURL: config.Options.MirrorURL,
LarkWebhookURL: config.Options.LarkWebhookURL,
NightlyInterval: nigthlyInterval,
})
if err != nil {
log.Fatal().Err(err).Msg("Error creating handler")
}
Expand Down
4 changes: 2 additions & 2 deletions publisher/design/design.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ package design
import (
_ "goa.design/plugins/v3/zerologger"

. "goa.design/goa/v3/dsl"
. "goa.design/goa/v3/dsl" //nolint
)

var _ = API("publisher", func() {
Expand Down Expand Up @@ -57,7 +57,7 @@ var _ = Service("tiup", func() {
Required("request_id")
})
Result(String, "request state", func() {
Enum("queued", "processing", "success", "failed")
Enum("queued", "processing", "success", "failed", "canceled")
})
HTTP(func() {
GET("/publish-request/{request_id}")
Expand Down
6 changes: 4 additions & 2 deletions publisher/example/config/worker-example.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@ kafka:
- example-bootstrap.kafka:9092
topic: example-topic
consumer_group: example-group
mirror_url: http://tiup.mirror.site
lark_webhook_url: https://feishu.custom-bot-webhook # create and copy the url then paste here.
options:
mirror_url: http://tiup.mirror.site
lark_webhook_url: https://feishu.custom-bot-webhook # create and copy the url then paste here.
nightly_interval: 1h
2 changes: 1 addition & 1 deletion publisher/gen/http/openapi.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"swagger":"2.0","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","email":"[email protected]","url":"https://github.com/wuhuizuo"},"version":"1.0.0"},"host":"0.0.0.0:80","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","parameters":[{"name":"Request-To-PublishRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/TiupRequestToPublishRequestBody","required":["artifact_url","tiup-mirror"]}}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Cupiditate suscipit hic quidem voluptates nostrum necessitatibus."}}}},"schemes":["http"]}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"string","enum":["queued","processing","success","failed"]}}},"schemes":["http"]}}},"definitions":{"TiupRequestToPublishRequestBody":{"title":"TiupRequestToPublishRequestBody","type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Optio necessitatibus ipsa incidunt."},"request_id":{"type":"string","description":"The request id","example":"Et ullam."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Illo harum quis voluptatem."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Expedita et necessitatibus ut molestias."}},"example":{"artifact_url":"Et suscipit et.","request_id":"Corrupti ut natus aut ipsam reprehenderit.","tiup-mirror":"Assumenda tempora autem accusantium est.","version":"Ab expedita repellendus."},"required":["artifact_url","tiup-mirror"]}}}
{"swagger":"2.0","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","email":"[email protected]","url":"https://github.com/wuhuizuo"},"version":"1.0.0"},"host":"0.0.0.0:80","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","parameters":[{"name":"Request-To-PublishRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/TiupRequestToPublishRequestBody","required":["artifact_url","tiup-mirror"]}}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Cupiditate suscipit hic quidem voluptates nostrum necessitatibus."}}}},"schemes":["http"]}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"string","enum":["queued","processing","success","failed","canceled"]}}},"schemes":["http"]}}},"definitions":{"TiupRequestToPublishRequestBody":{"title":"TiupRequestToPublishRequestBody","type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Optio necessitatibus ipsa incidunt."},"request_id":{"type":"string","description":"The request id","example":"Et ullam."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Illo harum quis voluptatem."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Expedita et necessitatibus ut molestias."}},"example":{"artifact_url":"Et suscipit et.","request_id":"Corrupti ut natus aut ipsam reprehenderit.","tiup-mirror":"Assumenda tempora autem accusantium est.","version":"Ab expedita repellendus."},"required":["artifact_url","tiup-mirror"]}}}
1 change: 1 addition & 0 deletions publisher/gen/http/openapi.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ paths:
- processing
- success
- failed
- canceled
schemes:
- http
definitions:
Expand Down
2 changes: 1 addition & 1 deletion publisher/gen/http/openapi3.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"openapi":"3.0.3","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","url":"https://github.com/wuhuizuo","email":"[email protected]"},"version":"1.0.0"},"servers":[{"url":"http://0.0.0.0:80"}],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestToPublishRequestBody"},"example":{"artifact_url":"Omnis expedita.","request_id":"Est sequi placeat.","tiup-mirror":"Pariatur rerum consectetur deleniti architecto sunt.","version":"Dicta id perferendis rem a."}}}},"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Eum saepe nihil omnis dolorem eveniet."},"description":"request track ids","example":["Rerum voluptas dolore.","Eveniet vero voluptas.","Voluptates voluptatem accusamus nisi omnis quia molestias."]},"example":["Pariatur consequuntur itaque est.","Exercitationem atque amet optio."]}}}}}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"schema":{"type":"string","description":"request track id","example":"Aliquam ut laborum nulla."},"example":"Error qui dolores et."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"string","description":"request state","example":"queued","enum":["queued","processing","success","failed"]},"example":"failed"}}}}}}},"components":{"schemas":{"RequestToPublishRequestBody":{"type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Et eum odit eos ratione."},"request_id":{"type":"string","description":"The request id","example":"Voluptas ratione hic libero nisi."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Rerum tempore voluptas."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Laboriosam optio omnis cupiditate magnam nisi."}},"example":{"artifact_url":"Rerum voluptate accusantium optio.","request_id":"Atque vero in molestiae odit consequatur.","tiup-mirror":"Quibusdam nisi quam.","version":"Magni quia adipisci excepturi."},"required":["artifact_url","tiup-mirror"]}}},"tags":[{"name":"tiup","description":"TiUP Publisher service"}]}
{"openapi":"3.0.3","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","url":"https://github.com/wuhuizuo","email":"[email protected]"},"version":"1.0.0"},"servers":[{"url":"http://0.0.0.0:80"}],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestToPublishRequestBody"},"example":{"artifact_url":"Omnis expedita.","request_id":"Est sequi placeat.","tiup-mirror":"Pariatur rerum consectetur deleniti architecto sunt.","version":"Dicta id perferendis rem a."}}}},"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Eum saepe nihil omnis dolorem eveniet."},"description":"request track ids","example":["Rerum voluptas dolore.","Eveniet vero voluptas.","Voluptates voluptatem accusamus nisi omnis quia molestias."]},"example":["Pariatur consequuntur itaque est.","Exercitationem atque amet optio."]}}}}}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"schema":{"type":"string","description":"request track id","example":"Aliquam ut laborum nulla."},"example":"Error qui dolores et."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"string","description":"request state","example":"processing","enum":["queued","processing","success","failed","canceled"]},"example":"queued"}}}}}}},"components":{"schemas":{"RequestToPublishRequestBody":{"type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Et eum odit eos ratione."},"request_id":{"type":"string","description":"The request id","example":"Voluptas ratione hic libero nisi."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Rerum tempore voluptas."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Laboriosam optio omnis cupiditate magnam nisi."}},"example":{"artifact_url":"Rerum voluptate accusantium optio.","request_id":"Atque vero in molestiae odit consequatur.","tiup-mirror":"Quibusdam nisi quam.","version":"Magni quia adipisci excepturi."},"required":["artifact_url","tiup-mirror"]}}},"tags":[{"name":"tiup","description":"TiUP Publisher service"}]}
5 changes: 3 additions & 2 deletions publisher/gen/http/openapi3.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,14 @@ paths:
schema:
type: string
description: request state
example: queued
example: processing
enum:
- queued
- processing
- success
- failed
example: failed
- canceled
example: queued
components:
schemas:
RequestToPublishRequestBody:
Expand Down
4 changes: 2 additions & 2 deletions publisher/gen/http/tiup/client/encode_decode.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 6 additions & 3 deletions publisher/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ type Worker struct {
KafkaBasic `yaml:",inline" json:",inline"`
ConsumerGroup string `yaml:"consumer_group" json:"consumer_group,omitempty"`
} `yaml:"kafka" json:"kafka,omitempty"`
Redis Redis `yaml:"redis" json:"redis,omitempty"`
MirrorUrl string `yaml:"mirror_url" json:"mirror_url,omitempty"`
LarkWebhookURL string `yaml:"lark_webhook_url" json:"lark_webhook_url,omitempty"`
Redis Redis `yaml:"redis" json:"redis,omitempty"`
Options struct {
MirrorURL string `yaml:"mirror_url" json:"mirror_url,omitempty"`
LarkWebhookURL string `yaml:"lark_webhook_url" json:"lark_webhook_url,omitempty"`
NightlyInterval string `yaml:"nightly_interval" json:"nightly_interval,omitempty"`
}
}

type Service struct {
Expand Down
2 changes: 1 addition & 1 deletion publisher/pkg/impl/tiup/funcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ func AnalyzeFromOciArtifact(repo, tag string) ([]PublishRequest, error) {
return publishRequests, nil
}

func AnalyzeFromOciArtifactUrl(url string) ([]PublishRequest, error) {
func analyzeFromOciArtifactUrl(url string) ([]PublishRequest, error) {
repo, tag, err := splitRepoAndTag(url)
if err != nil {
return nil, err
Expand Down
Loading

0 comments on commit 2e1e85d

Please sign in to comment.