diff --git a/publisher/pkg/impl/fileserver_service.go b/publisher/pkg/impl/fileserver_service.go index 04cca9b..e9e2cae 100644 --- a/publisher/pkg/impl/fileserver_service.go +++ b/publisher/pkg/impl/fileserver_service.go @@ -39,13 +39,13 @@ func NewFileserver(logger *zerolog.Logger, kafkaWriter *kafka.Writer, redisClien func (s *fileserversrvc) RequestToPublish(ctx context.Context, p *fileserver.RequestToPublishPayload) (res []string, err error) { s.logger.Info().Msgf("fileserver.request-to-publish") // 1. Analyze the artifact_url to get the repo and tag and the tiup package information. - publishRequests, err := analyzeFsFromOciArtifactUrl(p.ArtifactURL) + publishRequest, err := analyzeFsFromOciArtifactUrl(p.ArtifactURL) if err != nil { return nil, err } // 2. Compose cloud events with the analyzed results. - events := s.composeEvents(publishRequests) + events := s.composeEvents(publishRequest) // 3. Send it to kafka topic with the request id as key and the event as value. var messages []kafka.Message @@ -93,17 +93,15 @@ func (s *fileserversrvc) QueryPublishingStatus(ctx context.Context, p *fileserve return status, nil } -func (s *fileserversrvc) composeEvents(requests []PublishRequest) []cloudevents.Event { +func (s *fileserversrvc) composeEvents(request *PublishRequestFS) []cloudevents.Event { var ret []cloudevents.Event - for _, request := range requests { - event := cloudevents.NewEvent() - event.SetID(uuid.New().String()) - event.SetType(EventTypeFsPublishRequest) - event.SetSource(s.eventSource) - event.SetSubject(request.Publish.Name) - event.SetData(cloudevents.ApplicationJSON, request) - ret = append(ret, event) - } + event := cloudevents.NewEvent() + event.SetID(uuid.New().String()) + event.SetType(EventTypeFsPublishRequest) + event.SetSource(s.eventSource) + event.SetSubject(request.Publish.Repo) + event.SetData(cloudevents.ApplicationJSON, request) + ret = append(ret, event) return ret } diff --git a/publisher/pkg/impl/fileserver_worker.go b/publisher/pkg/impl/fileserver_worker.go index 7a1fc64..364c8c4 100644 --- a/publisher/pkg/impl/fileserver_worker.go +++ b/publisher/pkg/impl/fileserver_worker.go @@ -72,7 +72,7 @@ func (p *fsWorker) Handle(event cloudevents.Event) cloudevents.Result { } p.redisClient.SetXX(context.Background(), event.ID(), PublishStateProcessing, redis.KeepTTL) - data := new(PublishRequest) + data := new(PublishRequestFS) if err := event.DataAs(&data); err != nil { return cloudevents.NewReceipt(false, "invalid data: %v", err) } @@ -91,32 +91,57 @@ func (p *fsWorker) Handle(event cloudevents.Event) cloudevents.Result { return result } -func (p *fsWorker) handle(data *PublishRequest) cloudevents.Result { - // 1. get the the file content from data.From. - err := doWithOCIFile(data.From.Oci, func(input io.Reader) error { - return doWithTempFileFromReader(input, func(inputF *os.File) error { - // 2. publish the tarball. - return p.publish(inputF, &data.Publish) +func (p *fsWorker) handle(data *PublishRequestFS) cloudevents.Result { + // 1. upload all the tarballs + for fromFile, targetKey := range targetFsFullPaths(&data.Publish) { + from := *data.From.Oci + from.File = fromFile + err := doWithOCIFile(&from, func(input io.Reader) error { + return doWithTempFileFromReader(input, func(inputF *os.File) error { + // 2. publish the tarball. + return p.publish(inputF, targetKey) + }) }) + if err != nil { + p.logger. + Err(err). + Str("bucket", p.options.S3.BucketName). + Str("key", targetKey). + Msg("failed to upload file to KS3 bucket.") + return err + } + } + + // 2. update git ref sha: download/refs///sha1 + refKV := targetFsRefKeyValue(&data.Publish) + _, err := p.s3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(p.options.S3.BucketName), + Key: aws.String(refKV[0]), + Body: bytes.NewReader([]byte(refKV[1])), }) if err != nil { - p.logger.Err(err).Msg("publish to fileserver failed") + p.logger. + Err(err). + Str("bucket", p.options.S3.BucketName). + Str("key", refKV[0]). + Msg("failed to update content in KS3 bucket.") return cloudevents.NewReceipt(false, "publish to fileserver failed: %v", err) } - p.logger.Info().Msg("publish to fileserver success") + p.logger.Debug().Str("bucket", p.options.S3.BucketName). + Str("key", refKV[0]).Msg("publish success") return cloudevents.ResultACK } -func (p *fsWorker) notifyLark(publishInfo *PublishInfo, err error) { +func (p *fsWorker) notifyLark(publishInfo *PublishInfoFS, err error) { if p.options.LarkWebhookURL == "" { return } message := fmt.Sprintf("Failed to publish %s/%s/%s file to fileserver: %v", - publishInfo.Name, - publishInfo.Version, - publishInfo.EntryPoint, + publishInfo.Repo, + publishInfo.CommitSHA, + "*", err) payload := map[string]interface{}{ @@ -142,50 +167,17 @@ func (p *fsWorker) notifyLark(publishInfo *PublishInfo, err error) { } } -func (p *fsWorker) publish(content io.ReadSeeker, info *PublishInfo) error { - keys := targetFsFullPaths(info) - if len(keys) == 0 { - return nil - } - +func (p *fsWorker) publish(content io.ReadSeeker, targetKey string) error { bucketName := p.options.S3.BucketName - // upload the artifact files to KS3 bucket. - for _, key := range keys { - if _, err := content.Seek(0, io.SeekStart); err != nil { - return err - } - - _, err := p.s3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(bucketName), - Key: aws.String(key), - Body: content, - }) - if err != nil { - p.logger. - Err(err). - Str("bucket", bucketName). - Str("key", key). - Msg("failed to upload file to KS3 bucket.") - return err - } + if _, err := content.Seek(0, io.SeekStart); err != nil { + return err } - // update git ref sha: download/refs/pingcap///sha1 - refKV := targetFsRefKeyValue(info) _, err := p.s3Client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(bucketName), - Key: aws.String(refKV[0]), - Body: bytes.NewReader([]byte(refKV[1])), + Key: aws.String(targetKey), + Body: content, }) - if err != nil { - p.logger. - Err(err). - Str("bucket", bucketName). - Str("key", refKV[0]). - Msg("failed to upload content in KS3 bucket.") - return err - } - - return nil + return err } diff --git a/publisher/pkg/impl/funcs.go b/publisher/pkg/impl/funcs.go index 92f5eec..9f940fa 100644 --- a/publisher/pkg/impl/funcs.go +++ b/publisher/pkg/impl/funcs.go @@ -19,7 +19,7 @@ import ( "github.com/PingCAP-QE/ee-apps/dl/pkg/oci" ) -func downloadFile(data *PublishRequest) (string, error) { +func downloadFile(data *PublishRequestTiUP) (string, error) { switch data.From.Type { case FromTypeOci: // save to file with `saveTo` param: @@ -53,7 +53,7 @@ func downloadFileFromReader(input io.Reader) (ret string, err error) { ret = input.Name() return nil }) - + return } diff --git a/publisher/pkg/impl/funcs_fs.go b/publisher/pkg/impl/funcs_fs.go index 0f01f20..842c32a 100644 --- a/publisher/pkg/impl/funcs_fs.go +++ b/publisher/pkg/impl/funcs_fs.go @@ -3,15 +3,17 @@ package impl import ( "context" "fmt" - "path/filepath" + "path" "regexp" "strings" "github.com/PingCAP-QE/ee-apps/dl/pkg/oci" - "golang.org/x/mod/semver" ) -var reTagOsArchSuffix = regexp.MustCompile(`_((linux|darwin)_(amd64|arm64))$`) +var ( + reTagOsArchSuffix = regexp.MustCompile(`_((linux|darwin)_(amd64|arm64))$`) + reTarballNameSuffix = regexp.MustCompile(`(.+)-v\d+\.\d+\.\d+(-.+)?-(linux|darwin)-(amd64|arm64).tar.gz$`) +) // Anlyze the artifact config and return the publish requests. // @@ -29,75 +31,7 @@ var reTagOsArchSuffix = regexp.MustCompile(`_((linux|darwin)_(amd64|arm64))$`) // 3.2.3 set the publish info arch with value of "net.pingcap.tibuild.architecture" key in top config. // 3.2.4 set the publish info name with prefix part of the value of "file" key in the element, right trim from the "-vX.Y.Z" part. // 3.2.5 set the publish info description, entrypoint with value of same key in the element. -func analyzeFsFromOciArtifact(repo, tag string) ([]PublishRequest, error) { - switch { - case strings.HasSuffix(repo, "/pingcap/tidb-tools/package"): - return analyzeFsFromOciArtifactForTiDBTools(repo, tag) - case strings.HasSuffix(repo, "/pingcap/tidb-binlog/package"): - return analyzeFsFromOciArtifactForTiDBBinlog(repo, tag) - // more special cases can be added here. - } - - // 1. Fetch the artifact config - config, ociDigest, err := fetchOCIArtifactConfig(repo, tag) - if err != nil { - return nil, err - } - - // 2. Check if "net.pingcap.tibuild.tiup" exists - tiupPackages, ok := config["net.pingcap.tibuild.tiup"].([]interface{}) - if !ok || len(tiupPackages) == 0 { - return nil, nil // No fileserver packages to publish - } - - // Get common information - version := transformFsVer(config["net.pingcap.tibuild.git-sha"].(string), tag) - - // 3. Loop through TiUP packages - // TODO: set the prefix from the match group part with reTagOsArchSuffix from the `tag`: - entryPointPrefix := getFsEntryPointPrefix(tag) - var publishRequests []PublishRequest - for _, pkg := range tiupPackages { - pkgMap := pkg.(map[string]interface{}) - file := pkgMap["file"].(string) - - // 3.1 Set the publish 'from' part - from := From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: repo, - File: file, - // use digest to avoid the problem of new override on the tag. - Tag: ociDigest, - }, - } - - // 3.2 Set the publish info - publishInfo := PublishInfo{ - Name: tiupPkgName(file), - Version: version, - // TODO: if the pkgName is "tidb", then the entry point should be "tidb-server.tar.gz" - EntryPoint: transformFsEntryPoint(entryPointPrefix, file), - } - publishRequests = append(publishRequests, PublishRequest{ - From: from, - Publish: publishInfo, - }) - } - - return publishRequests, nil -} - -func analyzeFsFromOciArtifactForTiDBTools(repo, tag string) ([]PublishRequest, error) { - // 1. Fetch the artifact config - config, ociDigest, err := fetchOCIArtifactConfig(repo, tag) - if err != nil { - return nil, err - } - - // Get common information - version := transformFsVer(config["net.pingcap.tibuild.git-sha"].(string), tag) - +func analyzeFsFromOciArtifact(repo, tag string) (*PublishRequestFS, error) { // Find the file. repository, err := getOciRepo(repo) if err != nil { @@ -108,81 +42,57 @@ func analyzeFsFromOciArtifactForTiDBTools(repo, tag string) ([]PublishRequest, e if err != nil { return nil, fmt.Errorf("failed to get file list: %v", err) } - var file string - for _, f := range files { - if strings.HasSuffix(f, ".tar.gz") && strings.HasPrefix(f, "tidb-tools-") { - file = f - break - } + + // 1. Fetch the artifact config + config, ociDigest, err := fetchOCIArtifactConfig(repo, tag) + if err != nil { + return nil, err } - // 3.1 Set the publish 'from' part from := From{ Type: FromTypeOci, Oci: &FromOci{ Repo: repo, - File: file, - Tag: ociDigest, + // use digest to avoid the problem of new override on the tag. + Tag: ociDigest, }, } - - // 3.2 Set the publish info - publishInfo := PublishInfo{ - Name: "tidb-tools", - Version: version, - EntryPoint: fmt.Sprintf("%s/tidb-tools.tar.gz", getFsEntryPointPrefix(tag)), - } - return []PublishRequest{{From: from, Publish: publishInfo}}, nil -} - -func analyzeFsFromOciArtifactForTiDBBinlog(repo, tag string) ([]PublishRequest, error) { - // 1. Fetch the artifact config - config, ociDigest, err := fetchOCIArtifactConfig(repo, tag) + fm, err := getFileKeys(files) if err != nil { return nil, err } - // Get common information - version := transformFsVer(config["net.pingcap.tibuild.git-sha"].(string), tag) - - // Find the file. - repository, err := getOciRepo(repo) - if err != nil { - return nil, fmt.Errorf("failed to get OCI repository: %v", err) + publishInfo := PublishInfoFS{ + Repo: strings.TrimSuffix(strings.SplitN(repo, "/", 2)[1], "/package"), + Branch: reTagOsArchSuffix.ReplaceAllString(tag, ""), + CommitSHA: config["net.pingcap.tibuild.git-sha"].(string), + FileTransferMap: fm, } - files, err := oci.ListFiles(context.Background(), repository, tag) - if err != nil { - return nil, fmt.Errorf("failed to get file list: %v", err) - } - var file string + return &PublishRequestFS{From: from, Publish: publishInfo}, nil +} + +func getFileKeys(files []string) (map[string]string, error) { + fm := map[string]string{} for _, f := range files { - if strings.HasSuffix(f, ".tar.gz") && strings.HasPrefix(f, "binaries-") { - file = f - break + if reTarballNameSuffix.MatchString(f) { + fm[f] = reTarballNameSuffix.ReplaceAllString(f, "") + // how to extract the os and arch from the file name with regexp: `reTarballNameSuffix` + // example: tidb-server-v1.1.1-linux-amd64.tar.gz => linux_amd64/tidb-server.tar.gz + matches := reTarballNameSuffix.FindStringSubmatch(f) + if len(matches) < 5 { + return nil, fmt.Errorf("failed to match tarball name: %s", f) + } + baseName := matches[1] + os := matches[3] + arch := matches[4] + fm[f] = fmt.Sprintf("%s_%s/%s.tar.gz", os, arch, baseName) } } - - // 3.1 Set the publish 'from' part - from := From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: repo, - File: file, - Tag: ociDigest, - }, - } - - // 3.2 Set the publish info - publishInfo := PublishInfo{ - Name: "tidb-binlog", - Version: version, - EntryPoint: fmt.Sprintf("%s/tidb-binlog.tar.gz", getFsEntryPointPrefix(tag)), - } - return []PublishRequest{{From: from, Publish: publishInfo}}, nil + return fm, nil } -func analyzeFsFromOciArtifactUrl(url string) ([]PublishRequest, error) { +func analyzeFsFromOciArtifactUrl(url string) (*PublishRequestFS, error) { repo, tag, err := splitRepoAndTag(url) if err != nil { return nil, err @@ -190,56 +100,19 @@ func analyzeFsFromOciArtifactUrl(url string) ([]PublishRequest, error) { return analyzeFsFromOciArtifact(repo, tag) } -func transformFsVer(commitSHA1, tag string) string { - // Remove OS and arch suffix using regex - branch := reTagOsArchSuffix.ReplaceAllString(tag, "") - return fmt.Sprintf("%s#%s", branch, commitSHA1) -} - -func transformFsEntryPoint(prefix, file string) string { - base := tiupPkgName(file) - switch base { - case "tidb", "pd", "tikv": - return fmt.Sprintf("%s/%s-server.tar.gz", prefix, base) - default: - return fmt.Sprintf("%s/%s.tar.gz", prefix, base) - } -} - -func getFsEntryPointPrefix(tag string) string { - if match := reTagOsArchSuffix.FindStringSubmatch(tag); len(match) > 1 { - return match[1] +func targetFsFullPaths(p *PublishInfoFS) map[string]string { + keyPrefix := path.Join("download/builds", p.Repo, p.Branch, p.CommitSHA) + ret := map[string]string{} + for k, v := range p.FileTransferMap { + ret[k] = path.Join(keyPrefix, v) } - return "centos7" -} - -func targetFsFullPaths(p *PublishInfo) []string { - var ret []string - - // the / path: pingcap//// - ret = append(ret, filepath.Join("download/builds/pingcap", p.Name, strings.ReplaceAll(p.Version, "#", "/"), p.EntryPoint)) - // if the part before the '#' char in p.Version is semver git tag format, then we need only one path. - if semver.IsValid(strings.Split(p.Version, "#")[0]) { - return ret - } - - // the / path: pingcap/// - ret = append(ret, filepath.Join("download/builds/pingcap", p.Name, filepath.Base(strings.ReplaceAll(p.Version, "#", "/")), p.EntryPoint)) - return ret } -func targetFsRefKeyValue(p *PublishInfo) [2]string { - var ret [2]string - verParts := strings.Split(p.Version, "#") - if len(verParts) > 1 { - ret[0] = fmt.Sprintf("download/refs/pingcap/%s/%s/sha1", p.Name, verParts[0]) - ret[1] = verParts[1] - } else { - ret[0] = fmt.Sprintf("download/refs/pingcap/%s/%s/sha1", p.Name, "master") - ret[1] = verParts[0] - } +func targetFsRefKeyValue(p *PublishInfoFS) [2]string { + key := path.Join("download/refs", p.Repo, p.Branch, "sha1") + val := p.CommitSHA - return ret + return [2]string{key, val} } diff --git a/publisher/pkg/impl/funcs_fs_test.go b/publisher/pkg/impl/funcs_fs_test.go index f89f579..d20098c 100644 --- a/publisher/pkg/impl/funcs_fs_test.go +++ b/publisher/pkg/impl/funcs_fs_test.go @@ -15,7 +15,7 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { tests := []struct { name string args args - want []PublishRequest + want *PublishRequestFS wantErr bool }{ { @@ -33,65 +33,27 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/pingcap/tidb/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tidb/package", - Tag: "sha256:b99b4e4f301bae87fa30fa58319da55bb6bdec94cbb29dccc35cf296815c3276", - File: "tidb-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "tidb", - Version: "v8.1.1#a7df4f9845d5d6a590c5d45dad0dcc9f21aa8765", - EntryPoint: "linux_amd64/tidb-server.tar.gz", + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/pingcap/tidb/package", + Tag: "sha256:b99b4e4f301bae87fa30fa58319da55bb6bdec94cbb29dccc35cf296815c3276", }, }, - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tidb/package", - Tag: "sha256:b99b4e4f301bae87fa30fa58319da55bb6bdec94cbb29dccc35cf296815c3276", - File: "br-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "br", - Version: "v8.1.1#a7df4f9845d5d6a590c5d45dad0dcc9f21aa8765", - EntryPoint: "linux_amd64/br.tar.gz", - }, - }, - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tidb/package", - Tag: "sha256:b99b4e4f301bae87fa30fa58319da55bb6bdec94cbb29dccc35cf296815c3276", - File: "dumpling-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "dumpling", - Version: "v8.1.1#a7df4f9845d5d6a590c5d45dad0dcc9f21aa8765", - EntryPoint: "linux_amd64/dumpling.tar.gz", - }, - }, - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tidb/package", - Tag: "sha256:b99b4e4f301bae87fa30fa58319da55bb6bdec94cbb29dccc35cf296815c3276", - File: "tidb-lightning-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "tidb-lightning", - Version: "v8.1.1#a7df4f9845d5d6a590c5d45dad0dcc9f21aa8765", - EntryPoint: "linux_amd64/tidb-lightning.tar.gz", + // target key: download/builds//// + // tikv: download/builds/tikv/tkv/master//tikv.tar.gz + // tidb: download/builds/pingcap/tidb/master//tidb.tar.gz + Publish: PublishInfoFS{ + Repo: "pingcap/tidb", + Branch: "v8.1.1", + CommitSHA: "a7df4f9845d5d6a590c5d45dad0dcc9f21aa8765", + FileTransferMap: map[string]string{ + "tidb-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tidb.tar.gz", + "br-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/br.tar.gz", + "dumpling-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/dumpling.tar.gz", + "tidb-lightning-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tidb-lightning.tar.gz", + "tidb-lightning-ctl-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tidb-lightning-ctl.tar.gz", }, }, }, @@ -103,65 +65,23 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/pingcap/tiflow/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: "oci", - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tiflow/package", - Tag: "sha256:14e97372e2884406dc1b8c8a9390a1fbdd57d91eee67ba340032622409e5c288", - File: "cdc-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "cdc", - Version: "v8.1.1#8ee0f3783a277161397d38bc62c48823de486b0d", - EntryPoint: "linux_amd64/cdc.tar.gz", - }, - }, - { - From: From{ - Type: "oci", - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tiflow/package", - Tag: "sha256:14e97372e2884406dc1b8c8a9390a1fbdd57d91eee67ba340032622409e5c288", - File: "dm-master-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "dm-master", - Version: "v8.1.1#8ee0f3783a277161397d38bc62c48823de486b0d", - EntryPoint: "linux_amd64/dm-master.tar.gz", + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/pingcap/tiflow/package", + Tag: "sha256:14e97372e2884406dc1b8c8a9390a1fbdd57d91eee67ba340032622409e5c288", }, }, - { - From: From{ - Type: "oci", - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tiflow/package", - Tag: "sha256:14e97372e2884406dc1b8c8a9390a1fbdd57d91eee67ba340032622409e5c288", - File: "dm-worker-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "dm-worker", - Version: "v8.1.1#8ee0f3783a277161397d38bc62c48823de486b0d", - EntryPoint: "linux_amd64/dm-worker.tar.gz", - }, - }, - { - From: From{ - Type: "oci", - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tiflow/package", - Tag: "sha256:14e97372e2884406dc1b8c8a9390a1fbdd57d91eee67ba340032622409e5c288", - File: "dmctl-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "dmctl", - Version: "v8.1.1#8ee0f3783a277161397d38bc62c48823de486b0d", - EntryPoint: "linux_amd64/dmctl.tar.gz", + Publish: PublishInfoFS{ + Repo: "pingcap/tiflow", + Branch: "v8.1.1", + CommitSHA: "8ee0f3783a277161397d38bc62c48823de486b0d", + FileTransferMap: map[string]string{ + "cdc-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/cdc.tar.gz", + "dm-master-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/dm-master.tar.gz", + "dm-worker-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/dm-worker.tar.gz", + "dmctl-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/dmctl.tar.gz", }, }, }, @@ -173,20 +93,20 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/pingcap/tiflash/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tiflash/package", - Tag: "sha256:4ba33a9106feb2189a5b1726155b6e1c15b102b8094956ece069afc01d9bb4a2", - File: "tiflash-v8.1.1-pre-linux-amd64.tar.gz", - }, + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/pingcap/tiflash/package", + Tag: "sha256:4ba33a9106feb2189a5b1726155b6e1c15b102b8094956ece069afc01d9bb4a2", }, - Publish: PublishInfo{ - Name: "tiflash", - Version: "v8.1.1#eb585f7d95d588bf8450c3cec02c36bb42c5e429", - EntryPoint: "linux_amd64/tiflash.tar.gz", + }, + Publish: PublishInfoFS{ + Repo: "pingcap/tiflash", + Branch: "v8.1.1", + CommitSHA: "eb585f7d95d588bf8450c3cec02c36bb42c5e429", + FileTransferMap: map[string]string{ + "tiflash-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tiflash.tar.gz", }, }, }, @@ -198,35 +118,22 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/tikv/pd/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/tikv/pd/package", - Tag: "sha256:4d1222a01dd594176ec7f2dcf0b8e8cdaa2f838621d7738b56cdb39c9847f0a7", - File: "pd-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "pd", - Version: "v8.1.1#f3dd0b62857ba0da97842349808aa4d4d4eefb34", - EntryPoint: "linux_amd64/pd-server.tar.gz", + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/tikv/pd/package", + Tag: "sha256:4d1222a01dd594176ec7f2dcf0b8e8cdaa2f838621d7738b56cdb39c9847f0a7", }, }, - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/tikv/pd/package", - Tag: "sha256:4d1222a01dd594176ec7f2dcf0b8e8cdaa2f838621d7738b56cdb39c9847f0a7", - File: "pd-recover-v8.1.1-pre-linux-amd64.tar.gz", - }, - }, - Publish: PublishInfo{ - Name: "pd-recover", - Version: "v8.1.1#f3dd0b62857ba0da97842349808aa4d4d4eefb34", - EntryPoint: "linux_amd64/pd-recover.tar.gz", + Publish: PublishInfoFS{ + Repo: "tikv/pd", + Branch: "v8.1.1", + CommitSHA: "f3dd0b62857ba0da97842349808aa4d4d4eefb34", + FileTransferMap: map[string]string{ + "pd-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/pd.tar.gz", + "pd-recover-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/pd-recover.tar.gz", + "pd-ctl-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/pd-ctl.tar.gz", }, }, }, @@ -238,20 +145,21 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/tikv/tikv/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/tikv/tikv/package", - Tag: "sha256:b526c02e883d54f97162c445294e9aa805620d5af9979b24667958aff870be06", - File: "tikv-v8.1.1-pre-linux-amd64.tar.gz", - }, + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/tikv/tikv/package", + Tag: "sha256:b526c02e883d54f97162c445294e9aa805620d5af9979b24667958aff870be06", }, - Publish: PublishInfo{ - Name: "tikv", - Version: "v8.1.1#7793f1d5dc40206fe406ca001be1e0d7f1b83a8f", - EntryPoint: "linux_amd64/tikv-server.tar.gz", + }, + Publish: PublishInfoFS{ + Repo: "tikv/tikv", + Branch: "v8.1.1", + CommitSHA: "7793f1d5dc40206fe406ca001be1e0d7f1b83a8f", + FileTransferMap: map[string]string{ + "tikv-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tikv.tar.gz", + "tikv-ctl-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tikv-ctl.tar.gz", }, }, }, @@ -263,20 +171,20 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/pingcap/tidb-tools/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tidb-tools/package", - Tag: "sha256:b50b45ffb0f53e3bf7c6140aa57fb768c4f2a9f6471e2987c423c0da217338b6", - File: "tidb-tools-v8.1.1-linux-amd64.tar.gz", - }, + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/pingcap/tidb-tools/package", + Tag: "sha256:b50b45ffb0f53e3bf7c6140aa57fb768c4f2a9f6471e2987c423c0da217338b6", }, - Publish: PublishInfo{ - Name: "tidb-tools", - Version: "v8.1.1#d226440121147098eb5eb99cbc1efb94092ec68e", - EntryPoint: "linux_amd64/tidb-tools.tar.gz", + }, + Publish: PublishInfoFS{ + Repo: "pingcap/tidb-tools", + Branch: "v8.1.1", + CommitSHA: "d226440121147098eb5eb99cbc1efb94092ec68e", + FileTransferMap: map[string]string{ + "tidb-tools-v8.1.1-linux-amd64.tar.gz": "linux_amd64/tidb-tools.tar.gz", }, }, }, @@ -288,20 +196,22 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/pingcap/tidb-binlog/package", tag: "v8.1.1_linux_amd64", }, - want: []PublishRequest{ - { - From: From{ - Type: FromTypeOci, - Oci: &FromOci{ - Repo: "hub.pingcap.net/pingcap/tidb-binlog/package", - Tag: "sha256:2c4704588ef754eaaed5d0034f44c6077257cebd382b90b2241b5e0ff4be640c", - File: "binaries-v8.1.1-pre-linux-amd64.tar.gz", - }, + want: &PublishRequestFS{ + From: From{ + Type: FromTypeOci, + Oci: &FromOci{ + Repo: "hub.pingcap.net/pingcap/tidb-binlog/package", + Tag: "sha256:2c4704588ef754eaaed5d0034f44c6077257cebd382b90b2241b5e0ff4be640c", }, - Publish: PublishInfo{ - Name: "tidb-binlog", - Version: "v8.1.1#79416e288d69bb530247546a846e7a89a8ef6d2f", - EntryPoint: "linux_amd64/tidb-binlog.tar.gz", + }, + Publish: PublishInfoFS{ + Repo: "pingcap/tidb-binlog", + Branch: "v8.1.1", + CommitSHA: "79416e288d69bb530247546a846e7a89a8ef6d2f", + FileTransferMap: map[string]string{ + "binaries-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/binaries.tar.gz", + "drainer-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/drainer.tar.gz", + "pump-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/pump.tar.gz", }, }, }, @@ -317,6 +227,7 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { wantErr: true, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := analyzeFsFromOciArtifact(tt.args.repo, tt.args.tag) @@ -335,62 +246,77 @@ func Test_analyzeFsFromOciArtifact(t *testing.T) { func Test_targetFsFullPaths(t *testing.T) { tests := []struct { name string - p *PublishInfo - want []string + p *PublishInfoFS + want map[string]string }{ { - name: "Basic version with commit hash", - p: &PublishInfo{ - Name: "tidb", - Version: "master#abc123def", - EntryPoint: "bin/tidb-server", - }, - want: []string{ - "download/builds/pingcap/tidb/master/abc123def/bin/tidb-server", - "download/builds/pingcap/tidb/abc123def/bin/tidb-server", - }, - }, - { - name: "Version with multiple hash symbols", - p: &PublishInfo{ - Name: "pd", - Version: "release-5.0#abc", - EntryPoint: "pd-server", + name: "Basic version with commit hash - tidb", + p: &PublishInfoFS{ + Repo: "pingcap/tidb", + Branch: "master", + CommitSHA: "abc123def", + FileTransferMap: map[string]string{ + "tidb-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tidb.tar.gz", + "br-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/br.tar.gz", + "dumpling-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/dumpling.tar.gz", + "tidb-lightning-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tidb-lightning.tar.gz", + "tidb-lightning-ctl-v8.1.1-pre-linux-amd64.tar.gz": "linux_amd64/tidb-lightning-ctl.tar.gz", + }, }, - want: []string{ - "download/builds/pingcap/pd/release-5.0/abc/pd-server", - "download/builds/pingcap/pd/abc/pd-server", + want: map[string]string{ + "tidb-v8.1.1-pre-linux-amd64.tar.gz": "download/builds/pingcap/tidb/master/abc123def/linux_amd64/tidb.tar.gz", + "br-v8.1.1-pre-linux-amd64.tar.gz": "download/builds/pingcap/tidb/master/abc123def/linux_amd64/br.tar.gz", + "dumpling-v8.1.1-pre-linux-amd64.tar.gz": "download/builds/pingcap/tidb/master/abc123def/linux_amd64/dumpling.tar.gz", + "tidb-lightning-v8.1.1-pre-linux-amd64.tar.gz": "download/builds/pingcap/tidb/master/abc123def/linux_amd64/tidb-lightning.tar.gz", + "tidb-lightning-ctl-v8.1.1-pre-linux-amd64.tar.gz": "download/builds/pingcap/tidb/master/abc123def/linux_amd64/tidb-lightning-ctl.tar.gz", }, }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := targetFsFullPaths(tt.p) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("targetFsFullPaths() = %v, want %v", got, tt.want) + } + }) + } +} +func Test_getFileKeys(t *testing.T) { + tests := []struct { + name string + files []string + want map[string]string + wantErr bool + }{ { - name: "Complex entry point path", - p: &PublishInfo{ - Name: "tiflash", - Version: "nightly#xyz789", - EntryPoint: "opt/tiflash/bin/tiflash-server", - }, - want: []string{ - "download/builds/pingcap/tiflash/nightly/xyz789/opt/tiflash/bin/tiflash-server", - "download/builds/pingcap/tiflash/xyz789/opt/tiflash/bin/tiflash-server", - }, + name: "Empty file list", + files: []string{}, + want: map[string]string{}, }, { - name: "tag", - p: &PublishInfo{ - Name: "tiflash", - Version: "v7.1.1#abcdef", - EntryPoint: "opt/tiflash/bin/tiflash-server", + name: "Multiple valid tarballs", + files: []string{ + "tidb-server-v1.1.1-linux-amd64.tar.gz", + "tikv-server-v2.0.0-darwin-arm64.tar.gz", + "tidb-server-v1.1.1-alpha-pre-linux-amd64.tar.gz", }, - want: []string{ - "download/builds/pingcap/tiflash/v7.1.1/abcdef/opt/tiflash/bin/tiflash-server", + want: map[string]string{ + "tidb-server-v1.1.1-linux-amd64.tar.gz": "linux_amd64/tidb-server.tar.gz", + "tikv-server-v2.0.0-darwin-arm64.tar.gz": "darwin_arm64/tikv-server.tar.gz", + "tidb-server-v1.1.1-alpha-pre-linux-amd64.tar.gz": "linux_amd64/tidb-server.tar.gz", }, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := targetFsFullPaths(tt.p) + got, err := getFileKeys(tt.files) + if (err != nil) != tt.wantErr { + t.Errorf("newFunction() error = %v, wantErr %v", err, tt.wantErr) + return + } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("targetFsFullPaths() = %v, want %v", got, tt.want) + t.Errorf("newFunction() = %v, want %v", got, tt.want) } }) } diff --git a/publisher/pkg/impl/funcs_test.go b/publisher/pkg/impl/funcs_test.go index de0ffd2..ca4daa9 100644 --- a/publisher/pkg/impl/funcs_test.go +++ b/publisher/pkg/impl/funcs_test.go @@ -134,7 +134,7 @@ func Test_analyzeFromOciArtifact(t *testing.T) { tests := []struct { name string args args - want []PublishRequest + want []PublishRequestTiUP wantErr bool }{ { @@ -143,7 +143,7 @@ func Test_analyzeFromOciArtifact(t *testing.T) { repo: "hub.pingcap.net/pingcap/tidb/package", tag: "v8.3.0_linux_amd64", }, - want: []PublishRequest{ + want: []PublishRequestTiUP{ { From: From{ Type: FromTypeOci, @@ -153,7 +153,7 @@ func Test_analyzeFromOciArtifact(t *testing.T) { File: "tidb-v8.3.0-pre-linux-amd64.tar.gz", }, }, - Publish: PublishInfo{ + Publish: PublishInfoTiUP{ Version: "v8.3.0", OS: "linux", Arch: "amd64", @@ -171,7 +171,7 @@ func Test_analyzeFromOciArtifact(t *testing.T) { File: "br-v8.3.0-pre-linux-amd64.tar.gz", }, }, - Publish: PublishInfo{ + Publish: PublishInfoTiUP{ Version: "v8.3.0", OS: "linux", Arch: "amd64", @@ -189,7 +189,7 @@ func Test_analyzeFromOciArtifact(t *testing.T) { File: "dumpling-v8.3.0-pre-linux-amd64.tar.gz", }, }, - Publish: PublishInfo{ + Publish: PublishInfoTiUP{ Version: "v8.3.0", OS: "linux", Arch: "amd64", @@ -207,7 +207,7 @@ func Test_analyzeFromOciArtifact(t *testing.T) { File: "tidb-lightning-v8.3.0-pre-linux-amd64.tar.gz", }, }, - Publish: PublishInfo{ + Publish: PublishInfoTiUP{ Version: "v8.3.0", OS: "linux", Arch: "amd64", diff --git a/publisher/pkg/impl/funcs_tiup.go b/publisher/pkg/impl/funcs_tiup.go index 1ea4c30..2a605a1 100644 --- a/publisher/pkg/impl/funcs_tiup.go +++ b/publisher/pkg/impl/funcs_tiup.go @@ -87,7 +87,7 @@ func calculateSHA256(filePath string) (string, error) { // 3.2.3 set the publish info arch with value of "net.pingcap.tibuild.architecture" key in top config. // 3.2.4 set the publish info name with prefix part of the value of "file" key in the element, right trim from the "-vX.Y.Z" part. // 3.2.5 set the publish info description, entrypoint with value of same key in the element. -func analyzeTiupFromOciArtifact(repo, tag string) ([]PublishRequest, error) { +func analyzeTiupFromOciArtifact(repo, tag string) ([]PublishRequestTiUP, error) { // 1. Fetch the artifact config config, ociDigest, err := fetchOCIArtifactConfig(repo, tag) if err != nil { @@ -106,7 +106,7 @@ func analyzeTiupFromOciArtifact(repo, tag string) ([]PublishRequest, error) { version := transformTiupVer(config["org.opencontainers.image.version"].(string), tag) // 3. Loop through TiUP packages - var publishRequests []PublishRequest + var publishRequests []PublishRequestTiUP for _, pkg := range tiupPackages { pkgMap := pkg.(map[string]interface{}) file := pkgMap["file"].(string) @@ -123,7 +123,7 @@ func analyzeTiupFromOciArtifact(repo, tag string) ([]PublishRequest, error) { } // 3.2 Set the publish info - publishInfo := PublishInfo{ + publishInfo := PublishInfoTiUP{ Name: tiupPkgName(file), Version: version, OS: os, @@ -131,7 +131,7 @@ func analyzeTiupFromOciArtifact(repo, tag string) ([]PublishRequest, error) { Description: pkgMap["description"].(string), EntryPoint: pkgMap["entrypoint"].(string), } - publishRequests = append(publishRequests, PublishRequest{ + publishRequests = append(publishRequests, PublishRequestTiUP{ From: from, Publish: publishInfo, }) @@ -140,7 +140,7 @@ func analyzeTiupFromOciArtifact(repo, tag string) ([]PublishRequest, error) { return publishRequests, nil } -func analyzeTiupFromOciArtifactUrl(url string) ([]PublishRequest, error) { +func analyzeTiupFromOciArtifactUrl(url string) ([]PublishRequestTiUP, error) { repo, tag, err := splitRepoAndTag(url) if err != nil { return nil, err @@ -169,6 +169,6 @@ func transformTiupVer(version, tag string) string { } } -func isNightlyTiup(p PublishInfo) bool { +func isNightlyTiup(p PublishInfoTiUP) bool { return tiupVersionRegex.MatchString(p.Version) } diff --git a/publisher/pkg/impl/tiup_service.go b/publisher/pkg/impl/tiup_service.go index 1ea6abc..75f47c6 100644 --- a/publisher/pkg/impl/tiup_service.go +++ b/publisher/pkg/impl/tiup_service.go @@ -123,7 +123,7 @@ func (s *tiupsrvc) ResetRateLimit(ctx context.Context) error { return nil } -func (s *tiupsrvc) composeEvents(requests []PublishRequest) []cloudevents.Event { +func (s *tiupsrvc) composeEvents(requests []PublishRequestTiUP) []cloudevents.Event { var ret []cloudevents.Event for _, request := range requests { event := cloudevents.NewEvent() diff --git a/publisher/pkg/impl/tiup_worker.go b/publisher/pkg/impl/tiup_worker.go index 07cc982..7ea269a 100644 --- a/publisher/pkg/impl/tiup_worker.go +++ b/publisher/pkg/impl/tiup_worker.go @@ -85,7 +85,7 @@ func (p *tiupWorker) Handle(event cloudevents.Event) cloudevents.Result { } p.redisClient.SetXX(context.Background(), event.ID(), PublishStateProcessing, redis.KeepTTL) - data := new(PublishRequest) + data := new(PublishRequestTiUP) if err := event.DataAs(&data); err != nil { return cloudevents.NewReceipt(false, "invalid data: %v", err) } @@ -104,7 +104,7 @@ func (p *tiupWorker) Handle(event cloudevents.Event) cloudevents.Result { return result } -func (p *tiupWorker) rateLimit(data *PublishRequest, ttl time.Duration, run func(*PublishRequest) cloudevents.Result) cloudevents.Result { +func (p *tiupWorker) rateLimit(data *PublishRequestTiUP, ttl time.Duration, run func(*PublishRequestTiUP) cloudevents.Result) cloudevents.Result { // Skip rate limiting for no nightly builds if !isNightlyTiup(data.Publish) || ttl <= 0 { return run(data) @@ -138,7 +138,7 @@ func (p *tiupWorker) rateLimit(data *PublishRequest, ttl time.Duration, run func return fmt.Errorf("skip: rate limit exceeded for package %s", data.Publish.Name) } -func (p *tiupWorker) handle(data *PublishRequest) cloudevents.Result { +func (p *tiupWorker) handle(data *PublishRequestTiUP) cloudevents.Result { // 1. get the the tarball from data.From. saveTo, err := downloadFile(data) if err != nil { @@ -181,7 +181,7 @@ func (p *tiupWorker) handle(data *PublishRequest) cloudevents.Result { return cloudevents.ResultACK } -func (p *tiupWorker) notifyLark(req *PublishRequest, err error) { +func (p *tiupWorker) notifyLark(req *PublishRequestTiUP, err error) { if p.options.LarkWebhookURL == "" { return } @@ -217,7 +217,7 @@ func (p *tiupWorker) notifyLark(req *PublishRequest, err error) { } } -func (p *tiupWorker) publish(file string, info *PublishInfo) error { +func (p *tiupWorker) publish(file string, info *PublishInfoTiUP) error { // Obtain a lock for our given global TiUP mirrors mutex. // After this is successful, no one else can obtain the same // lock (the same mutex name) until we unlock it. diff --git a/publisher/pkg/impl/tiup_worker_test.go b/publisher/pkg/impl/tiup_worker_test.go index 28d0404..154891c 100644 --- a/publisher/pkg/impl/tiup_worker_test.go +++ b/publisher/pkg/impl/tiup_worker_test.go @@ -24,7 +24,7 @@ func Test_tiupWorker_notifyLark(t *testing.T) { } } type args struct { - req *PublishRequest + req *PublishRequestTiUP err error } tests := []struct { @@ -46,8 +46,8 @@ func Test_tiupWorker_notifyLark(t *testing.T) { }, }, args: args{ - req: &PublishRequest{ - Publish: PublishInfo{ + req: &PublishRequestTiUP{ + Publish: PublishInfoTiUP{ Name: "test-package", Version: "v1.0.0", }, @@ -71,7 +71,7 @@ func Test_tiupWorker_notifyLark(t *testing.T) { }, }, args: args{ - req: &PublishRequest{ + req: &PublishRequestTiUP{ From: From{ Type: FromTypeOci, Oci: &FromOci{ @@ -79,7 +79,7 @@ func Test_tiupWorker_notifyLark(t *testing.T) { Tag: "test-tag", }, }, - Publish: PublishInfo{ + Publish: PublishInfoTiUP{ Name: "test-package", Version: "v1.0.0", OS: "linux", @@ -103,8 +103,8 @@ func Test_tiupWorker_notifyLark(t *testing.T) { }, }, args: args{ - req: &PublishRequest{ - Publish: PublishInfo{ + req: &PublishRequestTiUP{ + Publish: PublishInfoTiUP{ Name: "test-package", Version: "v1.0.0", }, @@ -126,7 +126,7 @@ func Test_tiupWorker_notifyLark(t *testing.T) { }, }, args: args{ - req: &PublishRequest{}, + req: &PublishRequestTiUP{}, err: errors.New("missing publish info"), }, }, diff --git a/publisher/pkg/impl/types.go b/publisher/pkg/impl/types.go index d2ecc5c..2fcadb1 100644 --- a/publisher/pkg/impl/types.go +++ b/publisher/pkg/impl/types.go @@ -23,11 +23,6 @@ const ( DefaultTiupNightlyInternal = 12 * time.Hour ) -type PublishRequest struct { - From From `json:"from,omitempty"` - Publish PublishInfo `json:"publish,omitempty"` -} - type From struct { Type string `json:"type,omitempty"` Oci *FromOci `json:"oci,omitempty"` @@ -45,16 +40,6 @@ func (f From) String() string { } } -type PublishInfo struct { - Name string `json:"name,omitempty"` // tiup pkg name or component name for fileserver - OS string `json:"os,omitempty"` // ignore for `EventTypeFsPublishRequest` - Arch string `json:"arch,omitempty"` // ignore for `EventTypeFsPublishRequest` - Version string `json:"version,omitempty"` // SemVer format for `EventTypeTiupPublishRequest` and "#" for `EventTypeFsPublishRequest` - Description string `json:"description,omitempty"` // ignore for `EventTypeFsPublishRequest` - EntryPoint string `json:"entry_point,omitempty"` // if event is `EventTypeFsPublishRequest`, the the value is the basename for store file, like tidb-server.tar.gz - Standalone bool `json:"standalone,omitempty"` // ignore for `EventTypeFsPublishRequest` -} - type FromOci struct { Repo string `json:"repo,omitempty"` Tag string `json:"tag,omitempty"` @@ -62,13 +47,40 @@ type FromOci struct { } func (f FromOci) String() string { - return f.Repo + ":" + f.Tag + "#" + f.File + return f.Repo + ":" + f.Tag + "#" + f.File } type FromHTTP struct { URL string `json:"url,omitempty"` } +type PublishRequestTiUP struct { + From From `json:"from,omitempty"` + Publish PublishInfoTiUP `json:"publish,omitempty"` +} + +type PublishInfoTiUP struct { + Name string `json:"name,omitempty"` // tiup pkg name or component name for fileserver + OS string `json:"os,omitempty"` // ignore for `EventTypeFsPublishRequest` + Arch string `json:"arch,omitempty"` // ignore for `EventTypeFsPublishRequest` + Version string `json:"version,omitempty"` // SemVer format for `EventTypeTiupPublishRequest` and "#" for `EventTypeFsPublishRequest` + Description string `json:"description,omitempty"` // ignore for `EventTypeFsPublishRequest` + EntryPoint string `json:"entry_point,omitempty"` // if event is `EventTypeFsPublishRequest`, the the value is the basename for store file, like tidb-server.tar.gz + Standalone bool `json:"standalone,omitempty"` // ignore for `EventTypeFsPublishRequest` +} + +type PublishRequestFS struct { + From From `json:"from,omitempty"` + Publish PublishInfoFS `json:"publish,omitempty"` +} + +type PublishInfoFS struct { + Repo string `json:"repo,omitempty"` + Branch string `json:"branch,omitempty"` + CommitSHA string `json:"commit_sha,omitempty"` + FileTransferMap map[string]string `json:"file_transfer_map,omitempty"` +} + // Worker provides handling for cloud events. type Worker interface { Handle(event cloudevents.Event) cloudevents.Result