hack: Support merge multiple issues

- Strip header as workaround until dyweb/dy-weekly-generator#25 is fixed

Dockerfile

@@ -24,4 +24,4 @@ COPY --from=builder /usr/bin/weekly-gen .
 COPY --from=builder /go/bin/weekly .
 
 ENTRYPOINT ["weekly"]
-CMD ["issue"]
+CMD ["gen"]

scripts/weekly/build.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os/exec"
@@ -31,12 +32,15 @@ func BuildRecent(ctx context.Context, home string) error {
 	if err != nil {
 		return err
 	}
+
 	// Group issues by filename, this could happen if we are testing github workflow cron or manually open and closing issues.
 	issueByFile := make(map[string][]*github.Issue)
 	for _, issue := range issues {
 		fname := FileName(issue.GetCreatedAt())
 		issueByFile[fname] = append(issueByFile[fname], issue)
 	}
+
+	// Call dy-weekly-generator
 	for fname, issues := range issueByFile {
 		p := filepath.Join(home, fname)
 		if len(issues) == 1 {
@@ -45,7 +49,13 @@ func BuildRecent(ctx context.Context, home string) error {
 			}
 		}
 
-		// TODO: implement merging multiple issue, open a new issue in weekly generator
+		var ids []int
+		for _, issue := range issues {
+			ids = append(ids, issue.GetNumber())
+		}
+		if err := MergeIssues(ids, p); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -54,16 +64,53 @@ func BuildRecent(ctx context.Context, home string) error {
 // shell out to weekly-gen binary from https://github.com/dyweb/dy-weekly-generator
 // weekly-gen --repo dyweb/weekly --issue 183
 func BuildOne(issue int, dst string) error {
+	out, err := RunGenerator(issue)
+	if err != nil {
+		return err
+	}
+	return WriteWeeklyFile(out, dst)
+}
+
+const defaultFrontMatter = `---
+layout: post
+title: Weekly
+category: Weekly
+author: 东岳
+---`
+
+// MergeIssues write multiple issues into one weekly file.
+func MergeIssues(issueIds []int, dst string) error {
+	merged := []byte(defaultFrontMatter)
+	for _, id := range issueIds {
+		out, err := RunGenerator(id)
+		if err != nil {
+			return err
+		}
+		out = bytes.Trim(out, "\n")
+		out = StripHeader(out)
+		merged = append(merged, '\n')
+		merged = append(merged, out...)
+		merged = append(merged, '\n')
+	}
+	return WriteWeeklyFile(merged, dst)
+}
+
+// TODO(at15): strip header flag https://github.com/dyweb/dy-weekly-generator/issues/25
+func RunGenerator(issue int) ([]byte, error) {
 	cmd := exec.Command("weekly-gen", "--repo", "dyweb/weekly", "--issue", strconv.Itoa(issue))
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrapf(err, "error running weekly-gen for %d %s", issue, string(out))
+		return nil, errors.Wrapf(err, "error running weekly-gen for %d %s", issue, string(out))
 	}
+	return out, nil
+}
+
+func WriteWeeklyFile(b []byte, dst string) error {
 	dir := filepath.Dir(dst)
 	if err := fsutil.MkdirIfNotExists(dir); err != nil {
 		return err
 	}
-	if err := fsutil.WriteFile(dst, out); err != nil {
+	if err := fsutil.WriteFile(dst, b); err != nil {
 		return errors.Wrapf(err, "error writing weekly-gen output to %s", dst)
 	}
 	return nil
@@ -86,3 +133,19 @@ func FileName(createTime time.Time) string {
 	date := cfg.With(createTime).BeginningOfWeek()
 	return fmt.Sprintf("%d/%d-%.2d-%.2d-weekly.md", date.Year(), date.Year(), int(date.Month()), date.Day())
 }
+
+var frontMatterDash = []byte("---")
+
+// StripHeader remove the front matter https://jekyllrb.com/docs/front-matter/ from generated weekly.
+// TODO: work around until https://github.com/dyweb/dy-weekly-generator/issues/25 is fixed
+func StripHeader(b []byte) []byte {
+	if !bytes.HasPrefix(b, frontMatterDash) {
+		return b
+	}
+	b = b[len(frontMatterDash):]
+	end := bytes.Index(b, frontMatterDash)
+	if end == -1 {
+		return b
+	}
+	return b[end+len(frontMatterDash):]
+}

scripts/weekly/build_test.go

@@ -1,11 +1,21 @@
 package main
 
 import (
+	"bytes"
 	"testing"
 	"time"
+
+	"github.com/dyweb/gommon/util/testutil"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestFileName(t *testing.T) {
 	s := FileName(time.Now())
 	t.Logf("%s", s)
 }
+
+func TestStripHeader(t *testing.T) {
+	b := testutil.ReadFixture(t, "testdata/2020-01-06-weekly.md")
+	s := StripHeader(b)
+	assert.False(t, bytes.Contains(s, frontMatterDash))
+}

scripts/weekly/testdata/2020-01-06-weekly.md

@@ -0,0 +1,174 @@
+---
+layout: post
+title: Weekly
+category: Weekly
+author: 东岳
+
+---
+
+From https://github.com/zenany/weekly/commit/0337e2bf4dd4eaa554abd90c310b1f903bbaf87b
+
+- https://github.com/includeos/IncludeOS yet another unikernel
+- https://zhuanlan.zhihu.com/p/100660049 llhttp是如何使Node.js性能翻倍的
+
+***
+
+Notes from NeurIPS 19
+
+https://github.com/RobertTLange/conference-school-notes/tree/master/2019-12-NeuRIPS
+
+ *Submitted via [bookmarklet](https://gist.github.com/htfy96/301ae2b1c477a4a644e943bbc27c9588)* :sparkles:
+
+***
+
+https://github.com/DoctorWkt/acwj A Compiler Writing Journey
+
+***
+
+https://blog.algolia.com/inside-the-algolia-engine-part-1-indexing-vs-search/
+
+***
+
+http://bangbangcon.com/west/speakers/
+
+***
+
+http://composition.al/blog/2019/03/31/toward-domain-specific-solvers-for-distributed-consistency-will-appear-at-snapl-2019/ cc @LukeXuan 
+
+***
+
+https://trailofbits.github.io/ctf/ CTF field guide
+
+***
+
+https://github.com/Unikernel-Systems/unikernel.org a list of unikernels
+
+***
+
+https://github.com/cetic/unikernels a very detailed survey on unikernels
+
+***
+
+https://github.com/awslabs/deequ measure data quality in large datasets
+
+It has some anomaly detection https://github.com/awslabs/deequ/tree/master/src/main/scala/com/amazon/deequ/anomalydetection
+
+***
+
+https://github.com/deislabs/krustlet Kubernetes Rust Kubelet using wasm runtime
+
+***
+
+https://github.com/klauspost/compress/ high performance compression lib used by victoriametrics
+
+
+***
+
+[Is Parallel Programming Hard, And, If So, What Can You Do About It?](http://kernel.org/pub/linux/kernel/people/paulmck/perfbook/perfbook.html) 
+
+[Source](https://github.com/paulmckrcu/perfbook)
+
+***
+
+https://github.com/rotisserie/eris yet another go error handling library, should look at it and put into alternative sections for gommon/errors
+
+***
+
+https://www.aidungeon.io/ cc @arrowrowe 
+
+***
+
+Rust-powered Gitbook generator.
+
+https://github.com/rust-lang/mdBook
+
+ *Submitted via [bookmarklet](https://gist.github.com/htfy96/301ae2b1c477a4a644e943bbc27c9588)* :sparkles:
+
+***
+
+https://github.com/knqyf263/cob Continuous Benchmark for Go Project
+
+***
+
+https://github.com/microsoft/verona Research programming language for concurrent ownership
+
+btw: the name is same as my apartment ...
+
+***
+
+> https://github.com/microsoft/verona Research programming language for concurrent ownership
+> 
+> btw: the name is same as my apartment ...
+
+It’s a city in northern Italy, so stay calm.
+
+***
+
+https://gcc.godbolt.org/
+
+Compiler Explorer
+
+***
+
+https://lamport.azurewebsites.net/tla/paxos-algorithm.html?back-link=news.html The Paxos Algorithm
+or How to Win a Turing Award Leslie Lamport
+
+
+
+***
+
+https://github.com/learnedsystems/SOSD bechmark for learned index cc @gaocegege 
+
+***
+
+https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-172-performance-engineering-of-software-systems-fall-2018/lecture-slides/index.htm
+
+Performance Engineering of Software Systems (MIT 6.172) lecture notes
+
+***
+
+https://lemire.me/blog/2020/02/07/research-should-not-stop-with-the-research-paper/ cc @raspberryice 
+
+***
+
+https://github.com/sourcerer-io/hall-of-fame cc @gaocegege 
+
+***
+
+https://github.com/neuronetio/gantt-elastic saw it from @breeswish btw: cc @raspberryice 
+
+***
+
+> https://gcc.godbolt.org/
+> 
+> Compiler Explorer
+
+这个有点意思！！！
+
+***
+
+From https://github.com/zenany/weekly/commit/d8f92d061460a608783198d914b6d14c45e21c09
+
+- https://github.com/cli/cli official github cli cc @gaocegege 
+- https://github.com/fullcalendar/fullcalendar full sized drag & drop event calendar
+- https://legrajs.com/ draw using LEGO® like brick shapes on an HTML `<canvas>` element
+- https://ultimateelectronicsbook.com/ 
+
+***
+
+https://eli.thegreenplace.net/2019/go-internals-capturing-loop-variables-in-closures/
+
+***
+
+https://github.com/MaterializeInc/materialize SQL for streaming data, use [timely dataflow](https://github.com/TimelyDataflow/timely-dataflow) which is based on an SOSP 2013 Paper [Naiad: a timely dataflow system](http://dl.acm.org/citation.cfm?id=2522738) from [MS](https://github.com/MicrosoftResearch/Naiad)
+
+***
+
+
+
+http://olab.is.s.u-tokyo.ac.jp/~kamil.rocki/dca/
+
+ *Submitted via [bookmarklet](https://gist.github.com/htfy96/301ae2b1c477a4a644e943bbc27c9588)* :sparkles:
+
+***
+