From a37f37a745213a72f03bbcbe21b219bd55c1145d Mon Sep 17 00:00:00 2001 From: hasheddan Date: Wed, 9 Jun 2021 10:49:36 -0500 Subject: [PATCH 1/2] Write YAML document terminator on file end YAML supports a document terminator (...) when reading files in a stream. This allows for the reader to indicate that the current document has ended without indicating that the stream is finished. Anything after the document terminator is ignored by the consumer. Therefore, we write the terminator at the end of a read file, then immediately follow with a YAML document separator (---), which is mandatory according to the YAML spec when using document terminators. Signed-off-by: hasheddan --- pkg/parser/fsreader.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/parser/fsreader.go b/pkg/parser/fsreader.go index a9c7cd40d..40bf2d404 100644 --- a/pkg/parser/fsreader.go +++ b/pkg/parser/fsreader.go @@ -119,12 +119,14 @@ func (r *FsReadCloser) Read(p []byte) (n int, err error) { r.index++ r.position = 0 r.wroteBreak = false + n = copy(p, "\n---\n") + return n, nil } if r.index == len(r.paths) { return 0, io.EOF } if r.writeBreak { - n = copy(p, "\n---\n") + n = copy(p, "\n...\n") r.writeBreak = false r.wroteBreak = true return n, nil From d76ee9b0256603aadd07aa134de8ee4460e4e9fc Mon Sep 17 00:00:00 2001 From: hasheddan Date: Wed, 9 Jun 2021 10:56:24 -0500 Subject: [PATCH 2/2] Ignore empty YAML documents in parser If a document contains only whitespace and we are not able to decode it into one of the supported schemas, we ignore it and continue. We want to avoid skipping a type that does not have a registered schema so that we do not silently ignore errors when package building, but we consider it safe to assume that a YAML document with no content is safe to skip. Signed-off-by: hasheddan --- pkg/parser/parser.go | 13 +++++++++++++ pkg/parser/parser_test.go | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 13a3ea2ec..97bbf2b85 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -23,6 +23,7 @@ import ( "io" "io/ioutil" "strings" + "unicode" "github.com/pkg/errors" "github.com/spf13/afero" @@ -114,6 +115,18 @@ func (p *PackageParser) Parse(ctx context.Context, reader io.ReadCloser) (*Packa if err != nil { o, _, err := do.Decode(bytes, nil, nil) if err != nil { + empty := true + for _, b := range bytes { + if !unicode.IsSpace(rune(b)) { + empty = false + break + } + } + // If the YAML document only contains Unicode White Space we + // ignore and do not return an error. + if empty { + continue + } if anno, ok := reader.(AnnotatedReadCloser); ok { return pkg, errors.Wrap(err, fmt.Sprintf("%+v", anno.Annotate())) } diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index dc97c0cbf..e750996cf 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -35,6 +35,21 @@ var _ Parser = &PackageParser{} var ( crdBytes = []byte(`apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition +metadata: + name: test`) + + whitespaceBytes = []byte(`--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: test +--- + +--- + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: name: test`) @@ -53,6 +68,7 @@ func TestParser(t *testing.T) { allBytes := bytes.Join([][]byte{crdBytes, deployBytes}, []byte("\n---\n")) fs := afero.NewMemMapFs() _ = afero.WriteFile(fs, "crd.yaml", crdBytes, 0o644) + _ = afero.WriteFile(fs, "whitespace.yaml", whitespaceBytes, 0o644) _ = afero.WriteFile(fs, "deployment.yaml", deployBytes, 0o644) _ = afero.WriteFile(fs, "some/nested/dir/crd.yaml", crdBytes, 0o644) _ = afero.WriteFile(fs, ".crossplane/bad.yaml", crdBytes, 0o644) @@ -109,7 +125,7 @@ func TestParser(t *testing.T) { backend: NewFsBackend(fs, FsDir("."), FsFilters(SkipDirs(), SkipNotYAML(), SkipPath(".crossplane/*"))), pkg: &Package{ meta: []runtime.Object{deploy}, - objects: []runtime.Object{crd, crd}, + objects: []runtime.Object{crd, crd, crd, crd}, }, }, "FsBackendAll": {