https://developer.android.com/reference/android/security/NetworkSecurityPolicy#isCleartextTrafficPermitted()

README.md

@@ -1,7 +1,5 @@
 # Tools for authoring and serving codelabs
 
-[![Demo](https://storage.googleapis.com/claat/demo.png)](https://storage.googleapis.com/claat/demo.mp4)
-
 Codelabs are interactive instructional tutorials, which can be authored in Google Docs
 using some simple formatting conventions. You can also author codelabs using markdown syntax.
 This repo contains all the tools and documentation you’ll need for building and publishing

claat/README.md

@@ -14,7 +14,7 @@ The binaries, as well as their checksums are available at the
 
 Alternatively, if you have [Go installed](https://golang.org/doc/install):
 
-    go get github.com/googlecodelabs/tools/claat
+    go install github.com/googlecodelabs/tools/claat@latest
 
 If none of the above works, compile the tool from source following Dev workflow
 instructions below.

claat/VERSION

@@ -1 +1 @@
-1.0.4
+2.2.5

claat/cmd/export.go

@@ -111,20 +111,19 @@ func ExportCodelab(src string, rt http.RoundTripper, opts CmdExportOptions) (*ty
 	lastmod := types.ContextTime(clab.Mod)
 	clab.Meta.Source = src
 	meta := &clab.Meta
-	ctx := &types.Context{
-		Env:     opts.Expenv,
-		Format:  opts.Tmplout,
-		Prefix:  opts.Prefix,
-		MainGA:  opts.GlobalGA,
-		Updated: &lastmod,
-	}
 
 	dir := opts.Output // output dir or stdout
 	if !isStdout(dir) {
 		dir = codelabDir(dir, meta)
 	}
 	// write codelab and its metadata to disk
-	return meta, writeCodelab(dir, clab.Codelab, opts.ExtraVars, ctx)
+	return meta, writeCodelab(dir, clab.Codelab, opts.ExtraVars, &types.Context{
+		Env:     opts.Expenv,
+		Format:  opts.Tmplout,
+		Prefix:  opts.Prefix,
+		MainGA:  opts.GlobalGA,
+		Updated: &lastmod,
+	})
 }
 
 func ExportCodelabMemory(src io.ReadCloser, w io.Writer, opts CmdExportOptions) (*types.Meta, error) {

claat/fetch/drive/auth/auth.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"os"
 	"path"
@@ -43,14 +44,33 @@ var (
 		ClientID:     googClient,
 		ClientSecret: googSecret,
 		Scopes:       []string{scopeDriveReadOnly},
-		RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
+		RedirectURL:  "http://localhost:8091",
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  "https://accounts.google.com/o/oauth2/auth",
 			TokenURL: "https://accounts.google.com/o/oauth2/token",
 		},
 	}
 )
 
+// The webserver waits for an oauth code in the three-legged auth flow.
+func startWebServer() (code string, err error) {
+	listener, err := net.Listen("tcp", "localhost:8091")
+	if err != nil {
+		return "", err
+	}
+	codeCh := make(chan string)
+
+	go http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		code := r.FormValue("code")
+		codeCh <- code // send code to OAuth flow
+		listener.Close()
+		w.Header().Set("Content-Type", "text/plain")
+		fmt.Fprintf(w, "Received oauth code\r\nYou can now safely close this browser window.")
+	}))
+        code = <- codeCh
+	return code, nil
+}
+
 type authorizationHandler func(conf *oauth2.Config) (*oauth2.Token, error)
 
 type internalOptions struct {
@@ -214,10 +234,10 @@ func (c *cachedTokenSource) Token() (*oauth2.Token, error) {
 
 // authorize performs user authorization flow, asking for permissions grant.
 func authorize(conf *oauth2.Config) (*oauth2.Token, error) {
-	aurl := conf.AuthCodeURL("unused", oauth2.AccessTypeOffline)
-	fmt.Printf("Authorize me at following URL, please:\n\n%s\n\nCode: ", aurl)
-	var code string
-	if _, err := fmt.Scan(&code); err != nil {
+	aURL := conf.AuthCodeURL("unused", oauth2.AccessTypeOffline)
+	fmt.Printf("Authorize me at following URL, please:\n\n%s\n", aURL)
+	code, err := startWebServer()
+	if err != nil {
 		return nil, err
 	}
 	return conf.Exchange(context.Background(), code)

claat/fetch/fetch.go

@@ -46,6 +46,9 @@ const (
 
 	// driveAPI is a base URL for Drive API
 	driveAPI = "https://www.googleapis.com/drive/v3"
+
+	// Minimum image size in bytes for extension detection.
+	minImageSize = 11
 )
 
 // TODO: create an enum for use with "nometa" for readability's sake
@@ -111,6 +114,7 @@ type Fetcher struct {
 	roundTripper http.RoundTripper
 }
 
+// NewFetcher creates an instance of Fetcher.
 func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
 	return &Fetcher{
 		authHelper:   nil,
@@ -225,7 +229,7 @@ func (f *Fetcher) SlurpImages(src, dir string, n []nodes.Node, images map[string
 	for _, imageNode := range imageNodes {
 		go func(imageNode *nodes.ImageNode) {
 			url := imageNode.Src
-			file, err := f.slurpBytes(src, dir, url)
+			file, err := f.slurpBytes(src, dir, url, imageNode.Bytes)
 			if err == nil {
 				imageNode.Src = filepath.Join(util.ImgDirname, file)
 			}
@@ -247,43 +251,52 @@ func (f *Fetcher) SlurpImages(src, dir string, n []nodes.Node, images map[string
 	return nil
 }
 
-func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
-	// images can be local in Markdown cases or remote.
+func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string, imgBytes []byte) (string, error) {
+	// images can be data URLs, local in Markdown cases or remote.
 	// Only proceed a simple copy on local reference.
 	var b []byte
 	var ext string
-	u, err := url.Parse(imgURL)
-	if err != nil {
-		return "", err
-	}
+	var err error
 
-	// If the codelab source is being downloaded from the network, then we should interpret
-	// the image URL in the same way.
-	srcUrl, err := url.Parse(codelabSrc)
-	if err == nil && srcUrl.Host != "" {
-		u = srcUrl.ResolveReference(u)
-	}
-
-	if u.Host == "" {
-		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
-			return "", err
+	if len(imgBytes) > 0 {
+		// Slurp bytes from image URL data.
+		b = imgBytes
+		if ext, err = imgExtFromBytes(b); err != nil {
+			return "", fmt.Errorf("Error reading image type: %v", err)
 		}
-		b, err = ioutil.ReadFile(imgURL)
-		ext = filepath.Ext(imgURL)
 	} else {
-		b, err = f.slurpRemoteBytes(u.String(), 5)
-		if string(b[6:10]) == "JFIF" {
-			ext = ".jpeg"
-		} else if string(b[0:3]) == "GIF" {
-			ext = ".gif"
+		// Slurp bytes from local or remote URL.
+		u, err := url.Parse(imgURL)
+		if err != nil {
+			return "", err
+		}
+
+		// If the codelab source is being downloaded from the network, then we should interpret
+		// the image URL in the same way.
+		srcURL, err := url.Parse(codelabSrc)
+		if err == nil && srcURL.Host != "" {
+			u = srcURL.ResolveReference(u)
+		}
+
+		if u.Host == "" {
+			if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
+				return "", err
+			}
+			if b, err = ioutil.ReadFile(imgURL); err != nil {
+				return "", err
+			}
+			ext = filepath.Ext(imgURL)
 		} else {
-			ext = ".png"
+			if b, err = f.slurpRemoteBytes(u.String(), 5); err != nil {
+				return "", fmt.Errorf("Error downloading image at %s: %v", u.String(), err)
+			}
+			if ext, err = imgExtFromBytes(b); err != nil {
+				return "", fmt.Errorf("Error reading image type at %s: %v", u.String(), err)
+			}
 		}
 	}
-	if err != nil {
-		return "", err
-	}
 
+	// Generate image file from slurped bytes.
 	crc := crc64.Checksum(b, f.crcTable)
 	file := fmt.Sprintf("%x%s", crc, ext)
 	dst := filepath.Join(dir, file)
@@ -479,7 +492,10 @@ func gdocID(url string) string {
 }
 
 func gdocExportURL(id string) string {
-	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
+	q := url.Values{
+		"mimeType": {"text/html"},
+	}
+	return fmt.Sprintf("%s/files/%s/export?%s", driveAPI, id, q.Encode())
 }
 
 // restrictPathToParent will ensure that assetPath is in parent.
@@ -509,3 +525,17 @@ func isStdout(filename string) bool {
 func codelabDir(base string, m *types.Meta) string {
 	return filepath.Join(base, m.ID)
 }
+
+func imgExtFromBytes(b []byte) (string, error) {
+	if len(b) < minImageSize {
+		return "", fmt.Errorf("error parsing image - response \"%s\" is too small (< %d bytes)", b, minImageSize)
+	}
+	ext := ".png"
+	switch {
+	case string(b[6:10]) == "JFIF":
+		ext = ".jpeg"
+	case string(b[0:3]) == "GIF":
+		ext = ".gif"
+	}
+	return ext, nil
+}

claat/fetch/fetch_test.go

@@ -104,6 +104,34 @@ func TestFuzzRestrictPathToParent(t *testing.T) {
 	}
 }
 
+func TestImgExtFromBytes(t *testing.T) {
+	tests := []struct {
+		bytes []byte
+
+		wantExt string
+		wantErr bool
+	}{
+		{[]byte("012345JFIF0"), ".jpeg", false},
+		{[]byte("GIF34567890"), ".gif", false},
+		{[]byte("SOMETHINGELSE"), ".png", false},
+		{[]byte("GIF345JFIF0"), ".jpeg", false},
+		{[]byte("toosmall"), "", true},
+	}
+	for _, tc := range tests {
+		t.Run(fmt.Sprintf("bytes: %s", tc.bytes), func(t *testing.T) {
+			ext, err := imgExtFromBytes(tc.bytes)
+
+			if err != nil != tc.wantErr {
+				t.Errorf("imgExtFromBytes() error = %v, wantErr %v", err, tc.wantErr)
+				return
+			}
+			if ext != tc.wantExt {
+				t.Errorf("imgExtFromBytes() return: got %s, wanted %s", ext, tc.wantExt)
+			}
+		})
+	}
+}
+
 // safeAbs compute Abs of p and fail the test if not valid.
 // Empty string return empty path.
 func safeAbs(t *testing.T, p string) string {

claat/go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/google/go-cmp v0.5.6
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/x1ddos/csslex v0.0.0-20160125172232-7894d8ab8bfe
 	github.com/yuin/goldmark v1.3.7
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5

claat/go.sum

@@ -107,8 +107,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/x1ddos/csslex v0.0.0-20160125172232-7894d8ab8bfe h1:SX7lFdwn40ahL78CxofAh548P+dcWjdRNpirU7+sKiE=
 github.com/x1ddos/csslex v0.0.0-20160125172232-7894d8ab8bfe/go.mod h1:SwmD4V+Y0RjNqvt8hW2FpZNkQnoFVNtBF9qEnevUueU=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

claat/nodes/iframe.go

@@ -6,12 +6,16 @@ var IframeAllowlist = []string{
 	"carto.com",
 	"codepen.io",
 	"dartlang.org",
+	"dartpad.dev",
+	"demo.arcade.software",
 	"github.com",
 	"glitch.com",
 	"google.com",
 	"google.dev",
 	"observablehq.com",
 	"repl.it",
+	"stackblitz.com",
+	"vimeo.com",
 	"web.dev",
 }
 

claat/nodes/image.go

@@ -7,6 +7,7 @@ type NewImageNodeOptions struct {
 	Width float32
 	Alt   string
 	Title string
+	Bytes []byte
 }
 
 // NewImageNode creates a new ImageNode with the given options.
@@ -18,6 +19,7 @@ func NewImageNode(opts NewImageNodeOptions) *ImageNode {
 		Width: opts.Width,
 		Alt:   opts.Alt,
 		Title: opts.Title,
+		Bytes: opts.Bytes,
 	}
 }
 
@@ -28,11 +30,12 @@ type ImageNode struct {
 	Width float32
 	Alt   string
 	Title string
+	Bytes []byte
 }
 
 // Empty returns true if its Src is zero, excluding space runes.
 func (in *ImageNode) Empty() bool {
-	return strings.TrimSpace(in.Src) == ""
+	return strings.TrimSpace(in.Src) == "" && len(in.Bytes) == 0
 }
 
 // ImageNodes extracts everything except NodeImage nodes, recursively.

claat/nodes/image_test.go

@@ -1,11 +1,14 @@
 package nodes
 
 import (
+	"encoding/base64"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 )
 
+var testBytes, _ = base64.StdEncoding.DecodeString("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")
+
 func TestNewImageNode(t *testing.T) {
 	tests := []struct {
 		name   string
@@ -19,7 +22,7 @@ func TestNewImageNode(t *testing.T) {
 			},
 		},
 		{
-			name: "NonEmpty",
+			name: "StandardURL",
 			inOpts: NewImageNodeOptions{
 				Src:   "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png",
 				Width: 1.0,
@@ -34,6 +37,22 @@ func TestNewImageNode(t *testing.T) {
 				Alt:   "bar",
 			},
 		},
+		{
+			name: "DataURL",
+			inOpts: NewImageNodeOptions{
+				Width: 1.0,
+				Title: "foo",
+				Alt:   "bar",
+				Bytes: testBytes,
+			},
+			out: &ImageNode{
+				node:  node{typ: NodeImage},
+				Width: 1.0,
+				Title: "foo",
+				Alt:   "bar",
+				Bytes: testBytes,
+			},
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {