Skip to content

Commit

Permalink
add support for docs tutorial route and use etags
Browse files Browse the repository at this point in the history
  • Loading branch information
@riknoll
riknoll committed Oct 17, 2024
1 parent 2ab6330 commit f635ffb
Show file tree
Hide file tree
Showing 7 changed files with 151 additions and 35 deletions.
4 changes: 4 additions & 0 deletions cli/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,10 @@ class FileGithubDb implements pxt.github.IGithubDb {
loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {
return this.loadAsync(repopath, tag, "tutorial", (r, t) => this.db.loadTutorialMarkdown(r, t));
}

cacheReposAsync(resp: pxt.github.GHTutorialResponse) {
return this.db.cacheReposAsync(resp);
}
}

function searchAsync(...query: string[]) {
Expand Down
4 changes: 4 additions & 0 deletions pxtcompiler/simpledriver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ namespace pxt {
loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {
return this.loadAsync(repopath, tag, "tutorial", (r, t) => this.db.loadTutorialMarkdown(r, t));
}

cacheReposAsync(resp: pxt.github.GHTutorialResponse) {
return this.db.cacheReposAsync(resp);
}
}

function pkgOverrideAsync(pkg: pxt.Package) {
Expand Down
18 changes: 18 additions & 0 deletions pxtlib/browserutils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1409,6 +1409,24 @@ namespace pxt.BrowserUtils {
return `${url}${url.indexOf('?') > 0 ? "&" : "?"}rnd=${Math.random()}`
}

export function appendUrlQueryParams(url: string, params: URLSearchParams) {
const entries: string[] = [];
for (const [key, value] of params.entries()) {
entries.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`);
}

if (entries.length) {
if (url.indexOf("?") !== -1) {
url += "&" + entries.join("&");
}
else {
url += "?" + entries.join("&");
}
}

return url;
}

export function legacyCopyText(element: HTMLInputElement | HTMLTextAreaElement) {
element.focus();
element.setSelectionRange(0, 9999);
Expand Down
52 changes: 41 additions & 11 deletions pxtlib/emitter/cloud.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
namespace pxt.Cloud {
import Util = pxtc.Util;

export let apiRoot = (pxt.BrowserUtils.isLocalHost() || Util.isNodeJS) ? "https://www.makecode.com/api/" : "/api/";
export let apiRoot = "https://arcade.riknoll.staging.pxt.io/api/";

export let accessToken = "";
export let localToken = "";
Expand Down Expand Up @@ -32,6 +32,7 @@ namespace pxt.Cloud {
export function useCdnApi() {
return pxt.webConfig && !pxt.webConfig.isStatic
&& !BrowserUtils.isLocalHost() && !!pxt.webConfig.cdnUrl
&& !/nocdn=1/i.test(window.location.href);
}

export function cdnApiUrl(url: string) {
Expand Down Expand Up @@ -135,7 +136,7 @@ namespace pxt.Cloud {
return resp.json;
}

export async function markdownAsync(docid: string, locale?: string, propagateExceptions?: boolean): Promise<string> {
export async function markdownAsync(docid: string, locale?: string, propagateExceptions?: boolean, downloadTutorialBundle?: boolean): Promise<string> {
// 1h check on markdown content if not on development server
const MARKDOWN_EXPIRATION = pxt.BrowserUtils.isLocalHostDev() ? 0 : 1 * 60 * 60 * 1000;
// 1w check don't use cached version and wait for new content
Expand All @@ -148,7 +149,7 @@ namespace pxt.Cloud {

const downloadAndSetMarkdownAsync = async () => {
try {
const r = await downloadMarkdownAsync(docid, locale, entry?.etag);
const r = await downloadMarkdownAsync(docid, locale, entry?.etag, downloadTutorialBundle);
// TODO directly compare the entry/response etags after backend change
if (!entry || (r.md && entry.md !== r.md)) {
await db.setAsync(locale, docid, r.etag, undefined, r.md);
Expand Down Expand Up @@ -189,11 +190,13 @@ namespace pxt.Cloud {
return downloadAndSetMarkdownAsync();
}

function downloadMarkdownAsync(docid: string, locale?: string, etag?: string): Promise<{ md: string; etag?: string; }> {
async function downloadMarkdownAsync(docid: string, locale?: string, etag?: string, downloadTutorialBundle?: boolean): Promise<{ md: string; etag?: string; }> {
const packaged = pxt.webConfig?.isStatic;
const targetVersion = pxt.appTarget.versions && pxt.appTarget.versions.target || '?';
let url: string;

const searchParams = new URLSearchParams();

if (packaged) {
url = docid;
const isUnderDocs = /\/?docs\//.test(url);
Expand All @@ -205,24 +208,51 @@ namespace pxt.Cloud {
if (!hasExt) {
url = `${url}.md`;
}
} else {
url = `md/${pxt.appTarget.id}/${docid.replace(/^\//, "")}?targetVersion=${encodeURIComponent(targetVersion)}`;
}
else {
url = `md/${pxt.appTarget.id}/${docid.replace(/^\//, "")}`;
searchParams.set("targetVersion", targetVersion);
}
if (locale != "en") {
url += `${packaged ? "?" : "&"}lang=${encodeURIComponent(locale)}`
searchParams.set("lang", locale);
}

url = pxt.BrowserUtils.appendUrlQueryParams(url, searchParams);

if (pxt.BrowserUtils.isLocalHost() && !pxt.Util.liveLocalizationEnabled()) {
return localRequestAsync(url).then(resp => {
if (resp.statusCode == 404)
return privateRequestAsync({ url, method: "GET" })
.then(resp => { return { md: resp.text, etag: <string>resp.headers["etag"] }; });
else return { md: resp.text, etag: undefined };
});
} else {
const headers: pxt.Map<string> = etag && !useCdnApi() ? { "If-None-Match": etag } : undefined;
return apiRequestWithCdnAsync({ url, method: "GET", headers })
.then(resp => { return { md: resp.text, etag: <string>resp.headers["etag"] }; });
}

const headers: pxt.Map<string> = etag && !useCdnApi() ? { "If-None-Match": etag } : undefined;

let resp: Util.HttpResponse;
let md: string;

if (!packaged && downloadTutorialBundle) {
url = url.replace(/^md\//, "ghtutorial/docs/");
resp = await apiRequestWithCdnAsync({ url, method: "GET", headers });

const body = resp.json as pxt.github.GHTutorialResponse;
await pxt.github.db.cacheReposAsync(body);

md = body.markdown as string;
}
else {
resp = await apiRequestWithCdnAsync({ url, method: "GET", headers });
md = resp.text;
}

return (
{
md,
etag: (resp.headers["etag"] as string)
}
);
}

export function privateDeleteAsync(path: string) {
Expand Down
62 changes: 48 additions & 14 deletions pxtlib/github.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ namespace pxt.github {
}

export interface GHTutorialResponse {
filename: string;
markdownRepo: GHTutorialRepoInfo;
path: string;
markdown: string | { filename: string, repo: GHTutorialRepoInfo };
dependencies: GHTutorialRepoInfo[];
}

Expand Down Expand Up @@ -126,6 +126,7 @@ namespace pxt.github {
loadConfigAsync(repopath: string, tag: string): Promise<pxt.PackageConfig>;
loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage>;
loadTutorialMarkdown(repopath: string, tag?: string): Promise<CachedPackage>;
cacheReposAsync(response: GHTutorialResponse): Promise<void>;
}

function ghRequestAsync(options: U.HttpRequestOptions) {
Expand Down Expand Up @@ -299,14 +300,26 @@ namespace pxt.github {
}

async loadTutorialMarkdown(repoPath: string, tag?: string) {
const tutorialResponse = await downloadMarkdownTutorialInfoAsync(repoPath, tag);
const tutorialResponse = (await downloadMarkdownTutorialInfoAsync(repoPath, tag)).resp;

this.cacheRepo(tutorialResponse.markdownRepo);
for (const dep of tutorialResponse.dependencies) {
const repo = tutorialResponse.markdown as { filename: string, repo: GHTutorialRepoInfo };

pxt.Util.assert(typeof repo === "object");

await this.cacheReposAsync(tutorialResponse);

return repo.repo;
}

async cacheReposAsync(resp: GHTutorialResponse) {
if (typeof resp.markdown === "object") {
const repo = resp.markdown as { filename: string, repo: GHTutorialRepoInfo };

this.cacheRepo(repo.repo);
}
for (const dep of resp.dependencies) {
this.cacheRepo(dep);
}

return tutorialResponse.markdownRepo;
}

private cacheRepo(repo: GHTutorialRepoInfo) {
Expand Down Expand Up @@ -373,24 +386,45 @@ namespace pxt.github {
})
}

export async function downloadMarkdownTutorialInfoAsync(repopath: string, tag?: string, noCache?: boolean): Promise<GHTutorialResponse> {
export async function downloadMarkdownTutorialInfoAsync(repopath: string, tag?: string, noCache?: boolean, etag?: string): Promise<{ resp?: GHTutorialResponse, etag?: string }> {
let request = pxt.Cloud.apiRequestWithCdnAsync;
const queryParams = new URLSearchParams();
if (tag) {
queryParams.set("ref", tag);
}
if (noCache) {
queryParams.set("noCache", "1");
request = pxt.Cloud.privateRequestAsync;
}

const apiRoot = pxt.Cloud.apiRoot;
let url = `ghtutorial/${repopath}`;
url = pxt.BrowserUtils.appendUrlQueryParams(url, queryParams);

const headers: pxt.Map<string> = etag ? { "If-None-Match": etag } : undefined;

const resp = await request(
{
url,
method: "GET",
headers
}
);

const queryString = queryParams.toString();
let url = `${apiRoot}ghtutorial/${repopath}`;
if (queryString) {
url += "?" + queryParams;
let body: GHTutorialResponse;

if (resp.statusCode === 304) {
body = undefined;
}
else {
body = resp.json;
}

return pxt.U.httpGetJsonAsync(url);
return (
{
resp: body,
etag: resp.headers["etag"] as string
}
);
}

export async function downloadTutorialMarkdownAsync(repopath: string, tag?: string) {
Expand Down
2 changes: 1 addition & 1 deletion webapp/src/app.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4692,7 +4692,7 @@ export class ProjectView
if (/^\//.test(path)) {
filename = title || path.split('/').reverse()[0].replace('-', ' '); // drop any kind of sub-paths
pxt.perf.measureStart("downloadMarkdown");
const rawMarkdown = await pxt.Cloud.markdownAsync(path);
const rawMarkdown = await pxt.Cloud.markdownAsync(path, undefined, undefined, true);
pxt.perf.measureEnd("downloadMarkdown");
autoChooseBoard = true;
markdown = processMarkdown(rawMarkdown);
Expand Down
44 changes: 35 additions & 9 deletions webapp/src/idbworkspace.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ interface GitHubCacheEntry {
config?: pxt.PackageConfig;
cacheTime?: number;
version?: string;
etag?: string;
}

const TEXTS_TABLE = "texts";
Expand Down Expand Up @@ -492,37 +493,62 @@ export function initGitHubDb() {

const existing = await cache.getAsync(id);

if (existing && Date.now() - existing.cacheTime < GITHUB_TUTORIAL_CACHE_EXPIRATION_MILLIS) {
const readFromCache = async () => {
const elements = repopath.split("/");
const repo = elements[0] + "/" + elements[1]

// everything should be in the cache already
// everything should be in the cache already, so the latest version
// and load package calls should hit the DB rather than the network
tag = tag || await this.latestVersionAsync(repo, await pxt.packagesConfigAsync());

return this.loadPackageAsync(repo, tag);
}

const tutorialResponse = await pxt.github.downloadMarkdownTutorialInfoAsync(repopath, tag);
if (existing && Date.now() - existing.cacheTime < GITHUB_TUTORIAL_CACHE_EXPIRATION_MILLIS) {
// don't bother hitting the network if we are within the expiration time
return readFromCache();
}

// fill up the cache with all of the files, versions, and configs contained in this response
await this.cacheRepoAsync(tutorialResponse.markdownRepo);
for (const dep of tutorialResponse.dependencies) {
await this.cacheRepoAsync(dep);
const tutorialResponse = await pxt.github.downloadMarkdownTutorialInfoAsync(repopath, tag, undefined, existing?.etag);

const body = tutorialResponse.resp;

if (!body) {
// etag matched, so our cache should be up to date
return readFromCache();
}

const repo = body.markdown as { filename: string, repo: pxt.github.GHTutorialRepoInfo };
pxt.Util.assert(typeof repo === "object");

// fill up the cache with all of the files, versions, and configs contained in this response
await this.cacheReposAsync(body);

// set a marker in the cache to indicate that we shouldn't need to hit /api/ghtutorial/ again
try {
await cache.setAsync({
id,
cacheTime: Date.now()
cacheTime: Date.now(),
etag: tutorialResponse.etag
});
}
catch (e) {
// ignore cache failures
pxt.debug("Failed to cache tutorial for " + repopath);
}

return tutorialResponse.markdownRepo;
return repo.repo;
}

async cacheReposAsync(resp: pxt.github.GHTutorialResponse) {
if (typeof resp.markdown === "object") {
const repo = resp.markdown as { filename: string, repo: pxt.github.GHTutorialRepoInfo };

await this.cacheRepoAsync(repo.repo);
}
for (const dep of resp.dependencies) {
await this.cacheRepoAsync(dep);
}
}

private async cacheRepoAsync(repo: pxt.github.GHTutorialRepoInfo) {
Expand Down

0 comments on commit f635ffb

Please sign in to comment.