From 86956a3720920eb0450f74c3b16e8a9b0b0657c6 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 30 Apr 2025 18:55:16 +0200 Subject: [PATCH 01/26] add create-branch --- packages/hub/src/lib/create-branch.spec.ts | 159 +++++++++++++++++++++ packages/hub/src/lib/create-branch.ts | 50 +++++++ 2 files changed, 209 insertions(+) create mode 100644 packages/hub/src/lib/create-branch.spec.ts create mode 100644 packages/hub/src/lib/create-branch.ts diff --git a/packages/hub/src/lib/create-branch.spec.ts b/packages/hub/src/lib/create-branch.spec.ts new file mode 100644 index 000000000..b616fb4ce --- /dev/null +++ b/packages/hub/src/lib/create-branch.spec.ts @@ -0,0 +1,159 @@ +import { assert, it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { createBranch } from "./create-branch"; +import { uploadFile } from "./upload-file"; +import { downloadFile } from "./download-file"; + +describe("createBranch", () => { + it("should create a new branch from the default branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "new-branch", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "new-branch", + }); + + assert.equal(await content?.text(), "file content"); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + + it("should create an empty branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "empty-branch", + empty: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "empty-branch", + }); + + assert.equal(content, null); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + + it("should overwrite an existing branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "overwrite-branch", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await createBranch({ + repo, + branch: "overwrite-branch", + overwrite: true, + empty: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "overwrite-branch", + }); + + assert.equal(content, null); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts new file mode 100644 index 000000000..072b88bfb --- /dev/null +++ b/packages/hub/src/lib/create-branch.ts @@ -0,0 +1,50 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { AccessToken, RepoDesignation } from "../types/public"; + +export async function createBranch(params: { + repo: RepoDesignation; + /** + * Revision to create the branch from. Defaults to the default branch. + * + * Use empty: true to create an empty branch. + */ + revision?: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; + /** + * The name of the branch to create + */ + branch: string; + /** + * Use this to create an empty branch, with no commits. + */ + empty?: boolean; + /** + * Use this to overwrite the branch if it already exists. + */ + overwrite?: boolean; +}): Promise { + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...(params.empty && { emptyBranch: true }), + overwrite: params.overwrite, + }), + } + ); + + if (!res.ok) { + throw await createApiError(res); + } +} From b175cf498f0b8f95a938d073b3c0a92482539f43 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 30 Apr 2025 18:55:27 +0200 Subject: [PATCH 02/26] add delete-branch --- packages/hub/src/lib/delete-branch.spec.ts | 43 ++++++++++++++++++++++ packages/hub/src/lib/delete-branch.ts | 30 +++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 packages/hub/src/lib/delete-branch.spec.ts create mode 100644 packages/hub/src/lib/delete-branch.ts diff --git a/packages/hub/src/lib/delete-branch.spec.ts b/packages/hub/src/lib/delete-branch.spec.ts new file mode 100644 index 000000000..dcd253214 --- /dev/null +++ b/packages/hub/src/lib/delete-branch.spec.ts @@ -0,0 +1,43 @@ +import { it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { createBranch } from "./create-branch"; +import { deleteBranch } from "./delete-branch"; + +describe("deleteBranch", () => { + it("should delete an existing branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await createBranch({ + repo, + branch: "branch-to-delete", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await deleteBranch({ + repo, + branch: "branch-to-delete", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/packages/hub/src/lib/delete-branch.ts b/packages/hub/src/lib/delete-branch.ts new file mode 100644 index 000000000..30cb575ae --- /dev/null +++ b/packages/hub/src/lib/delete-branch.ts @@ -0,0 +1,30 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { AccessToken, RepoDesignation } from "../types/public"; + +export async function deleteBranch(params: { + repo: RepoDesignation; + /** + * The name of the branch to delete + */ + branch: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; +}): Promise { + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + } + ); + + if (!res.ok) { + throw await createApiError(res); + } +} From 80faefbddd5792fa0e8be5014c3afef933635bd7 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 30 Apr 2025 19:00:37 +0200 Subject: [PATCH 03/26] cli file --- packages/hub/cli.ts | 285 ++++++++++++++++++++++++++++++++++ packages/hub/package.json | 3 + packages/hub/src/lib/index.ts | 2 + packages/hub/tsconfig.json | 2 +- 4 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 packages/hub/cli.ts diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts new file mode 100644 index 000000000..b7d9ff697 --- /dev/null +++ b/packages/hub/cli.ts @@ -0,0 +1,285 @@ +#! /usr/bin/env node + +import { parseArgs } from "node:util"; +import { typedEntries } from "./src/utils/typedEntries"; +import { createBranch, uploadFilesWithProgress } from "./src"; +import { pathToFileURL } from "node:url"; + +const command = process.argv[2]; +const args = process.argv.slice(3); + +type Camelize = T extends `${infer A}-${infer B}` ? `${A}${Camelize>}` : T; + +const commands = { + upload: { + description: "Upload a folder to a repo on the Hub", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to create", + positional: true, + required: true, + }, + { + name: "local-folder" as const, + description: "The local folder to upload. Defaults to the current working directory", + positional: true, + default: () => process.cwd(), + }, + // { + // name: "path-in-repo" as const, + // description: "The path in the repo to upload the folder to. Defaults to the root of the repo", + // positional: true, + // default: "/", + // }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "repo-type" as const, + short: "t", + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to upload to. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + short: "r", + description: "The revision to upload to. Defaults to the main branch", + default: "main", + }, + { + name: "from-revision" as const, + short: "c", + description: + "The revision to upload from. Defaults to the latest commit on main or on the branch if it exists.", + }, + { + name: "from-empty" as const, + short: "e", + boolean: true, + description: + "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", + }, + { + name: "token" as const, + short: "k", + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ], + }, +} satisfies Record< + string, + { + description: string; + args?: Array<{ + name: string; + short?: string; + positional?: boolean; + description?: string; + required?: boolean; + boolean?: boolean; + enum?: Array; + default?: string | (() => string); + }>; + } +>; + +type Command = keyof typeof commands; + +async function run() { + switch (command) { + case "help": { + const positionals = parseArgs({ allowPositionals: true, args }).positionals; + + if (positionals.length > 0 && positionals[0] in commands) { + const commandName = positionals[0] as Command; + console.log(detailedUsage(commandName)); + break; + } + + console.log( + `Available commands\n\n` + + typedEntries(commands) + .map(([name, { description }]) => `- ${usage(name)}: ${description}`) + .join("\n") + ); + break; + } + + case "upload": { + if (args[1] === "--help" || args[1] === "-h") { + console.log(usage("upload")); + break; + } + const parsedArgs = advParseArgs(args, "upload"); + const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet } = parsedArgs; + + if (revision && (fromEmpty || fromRevision)) { + await createBranch({ + branch: revision, + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + accessToken: token, + revision: fromRevision, + empty: fromEmpty ? true : undefined, + overwrite: true, + }); + } + + for await (const event of uploadFilesWithProgress({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + files: [pathToFileURL(localFolder)], + branch: revision, + accessToken: token, + })) { + if (!quiet) { + console.log(event); + } + } + break; + } + default: + throw new Error("Command not found: " + command); + } +} +run(); + +function usage(commandName: Command) { + const command = commands[commandName]; + + return `${commandName} ${(command.args || []) + .map((arg) => { + if (arg.positional) { + if (arg.required) { + return `<${arg.name}>`; + } else { + return `[${arg.name}]`; + } + } + return `[--${arg.name} ${arg.enum ? `{${arg.enum.join(",")}}` : arg.name.toLocaleUpperCase()}]`; + }) + .join("")}`.trim(); +} + +function detailedUsage(commandName: Command) { + let ret = `usage: ${usage(commandName)}\n\n`; + const command = commands[commandName]; + + if (command.args.some((p) => p.positional)) { + ret += `Positional arguments:\n`; + + for (const arg of command.args) { + if (arg.positional) { + ret += ` ${arg.name}: ${arg.description}\n`; + } + } + + ret += `\n`; + } + + if (command.args.some((p) => !p.positional)) { + ret += `Options:\n`; + + for (const arg of command.args) { + if (!arg.positional) { + ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; + } + } + + ret += `\n`; + } + + return ret; +} + +function advParseArgs( + args: string[], + commandName: C +): { + // Todo : better typing + [key in Camelize<(typeof commands)[C]["args"][number]["name"]>]: string; +} { + const { tokens } = parseArgs({ + options: Object.fromEntries( + commands[commandName].args + .filter((arg) => !arg.positional) + .map((arg) => { + const option = { + name: arg.name, + short: arg.short, + type: arg.boolean ? "boolean" : "string", + } as const; + return [arg.name, option]; + }) + ), + args, + allowPositionals: true, + strict: false, + tokens: true, + }); + + const command = commands[commandName]; + const expectedPositionals = command.args.filter((arg) => arg.positional); + const requiredPositionals = expectedPositionals.filter((arg) => arg.required).length; + const providedPositionals = tokens.filter((token) => token.kind === "positional").length; + + if (providedPositionals < requiredPositionals) { + throw new Error( + `Missing required positional arguments. Expected: ${requiredPositionals}, Provided: ${providedPositionals}` + ); + } + + if (providedPositionals > expectedPositionals.length) { + throw new Error( + `Too many positional arguments. Expected: ${expectedPositionals.length}, Provided: ${providedPositionals}` + ); + } + + const positionals = Object.fromEntries( + tokens.filter((token) => token.kind === "positional").map((token, i) => [expectedPositionals[i].name, token.value]) + ); + + const options = Object.fromEntries( + tokens + .filter((token) => token.kind === "option") + .map((token) => { + const arg = command.args.find((arg) => arg.name === token.name || arg.short === token.name); + if (!arg) { + throw new Error(`Unknown option: ${token.name}`); + } + + if (!token.value) { + throw new Error(`Missing value for option: ${token.name}`); + } + + if (arg.enum && !arg.enum.includes(token.value)) { + throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); + } + + return [arg.name, arg.boolean ? true : token.value]; + }) + ); + const defaults = Object.fromEntries( + command.args + .filter((arg) => arg.default) + .map((arg) => { + const value = typeof arg.default === "function" ? arg.default() : arg.default; + return [arg.name, value]; + }) + ); + return Object.fromEntries( + Object.entries({ ...defaults, ...positionals, ...options }).map(([name, val]) => [kebabToCamelCase(name), val]) + ) as { + [key in Camelize<(typeof commands)[C]["args"][number]["name"]>]: string; + }; +} + +function kebabToCamelCase(str: string) { + return str.replace(/-./g, (match) => match[1].toUpperCase()); +} diff --git a/packages/hub/package.json b/packages/hub/package.json index 15d2e31c8..cbe58ac21 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -57,6 +57,9 @@ "hugging", "face" ], + "bin": { + "@huggingface/hub": "./dist/cli.js" + }, "author": "Hugging Face", "license": "MIT", "devDependencies": { diff --git a/packages/hub/src/lib/index.ts b/packages/hub/src/lib/index.ts index 24e239bdc..d4b771f2b 100644 --- a/packages/hub/src/lib/index.ts +++ b/packages/hub/src/lib/index.ts @@ -3,7 +3,9 @@ export * from "./check-repo-access"; export * from "./commit"; export * from "./count-commits"; export * from "./create-repo"; +export * from "./create-branch"; export * from "./dataset-info"; +export * from "./delete-branch"; export * from "./delete-file"; export * from "./delete-files"; export * from "./delete-repo"; diff --git a/packages/hub/tsconfig.json b/packages/hub/tsconfig.json index 254606a30..9dd335c6b 100644 --- a/packages/hub/tsconfig.json +++ b/packages/hub/tsconfig.json @@ -15,6 +15,6 @@ "declaration": true, "declarationMap": true }, - "include": ["src", "index.ts"], + "include": ["src", "index.ts", "cli.ts"], "exclude": ["dist"] } From 28f1050001b033c43675380c55d7e27b2d2afa58 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 14:43:56 +0200 Subject: [PATCH 04/26] document cli & name it to 'hfx' --- packages/hub/README.md | 26 ++++++++++++++++++++++++++ packages/hub/package.json | 2 +- packages/hub/src/lib/create-branch.ts | 2 ++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index 0633f0627..eda66aad6 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -93,6 +93,32 @@ for await (const fileInfo of hub.listFiles({repo})) { await hub.deleteRepo({ repo, accessToken: "hf_..." }); ``` +## CLI usage + +You can use `@huggingface/hub` in CLI mode to upload files and folders to your repo. + +```console +npx @huggingface/hub upload coyotte508/test-model . +npx @huggingface/hub upload datasets/coyotte508/test-dataset . +# Same thing +npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . +# Upload new data with 0 history in a separate branch +npx @huggingface/hub upload coyotte508/test-model . --revision release --empty + +npx @huggingface/hub --help +npx @huggingface/hub upload --help +``` + +You can also instal globally with `npm install -g @huggingface/hub`. Then you can do: + +```console +hfx upload coyotte508/test-model . +hfx upload --repo-type dataset coyotte508/test-dataset . --revision release --empty + +hfx --help +hfx upload --help +``` + ## OAuth Login It's possible to login using OAuth (["Sign in with HF"](https://huggingface.co/docs/hub/oauth)). diff --git a/packages/hub/package.json b/packages/hub/package.json index 1e322145a..a7b082bd7 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "@huggingface/hub": "./dist/cli.js" + "hfx": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts index 072b88bfb..e47a0e0c2 100644 --- a/packages/hub/src/lib/create-branch.ts +++ b/packages/hub/src/lib/create-branch.ts @@ -23,6 +23,8 @@ export async function createBranch(params: { empty?: boolean; /** * Use this to overwrite the branch if it already exists. + * + * If you only specify `overwrite` and no `revision`/`empty`, and the branch already exists, it will be a no-op. */ overwrite?: boolean; }): Promise { From e886d5448f3ac29bbe2d2e4da2f4551f7cd8db91 Mon Sep 17 00:00:00 2001 From: "Eliott C." Date: Tue, 6 May 2025 15:01:49 +0200 Subject: [PATCH 05/26] Update packages/hub/package.json Co-authored-by: Julien Chaumond --- packages/hub/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/package.json b/packages/hub/package.json index a7b082bd7..67bc12009 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "hfx": "./dist/cli.js" + "hf": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", From dda33e70ee050dc8982f29d09c1c0c42ddb5910d Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:21:09 +0200 Subject: [PATCH 06/26] fix TS --- packages/hub/cli.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index b7d9ff697..fb090431e 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -5,6 +5,18 @@ import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; +// Didn't find the import from "node:util", so duplicated it here +type OptionToken = + | { kind: "option"; index: number; name: string; rawName: string; value: string; inlineValue: boolean } + | { + kind: "option"; + index: number; + name: string; + rawName: string; + value: undefined; + inlineValue: undefined; + }; + const command = process.argv[2]; const args = process.argv.slice(3); @@ -242,12 +254,14 @@ function advParseArgs( } const positionals = Object.fromEntries( - tokens.filter((token) => token.kind === "positional").map((token, i) => [expectedPositionals[i].name, token.value]) + tokens + .filter((token): token is { kind: "positional"; index: number; value: string } => token.kind === "positional") + .map((token, i) => [expectedPositionals[i].name, token.value]) ); const options = Object.fromEntries( tokens - .filter((token) => token.kind === "option") + .filter((token): token is OptionToken => token.kind === "option") .map((token) => { const arg = command.args.find((arg) => arg.name === token.name || arg.short === token.name); if (!arg) { From 0de8bea4cac2292536782ea66166bf3cd550596a Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:21:36 +0200 Subject: [PATCH 07/26] switch back to hfx for now @julien-c, see slack --- packages/hub/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/package.json b/packages/hub/package.json index 67bc12009..a7b082bd7 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "hf": "./dist/cli.js" + "hfx": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", From 13569ac5d71c1bfb0cf15a35f9874b37775e6002 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:26:10 +0200 Subject: [PATCH 08/26] fix repo param in url --- packages/hub/src/lib/create-branch.ts | 3 ++- packages/hub/src/lib/delete-branch.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts index e47a0e0c2..fb80103d8 100644 --- a/packages/hub/src/lib/create-branch.ts +++ b/packages/hub/src/lib/create-branch.ts @@ -1,6 +1,7 @@ import { HUB_URL } from "../consts"; import { createApiError } from "../error"; import type { AccessToken, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; export async function createBranch(params: { repo: RepoDesignation; @@ -29,7 +30,7 @@ export async function createBranch(params: { overwrite?: boolean; }): Promise { const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, { method: "POST", headers: { diff --git a/packages/hub/src/lib/delete-branch.ts b/packages/hub/src/lib/delete-branch.ts index 30cb575ae..4c8b0bda5 100644 --- a/packages/hub/src/lib/delete-branch.ts +++ b/packages/hub/src/lib/delete-branch.ts @@ -1,6 +1,7 @@ import { HUB_URL } from "../consts"; import { createApiError } from "../error"; import type { AccessToken, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; export async function deleteBranch(params: { repo: RepoDesignation; @@ -13,7 +14,7 @@ export async function deleteBranch(params: { fetch?: typeof fetch; }): Promise { const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, { method: "DELETE", headers: { From 618bbce325a7861b40ef4ca94838b2413850792a Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:30:07 +0200 Subject: [PATCH 09/26] fixup! fix repo param in url --- packages/hub/src/lib/create-branch.ts | 3 ++- packages/hub/src/lib/delete-branch.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts index fb80103d8..100e4d1b9 100644 --- a/packages/hub/src/lib/create-branch.ts +++ b/packages/hub/src/lib/create-branch.ts @@ -29,8 +29,9 @@ export async function createBranch(params: { */ overwrite?: boolean; }): Promise { + const repoId = toRepoId(params.repo); const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, { method: "POST", headers: { diff --git a/packages/hub/src/lib/delete-branch.ts b/packages/hub/src/lib/delete-branch.ts index 4c8b0bda5..70227b185 100644 --- a/packages/hub/src/lib/delete-branch.ts +++ b/packages/hub/src/lib/delete-branch.ts @@ -13,8 +13,9 @@ export async function deleteBranch(params: { accessToken?: AccessToken; fetch?: typeof fetch; }): Promise { + const repoId = toRepoId(params.repo); const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, { method: "DELETE", headers: { From d982af9a46dd422c8144c80bf74f9e5440b27ea6 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:39:53 +0200 Subject: [PATCH 10/26] hfjs --- packages/hub/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/package.json b/packages/hub/package.json index a7b082bd7..022653e6f 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "hfx": "./dist/cli.js" + "hfjs": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", From ef81d23f7f48dee6a0f22b797e8099cb0a1ae592 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:41:30 +0200 Subject: [PATCH 11/26] fixup! hfjs --- packages/hub/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index eda66aad6..6d2668ba0 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -112,11 +112,11 @@ npx @huggingface/hub upload --help You can also instal globally with `npm install -g @huggingface/hub`. Then you can do: ```console -hfx upload coyotte508/test-model . -hfx upload --repo-type dataset coyotte508/test-dataset . --revision release --empty +hfjs upload coyotte508/test-model . +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --empty -hfx --help -hfx upload --help +hfjs --help +hfjs upload --help ``` ## OAuth Login From f9a57471120958b3b9c943d110a2ffdf8e805a91 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:48:18 +0200 Subject: [PATCH 12/26] doc --- packages/hub/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index 6d2668ba0..80808f987 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -103,7 +103,7 @@ npx @huggingface/hub upload datasets/coyotte508/test-dataset . # Same thing npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . # Upload new data with 0 history in a separate branch -npx @huggingface/hub upload coyotte508/test-model . --revision release --empty +npx @huggingface/hub upload coyotte508/test-model . --revision release --from-empty npx @huggingface/hub --help npx @huggingface/hub upload --help @@ -113,7 +113,7 @@ You can also instal globally with `npm install -g @huggingface/hub`. Then you ca ```console hfjs upload coyotte508/test-model . -hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --empty +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --from-empty hfjs --help hfjs upload --help From fa312489b90ca40cc3bf1b288e76d6a16c559c68 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:59:57 +0200 Subject: [PATCH 13/26] remove all shorthands (except 'q') --- packages/hub/cli.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index fb090431e..28bfd09fc 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -52,7 +52,6 @@ const commands = { }, { name: "repo-type" as const, - short: "t", enum: ["dataset", "model", "space"], default: "model", description: @@ -60,26 +59,22 @@ const commands = { }, { name: "revision" as const, - short: "r", description: "The revision to upload to. Defaults to the main branch", default: "main", }, { name: "from-revision" as const, - short: "c", description: "The revision to upload from. Defaults to the latest commit on main or on the branch if it exists.", }, { name: "from-empty" as const, - short: "e", boolean: true, description: "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", }, { name: "token" as const, - short: "k", description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, From 0fa9af71383a2a4e8b0cd6db1c8cc89d90b8d8f8 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 16:32:41 +0200 Subject: [PATCH 14/26] cli fixes --- package.json | 3 +++ packages/hub/cli.ts | 15 ++++++++++++--- packages/hub/tsup.config.ts | 5 +++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 0cbf7b922..36fa9dab5 100644 --- a/package.json +++ b/package.json @@ -30,5 +30,8 @@ "vite": "^5.0.2", "vitest": "^0.34.6", "webdriverio": "^8.6.7" + }, + "dependencies": { + "@huggingface/hub": "link:packages/hub" } } diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 28bfd09fc..84237a70c 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -102,6 +102,8 @@ type Command = keyof typeof commands; async function run() { switch (command) { + case undefined: + case "--help": case "help": { const positionals = parseArgs({ allowPositionals: true, args }).positionals; @@ -117,11 +119,17 @@ async function run() { .map(([name, { description }]) => `- ${usage(name)}: ${description}`) .join("\n") ); + + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + + if (command === undefined) { + process.exitCode = 1; + } break; } case "upload": { - if (args[1] === "--help" || args[1] === "-h") { + if (args[0] === "--help" || args[0] === "-h") { console.log(usage("upload")); break; } @@ -171,7 +179,7 @@ function usage(commandName: Command) { } return `[--${arg.name} ${arg.enum ? `{${arg.enum.join(",")}}` : arg.name.toLocaleUpperCase()}]`; }) - .join("")}`.trim(); + .join(" ")}`.trim(); } function detailedUsage(commandName: Command) { @@ -219,8 +227,9 @@ function advParseArgs( .map((arg) => { const option = { name: arg.name, - short: arg.short, + ...(arg.short && { short: arg.short }), type: arg.boolean ? "boolean" : "string", + default: typeof arg.default === "function" ? arg.default() : arg.default, } as const; return [arg.name, option]; }) diff --git a/packages/hub/tsup.config.ts b/packages/hub/tsup.config.ts index 6be4e128a..adbb9fdfb 100644 --- a/packages/hub/tsup.config.ts +++ b/packages/hub/tsup.config.ts @@ -1,14 +1,15 @@ import type { Options } from "tsup"; -const baseConfig: Options = { +const baseConfig = { entry: ["./index.ts"], format: ["cjs", "esm"], outDir: "dist", clean: true, -}; +} satisfies Options; const nodeConfig: Options = { ...baseConfig, + entry: [...baseConfig.entry, "./cli.ts"], platform: "node", }; From 23498e0aa778fb8d1540bd0b76c1eba3e852ac48 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 16:47:12 +0200 Subject: [PATCH 15/26] cli fixes --- packages/hub/cli.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 84237a70c..dfb5f416b 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -73,6 +73,10 @@ const commands = { description: "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", }, + { + name: "commit-message" as const, + description: "The commit message to use. Defaults to 'Add [x] files'", + }, { name: "token" as const, description: @@ -134,7 +138,8 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet } = parsedArgs; + const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet, commitMessage } = + parsedArgs; if (revision && (fromEmpty || fromRevision)) { await createBranch({ @@ -152,6 +157,8 @@ async function run() { files: [pathToFileURL(localFolder)], branch: revision, accessToken: token, + commitTitle: commitMessage?.trim().split("\n")[0], + commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), })) { if (!quiet) { console.log(event); @@ -272,12 +279,14 @@ function advParseArgs( throw new Error(`Unknown option: ${token.name}`); } - if (!token.value) { - throw new Error(`Missing value for option: ${token.name}`); - } + if (!arg.boolean) { + if (!token.value) { + throw new Error(`Missing value for option: ${token.name}: ${JSON.stringify(token)}`); + } - if (arg.enum && !arg.enum.includes(token.value)) { - throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); + if (arg.enum && !arg.enum.includes(token.value)) { + throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); + } } return [arg.name, arg.boolean ? true : token.value]; From f735cfa8514463ae9315f1c6b1300614151f739d Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 16:59:27 +0200 Subject: [PATCH 16/26] fixup! cli fixes --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index 36fa9dab5..0cbf7b922 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,5 @@ "vite": "^5.0.2", "vitest": "^0.34.6", "webdriverio": "^8.6.7" - }, - "dependencies": { - "@huggingface/hub": "link:packages/hub" } } From f831b3cd6e4750f7bf9f74f5b207a658e4362be4 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 17:21:14 +0200 Subject: [PATCH 17/26] add hidden hub-url CLI param --- packages/hub/cli.ts | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index dfb5f416b..0391a2635 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -4,6 +4,7 @@ import { parseArgs } from "node:util"; import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; +import { HUB_URL } from "./src/consts"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -83,6 +84,13 @@ const commands = { "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, }, + { + name: "hub-url" as const, + description: + "The URL of the Hub to upload to. Defaults to https://huggingface.co. Use this to upload to a private Hub.", + hidden: true, + default: HUB_URL, + }, ], }, } satisfies Record< @@ -97,6 +105,7 @@ const commands = { required?: boolean; boolean?: boolean; enum?: Array; + hidden?: boolean; default?: string | (() => string); }>; } @@ -138,8 +147,18 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet, commitMessage } = - parsedArgs; + const { + repoName, + localFolder, + repoType, + revision, + fromEmpty, + fromRevision, + token, + quiet, + commitMessage, + hubUrl, + } = parsedArgs; if (revision && (fromEmpty || fromRevision)) { await createBranch({ @@ -149,6 +168,7 @@ async function run() { revision: fromRevision, empty: fromEmpty ? true : undefined, overwrite: true, + hubUrl, }); } @@ -159,6 +179,7 @@ async function run() { accessToken: token, commitTitle: commitMessage?.trim().split("\n")[0], commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), + hubUrl, })) { if (!quiet) { console.log(event); @@ -176,6 +197,7 @@ function usage(commandName: Command) { const command = commands[commandName]; return `${commandName} ${(command.args || []) + .filter((arg) => !arg.hidden) .map((arg) => { if (arg.positional) { if (arg.required) { @@ -205,11 +227,11 @@ function detailedUsage(commandName: Command) { ret += `\n`; } - if (command.args.some((p) => !p.positional)) { + if (command.args.some((p) => !p.positional && !p.hidden)) { ret += `Options:\n`; for (const arg of command.args) { - if (!arg.positional) { + if (!arg.positional && !arg.hidden) { ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; } } From 3aacda1bcf832ea20ebd7125935d9b5c2ac44705 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 17:30:33 +0200 Subject: [PATCH 18/26] fix uploading to new rev not from empty --- packages/hub/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 0391a2635..c880a9954 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -160,7 +160,7 @@ async function run() { hubUrl, } = parsedArgs; - if (revision && (fromEmpty || fromRevision)) { + if (revision && revision !== "main") { await createBranch({ branch: revision, repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, From 4aefa05997f4a12b2fa0fa3fa36333cb194a1867 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 15:10:20 +0200 Subject: [PATCH 19/26] Also support pathInRepo positional arg --- packages/hub/cli.ts | 27 ++++++++++++++++++++------- packages/hub/src/utils/createBlobs.ts | 5 ++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index c880a9954..378411542 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -5,6 +5,8 @@ import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; import { HUB_URL } from "./src/consts"; +import { stat } from "node:fs/promises"; +import { basename, join } from "node:path"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -39,12 +41,12 @@ const commands = { positional: true, default: () => process.cwd(), }, - // { - // name: "path-in-repo" as const, - // description: "The path in the repo to upload the folder to. Defaults to the root of the repo", - // positional: true, - // default: "/", - // }, + { + name: "path-in-repo" as const, + description: "The path in the repo to upload the folder to. Defaults to the root of the repo", + positional: true, + default: ".", + }, { name: "quiet" as const, short: "q", @@ -158,6 +160,7 @@ async function run() { quiet, commitMessage, hubUrl, + pathInRepo, } = parsedArgs; if (revision && revision !== "main") { @@ -172,9 +175,19 @@ async function run() { }); } + const isFile = (await stat(localFolder)).isFile(); + const files = isFile + ? [ + { + content: pathToFileURL(localFolder), + path: join(pathInRepo, `${basename(localFolder)}`).replace(/^[.]?\//, ""), + }, + ] + : [{ content: pathToFileURL(localFolder), path: pathInRepo.replace(/^[.]?\//, "") }]; + for await (const event of uploadFilesWithProgress({ repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, - files: [pathToFileURL(localFolder)], + files, branch: revision, accessToken: token, commitTitle: commitMessage?.trim().split("\n")[0], diff --git a/packages/hub/src/utils/createBlobs.ts b/packages/hub/src/utils/createBlobs.ts index 1a261c4c4..625bef5fd 100644 --- a/packages/hub/src/utils/createBlobs.ts +++ b/packages/hub/src/utils/createBlobs.ts @@ -38,7 +38,10 @@ export async function createBlobs( return Promise.all( paths.map(async (path) => ({ - path: `${destPath}/${path.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/"), + path: `${destPath}/${path.relativePath}` + .replace(/\/[.]$/, "") + .replaceAll("//", "/") + .replace(/^[.]?\//, ""), blob: await FileBlob.create(new URL(path.path)), })) ); From 03e615dd74dc4a7fa962ce786b8449dda39ac4ba Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:30:08 +0200 Subject: [PATCH 20/26] Add create-branch to the cli --- packages/hub/README.md | 7 +- packages/hub/cli.ts | 141 ++++++++++++++++++++++------------------- 2 files changed, 81 insertions(+), 67 deletions(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index 80808f987..625cd5e4e 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -103,7 +103,8 @@ npx @huggingface/hub upload datasets/coyotte508/test-dataset . # Same thing npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . # Upload new data with 0 history in a separate branch -npx @huggingface/hub upload coyotte508/test-model . --revision release --from-empty +npx @huggingface/hub create-branch coyotte508/test-model release --empty +npx @huggingface/hub upload coyotte508/test-model . --revision release npx @huggingface/hub --help npx @huggingface/hub upload --help @@ -113,7 +114,9 @@ You can also instal globally with `npm install -g @huggingface/hub`. Then you ca ```console hfjs upload coyotte508/test-model . -hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --from-empty + +hfjs create-branch --repo-type dataset coyotte508/test-dataset release --empty +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release hfjs --help hfjs upload --help diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 378411542..ca33ebb6b 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -4,9 +4,9 @@ import { parseArgs } from "node:util"; import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; -import { HUB_URL } from "./src/consts"; import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; +import { HUB_URL } from "./src/consts"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -25,6 +25,17 @@ const args = process.argv.slice(3); type Camelize = T extends `${infer A}-${infer B}` ? `${A}${Camelize>}` : T; +interface ArgDef { + name: string; + short?: string; + positional?: boolean; + description?: string; + required?: boolean; + boolean?: boolean; + enum?: Array; + default?: string | (() => string); +} + const commands = { upload: { description: "Upload a folder to a repo on the Hub", @@ -65,17 +76,6 @@ const commands = { description: "The revision to upload to. Defaults to the main branch", default: "main", }, - { - name: "from-revision" as const, - description: - "The revision to upload from. Defaults to the latest commit on main or on the branch if it exists.", - }, - { - name: "from-empty" as const, - boolean: true, - description: - "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", - }, { name: "commit-message" as const, description: "The commit message to use. Defaults to 'Add [x] files'", @@ -86,30 +86,48 @@ const commands = { "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, }, + ], + }, + "create-branch": { + description: "Create a new branch in a repo, or update an existing one", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to create", + positional: true, + required: true, + }, + { + name: "branch" as const, + description: "The name of the branch to create", + positional: true, + required: true, + }, { - name: "hub-url" as const, + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to create. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, description: - "The URL of the Hub to upload to. Defaults to https://huggingface.co. Use this to upload to a private Hub.", - hidden: true, - default: HUB_URL, + "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists.", + default: "main", + }, + { + name: "empty" as const, + boolean: true, + description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", }, ], - }, + } as const, } satisfies Record< string, { description: string; - args?: Array<{ - name: string; - short?: string; - positional?: boolean; - description?: string; - required?: boolean; - boolean?: boolean; - enum?: Array; - hidden?: boolean; - default?: string | (() => string); - }>; + args?: ArgDef[]; } >; @@ -149,31 +167,7 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { - repoName, - localFolder, - repoType, - revision, - fromEmpty, - fromRevision, - token, - quiet, - commitMessage, - hubUrl, - pathInRepo, - } = parsedArgs; - - if (revision && revision !== "main") { - await createBranch({ - branch: revision, - repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, - accessToken: token, - revision: fromRevision, - empty: fromEmpty ? true : undefined, - overwrite: true, - hubUrl, - }); - } + const { repoName, localFolder, repoType, revision, token, quiet, commitMessage, pathInRepo } = parsedArgs; const isFile = (await stat(localFolder)).isFile(); const files = isFile @@ -192,7 +186,7 @@ async function run() { accessToken: token, commitTitle: commitMessage?.trim().split("\n")[0], commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), - hubUrl, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, })) { if (!quiet) { console.log(event); @@ -200,6 +194,24 @@ async function run() { } break; } + case "create-branch": { + if (args[0] === "--help" || args[0] === "-h") { + console.log(usage("create-branch")); + break; + } + const parsedArgs = advParseArgs(args, "create-branch"); + const { repoName, branch, revision, empty, repoType } = parsedArgs; + + await createBranch({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + branch, + accessToken: process.env.HF_TOKEN, + revision, + empty: empty ? true : undefined, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + }); + break; + } default: throw new Error("Command not found: " + command); } @@ -209,8 +221,7 @@ run(); function usage(commandName: Command) { const command = commands[commandName]; - return `${commandName} ${(command.args || []) - .filter((arg) => !arg.hidden) + return `${commandName} ${((command.args as ArgDef[]) || []) .map((arg) => { if (arg.positional) { if (arg.required) { @@ -228,10 +239,10 @@ function detailedUsage(commandName: Command) { let ret = `usage: ${usage(commandName)}\n\n`; const command = commands[commandName]; - if (command.args.some((p) => p.positional)) { + if ((command.args as ArgDef[]).some((p) => p.positional)) { ret += `Positional arguments:\n`; - for (const arg of command.args) { + for (const arg of command.args as ArgDef[]) { if (arg.positional) { ret += ` ${arg.name}: ${arg.description}\n`; } @@ -240,11 +251,11 @@ function detailedUsage(commandName: Command) { ret += `\n`; } - if (command.args.some((p) => !p.positional && !p.hidden)) { + if ((command.args as ArgDef[]).some((p) => !p.positional)) { ret += `Options:\n`; - for (const arg of command.args) { - if (!arg.positional && !arg.hidden) { + for (const arg of command.args as ArgDef[]) { + if (!arg.positional) { ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; } } @@ -264,7 +275,7 @@ function advParseArgs( } { const { tokens } = parseArgs({ options: Object.fromEntries( - commands[commandName].args + (commands[commandName].args as ArgDef[]) .filter((arg) => !arg.positional) .map((arg) => { const option = { @@ -283,7 +294,7 @@ function advParseArgs( }); const command = commands[commandName]; - const expectedPositionals = command.args.filter((arg) => arg.positional); + const expectedPositionals = (command.args as ArgDef[]).filter((arg) => arg.positional); const requiredPositionals = expectedPositionals.filter((arg) => arg.required).length; const providedPositionals = tokens.filter((token) => token.kind === "positional").length; @@ -309,7 +320,7 @@ function advParseArgs( tokens .filter((token): token is OptionToken => token.kind === "option") .map((token) => { - const arg = command.args.find((arg) => arg.name === token.name || arg.short === token.name); + const arg = (command.args as ArgDef[]).find((arg) => arg.name === token.name || arg.short === token.name); if (!arg) { throw new Error(`Unknown option: ${token.name}`); } @@ -328,7 +339,7 @@ function advParseArgs( }) ); const defaults = Object.fromEntries( - command.args + (commands[commandName].args as ArgDef[]) .filter((arg) => arg.default) .map((arg) => { const value = typeof arg.default === "function" ? arg.default() : arg.default; From 0dec5a0e622de917c4ea66644eef5c328b5817ca Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:35:24 +0200 Subject: [PATCH 21/26] token param too --- packages/hub/cli.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index ca33ebb6b..6525cf151 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -121,6 +121,12 @@ const commands = { boolean: true, description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, ], } as const, } satisfies Record< @@ -200,12 +206,12 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "create-branch"); - const { repoName, branch, revision, empty, repoType } = parsedArgs; + const { repoName, branch, revision, empty, repoType, token } = parsedArgs; await createBranch({ repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, branch, - accessToken: process.env.HF_TOKEN, + accessToken: token, revision, empty: empty ? true : undefined, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, From 3b84137c11f6d7aaff5c16532bfe449dd6a4e7a3 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:45:43 +0200 Subject: [PATCH 22/26] better usage text --- packages/hub/cli.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 6525cf151..378c93396 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -236,7 +236,9 @@ function usage(commandName: Command) { return `[${arg.name}]`; } } - return `[--${arg.name} ${arg.enum ? `{${arg.enum.join(",")}}` : arg.name.toLocaleUpperCase()}]`; + return `[--${arg.name}${ + arg.enum ? ` {${arg.enum.join(",")}}` : arg.boolean ? "" : " " + arg.name.toLocaleUpperCase() + }]`; }) .join(" ")}`.trim(); } @@ -262,7 +264,9 @@ function detailedUsage(commandName: Command) { for (const arg of command.args as ArgDef[]) { if (!arg.positional) { - ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; + ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}${ + arg.enum ? ` {${arg.enum.join(",")}}` : arg.boolean ? "" : " " + arg.name.toLocaleUpperCase() + }: ${arg.description}\n`; } } From 43aae0441935c90a663193a52717b0a98e58f77e Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:50:24 +0200 Subject: [PATCH 23/26] Add --force param to create-branch --- packages/hub/cli.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 378c93396..49f99280d 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -121,6 +121,13 @@ const commands = { boolean: true, description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", }, + { + name: "force" as const, + short: "f", + boolean: true, + description: + "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists.", + }, { name: "token" as const, description: @@ -206,7 +213,7 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "create-branch"); - const { repoName, branch, revision, empty, repoType, token } = parsedArgs; + const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs; await createBranch({ repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, @@ -214,6 +221,7 @@ async function run() { accessToken: token, revision, empty: empty ? true : undefined, + overwrite: force ? true : undefined, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, }); break; From e25930ede87e8197a70822d1aa3dcfc7079285ce Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:52:08 +0200 Subject: [PATCH 24/26] fixup! Add --force param to create-branch --- packages/hub/cli.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 49f99280d..2809bff79 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -114,7 +114,6 @@ const commands = { name: "revision" as const, description: "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists.", - default: "main", }, { name: "empty" as const, From be57754380a8da1c0e83b480a5b13a65575e7f86 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:54:55 +0200 Subject: [PATCH 25/26] more detailed usage --- packages/hub/cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 2809bff79..20d9047b7 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -175,7 +175,7 @@ async function run() { case "upload": { if (args[0] === "--help" || args[0] === "-h") { - console.log(usage("upload")); + console.log(detailedUsage("upload")); break; } const parsedArgs = advParseArgs(args, "upload"); @@ -208,7 +208,7 @@ async function run() { } case "create-branch": { if (args[0] === "--help" || args[0] === "-h") { - console.log(usage("create-branch")); + console.log(detailedUsage("create-branch")); break; } const parsedArgs = advParseArgs(args, "create-branch"); From 8f96728c90299eaaa47b48aec8179dfeca5a00b7 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 23:28:08 +0200 Subject: [PATCH 26/26] update repo name description --- packages/hub/cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 20d9047b7..00935645c 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -42,7 +42,7 @@ const commands = { args: [ { name: "repo-name" as const, - description: "The name of the repo to create", + description: "The name of the repo to upload to", positional: true, required: true, }, @@ -93,7 +93,7 @@ const commands = { args: [ { name: "repo-name" as const, - description: "The name of the repo to create", + description: "The name of the repo to create the branch in", positional: true, required: true, },