-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add
withConsolidated store helper (#119)
- Loading branch information
Showing
10 changed files
with
163 additions
and
141 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| --- | ||
| "@zarrita/indexing": minor | ||
| "zarrita": minor | ||
| "@zarrita/core": minor | ||
| --- | ||
|
|
||
| feat: Add `withConsolidated` store utility | ||
|
|
||
| **BREAKING**: Replaces [`openConsolidated`](https://github.com/manzt/zarrita.js/pull/91) | ||
| to provide a consistent interface for accessing consolidated and non-consolidated stores. | ||
|
|
||
| ```javascript | ||
| import * as zarr from "zarrita"; | ||
|
|
||
| // non-consolidated | ||
| let store = new zarr.FetchStore("https://localhost:8080/data.zarr"); | ||
| let grp = await zarr.open(store); // network request for .zgroup/.zattrs | ||
| let foo = await zarr.open(grp.resolve("/foo"), { kind: array }); // network request for .zarray/.zattrs | ||
|
|
||
| // consolidated | ||
| let store = new zarr.FetchStore("https://localhost:8080/data.zarr"); | ||
| let consolidatedStore = await zarr.withConsolidated(store); // opens ./zmetadata | ||
| let contents = consolidatedStore.contents(); // [ {path: "/", kind: "group" }, { path: "/foo", kind: "array" }, ...] | ||
| let grp = await zarr.open(consolidatedStore); // no network request | ||
| let foo = await zarr.open(grp.resolve(contents[1].path), { | ||
| kind: contents[1].kind, | ||
| }); // no network request | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,102 +1,102 @@ | ||
| import type { AbsolutePath, Readable } from "@zarrita/storage"; | ||
|
|
||
| import { Array, Group, Location } from "./hierarchy.js"; | ||
| import { | ||
| json_decode_object, | ||
| v2_to_v3_array_metadata, | ||
| v2_to_v3_group_metadata, | ||
| } from "./util.js"; | ||
| import type { ArrayMetadataV2, DataType, GroupMetadataV2 } from "./metadata.js"; | ||
| import { NodeNotFoundError } from "./errors.js"; | ||
| import { type AbsolutePath, type Readable } from "@zarrita/storage"; | ||
| import { json_decode_object, json_encode_object } from "./util.js"; | ||
| import { KeyError, NodeNotFoundError } from "./errors.js"; | ||
| import type { | ||
| ArrayMetadata, | ||
| ArrayMetadataV2, | ||
| Attributes, | ||
| GroupMetadata, | ||
| GroupMetadataV2, | ||
| } from "./metadata.js"; | ||
|
|
||
| type ConsolidatedMetadata = { | ||
| metadata: Record<string, any>; | ||
| metadata: Record<string, ArrayMetadataV2 | GroupMetadataV2>; | ||
| zarr_consolidated_format: 1; | ||
| }; | ||
|
|
||
| type Listable<Store extends Readable> = { | ||
| get: Store["get"]; | ||
| contents(): { path: AbsolutePath; kind: "array" | "group" }[]; | ||
| }; | ||
|
|
||
| async function get_consolidated_metadata( | ||
| store: Readable, | ||
| ): Promise<ConsolidatedMetadata> { | ||
| let bytes = await store.get("/.zmetadata"); | ||
| if (!bytes) throw new Error("No consolidated metadata found."); | ||
| if (!bytes) { | ||
| throw new NodeNotFoundError("v2 consolidated metadata", { | ||
| cause: new KeyError("/.zmetadata"), | ||
| }); | ||
| } | ||
| let meta: ConsolidatedMetadata = json_decode_object(bytes); | ||
| if (meta.zarr_consolidated_format !== 1) { | ||
| throw new Error("Unsupported consolidated format."); | ||
| } | ||
| return meta; | ||
| } | ||
|
|
||
| /** Proxies requests to the underlying store. */ | ||
| export async function openConsolidated<Store extends Readable>( | ||
| type Metadata = | ||
| | ArrayMetadataV2 | ||
| | GroupMetadataV2 | ||
| | ArrayMetadata | ||
| | GroupMetadata | ||
| | Attributes; | ||
|
|
||
| function is_meta_key(key: string): boolean { | ||
| return ( | ||
| key.endsWith(".zarray") || | ||
| key.endsWith(".zgroup") || | ||
| key.endsWith(".zattrs") || | ||
| key.endsWith("zarr.json") | ||
| ); | ||
| } | ||
|
|
||
| function is_v3(meta: Metadata): meta is ArrayMetadata | GroupMetadata { | ||
| return "zarr_format" in meta && meta.zarr_format === 3; | ||
| } | ||
|
|
||
| export async function withConsolidated<Store extends Readable>( | ||
| store: Store, | ||
| ) { | ||
| let { metadata } = await get_consolidated_metadata(store); | ||
| let meta_nodes = Object | ||
| .entries(metadata) | ||
| .reduce( | ||
| (acc, [path, content]) => { | ||
| let parts = path.split("/"); | ||
| let file_name = parts.pop()!; | ||
| let key: AbsolutePath = `/${parts.join("/")}`; | ||
| if (!acc[key]) acc[key] = {}; | ||
| if (file_name === ".zarray") { | ||
| acc[key].meta = content; | ||
| } else if (file_name === ".zgroup") { | ||
| acc[key].meta = content; | ||
| } else if (file_name === ".zattrs") { | ||
| acc[key].attrs = content; | ||
| } | ||
| return acc; | ||
| }, | ||
| {} as Record< | ||
| AbsolutePath, | ||
| { | ||
| meta?: ArrayMetadataV2 | GroupMetadataV2; | ||
| attrs?: Record<string, any>; | ||
| ): Promise<Listable<Store>> { | ||
| let known_meta: Record<AbsolutePath, Metadata> = | ||
| await get_consolidated_metadata(store) | ||
| .then((meta) => { | ||
| let new_meta: Record<AbsolutePath, Metadata> = {}; | ||
| for (let [key, value] of Object.entries(meta.metadata)) { | ||
| new_meta[`/${key}`] = value; | ||
| } | ||
| >, | ||
| ); | ||
| let nodes = new Map<AbsolutePath, Array<DataType, Store> | Group<Store>>(); | ||
| for (let [path, { meta, attrs }] of Object.entries(meta_nodes)) { | ||
| if (!meta) throw new Error("missing metadata"); | ||
| let node: Array<DataType, Store> | Group<Store>; | ||
| if ("shape" in meta) { | ||
| let metadata = v2_to_v3_array_metadata(meta, attrs); | ||
| node = new Array(store, path as AbsolutePath, metadata); | ||
| } else { | ||
| let metadata = v2_to_v3_group_metadata(meta, attrs); | ||
| node = new Group(store, path as AbsolutePath, metadata); | ||
| } | ||
| nodes.set(path as AbsolutePath, node); | ||
| } | ||
| return new ConsolidatedHierarchy(nodes); | ||
| } | ||
| return new_meta; | ||
| }) | ||
| .catch(() => ({})); | ||
|
|
||
| class ConsolidatedHierarchy<Store extends Readable> { | ||
| constructor( | ||
| public contents: Map<AbsolutePath, Array<DataType, Store> | Group<Store>>, | ||
| ) {} | ||
| open( | ||
| where: AbsolutePath | Location<unknown>, | ||
| options: { kind: "group" }, | ||
| ): Group<Store>; | ||
| open( | ||
| where: AbsolutePath | Location<unknown>, | ||
| options: { kind: "array" }, | ||
| ): Array<DataType, Store>; | ||
| open( | ||
| where: AbsolutePath | Location<unknown>, | ||
| ): Array<DataType, Store> | Group<Store>; | ||
| open( | ||
| where: AbsolutePath | Location<unknown>, | ||
| options: { kind?: "array" | "group" } = {}, | ||
| ) { | ||
| let path = typeof where === "string" ? where : where.path; | ||
| let node = this.contents.get(path); | ||
| if (node && (!options.kind || options.kind == node.kind)) return node; | ||
| throw new NodeNotFoundError(path); | ||
| } | ||
| root() { | ||
| return this.open("/", { kind: "group" }); | ||
| } | ||
| return { | ||
| async get( | ||
| ...args: Parameters<Store["get"]> | ||
| ): Promise<Uint8Array | undefined> { | ||
| let [key, opts] = args; | ||
| if (known_meta[key]) { | ||
| return json_encode_object(known_meta[key]); | ||
| } | ||
| let maybe_bytes = await store.get(key, opts); | ||
| if (is_meta_key(key) && maybe_bytes) { | ||
| let meta = json_decode_object(maybe_bytes); | ||
| known_meta[key] = meta; | ||
| } | ||
| return maybe_bytes; | ||
| }, | ||
| contents(): { path: AbsolutePath; kind: "array" | "group" }[] { | ||
| let contents: { path: AbsolutePath; kind: "array" | "group" }[] = []; | ||
| for (let [key, value] of Object.entries(known_meta)) { | ||
| let parts = key.split("/"); | ||
| let filename = parts.pop()!; | ||
| let path = (parts.join("/") || "/") as AbsolutePath; | ||
| if (filename === ".zarray") contents.push({ path, kind: "array" }); | ||
| if (filename === ".zgroup") contents.push({ path, kind: "group" }); | ||
| if (is_v3(value)) { | ||
| contents.push({ path, kind: value.node_type }); | ||
| } | ||
| } | ||
| return contents; | ||
| }, | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,13 @@ | ||
| export class NodeNotFoundError extends Error { | ||
| constructor(msg: string) { | ||
| super(msg); | ||
| constructor(context: string, options: { cause?: Error } = {}) { | ||
| super(`Node not found: ${context}`, options); | ||
| this.name = "NodeNotFoundError"; | ||
| } | ||
| } | ||
|
|
||
| export class KeyError extends Error { | ||
| constructor(msg: string) { | ||
| super(msg); | ||
| constructor(path: string) { | ||
| super(`Missing key: ${path}`); | ||
| this.name = "KeyError"; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.