diff --git a/README.md b/README.md index f2e452b5..4df406b2 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ async function exampleCode() { // get information about the latest applied storage configuration, // current storage state, and all related Powegate storage jobs - const { cidInfosList } = await pow.data.cidInfo(cid) + const { cidInfo } = await pow.data.cidInfo(cid) // retrieve data stored in the user by cid const bytes = await pow.data.get(cid) diff --git a/package-lock.json b/package-lock.json index 9b7c3c4e..3debe1e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -562,13 +562,13 @@ } }, "@textile/grpc-powergate-client": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-1.2.1.tgz", - "integrity": "sha512-Lj/TWo1zMZRVyp8AyKKisB28DNvV6R2xqkGXLXnwnG0s/FNDTLDkzMbJIQuepq+xwc9GRZKnmiEAQRdu65mcIQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@textile/grpc-powergate-client/-/grpc-powergate-client-2.0.0.tgz", + "integrity": "sha512-iVbLzzAZiRzHzT4ieRrvr5MJGXKoieSOCU3zvMxEOCIvFaGdYnsjKHfb5H1bUBHgLuuculPeHWO+LqAS7oL3+A==", "requires": { "@improbable-eng/grpc-web": "^0.13.0", "@types/google-protobuf": "^3.7.4", - "google-protobuf": "^3.13.0" + "google-protobuf": "^3.14.0" } }, "@textile/grpc-transport": { diff --git a/package.json b/package.json index f8f828cb..030113d4 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ } }, "dependencies": { - "@textile/grpc-powergate-client": "^1.2.1", + "@textile/grpc-powergate-client": "2.0.0", "@textile/grpc-transport": "0.0.3", "ipfs-http-client": "^47.0.1", "it-block": "^2.0.0" diff --git a/src/admin/data/index.ts b/src/admin/data/index.ts new file mode 100644 index 00000000..2e3d2bf7 --- /dev/null +++ b/src/admin/data/index.ts @@ -0,0 +1,45 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + GCStagedRequest, + GCStagedResponse, + PinnedCidsRequest, + PinnedCidsResponse, +} from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb" +import { AdminServiceClient } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb_service" +import { Config } from "../../types" +import { promise } from "../../util" + +export interface Data { + /** + * Unpins staged data not related to queued or executing jobs. + * @returns An object containing a list of unpinned cids. + */ + gcStaged: () => Promise + + /** + * Get pinned cids information of hot-storage. + * @returns Pinned cids information of hot-storage. + */ + pinnedCids: () => Promise +} + +/** + * @ignore + */ +export const createData = (config: Config, getMeta: () => grpc.Metadata): Data => { + const client = new AdminServiceClient(config.host, config) + return { + gcStaged: () => { + return promise( + (cb) => client.gCStaged(new GCStagedRequest(), getMeta(), cb), + (resp: GCStagedResponse) => resp.toObject(), + ) + }, + + pinnedCids: () => + promise( + (cb) => client.pinnedCids(new PinnedCidsRequest(), getMeta(), cb), + (resp: PinnedCidsResponse) => resp.toObject(), + ), + } +} diff --git a/src/admin/index.ts b/src/admin/index.ts index 1d135c61..5c62d020 100644 --- a/src/admin/index.ts +++ b/src/admin/index.ts @@ -1,11 +1,18 @@ import { grpc } from "@improbable-eng/grpc-web" import { Config } from "../types" +import { createData, Data } from "./data" +import { createStorageInfo, StorageInfo } from "./storage-info" import { createStorageJobs, StorageJobs } from "./storage-jobs" import { createUsers, Users } from "./users" import { createWallet, Wallet } from "./wallet" -export { Users, StorageJobs, Wallet } +export { Users, StorageJobs, Wallet, StorageInfo } export interface Admin { + /** + * The admin Data API. + */ + data: Data + /** * The admin Users API. */ @@ -16,6 +23,11 @@ export interface Admin { */ wallet: Wallet + /** + * The admin StorageInfo API. + */ + storageInfo: StorageInfo + /** * The admin Storage Jobs API. */ @@ -27,8 +39,10 @@ export interface Admin { */ export const createAdmin = (config: Config, getMeta: () => grpc.Metadata): Admin => { return { + data: createData(config, getMeta), users: createUsers(config, getMeta), wallet: createWallet(config, getMeta), + storageInfo: createStorageInfo(config, getMeta), storageJobs: createStorageJobs(config, getMeta), } } diff --git a/src/admin/storage-info/index.ts b/src/admin/storage-info/index.ts new file mode 100644 index 00000000..bc6c2b1b --- /dev/null +++ b/src/admin/storage-info/index.ts @@ -0,0 +1,59 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + ListStorageInfoRequest, + ListStorageInfoResponse, + StorageInfoRequest, + StorageInfoResponse, +} from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb" +import { AdminServiceClient } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb_service" +import { Config } from "../../types" +import { promise } from "../../util" + +export interface StorageInfo { + /** + * Get the current storage state of a cid. + * @param userId The user id to query. + * @param cid The cid to query. + * @returns The current storage state of the cid. + */ + get: (userId: string, cid: string) => Promise + + /** + * Lists the current storage state for many or all user ids and cids. + * @param cids Optional list of cids to filter the results by. + * @returns An object containing a list of storage info. + */ + list: (userIds?: string[], cids?: string[]) => Promise +} + +/** + * @ignore + */ +export const createStorageInfo = (config: Config, getMeta: () => grpc.Metadata): StorageInfo => { + const client = new AdminServiceClient(config.host, config) + return { + get: (userId: string, cid: string) => { + const req = new StorageInfoRequest() + req.setUserId(userId) + req.setCid(cid) + return promise( + (cb) => client.storageInfo(req, getMeta(), cb), + (res: StorageInfoResponse) => res.toObject(), + ) + }, + + list: (userIds?: string[], cids?: string[]) => { + const req = new ListStorageInfoRequest() + if (userIds) { + req.setUserIdsList(userIds) + } + if (cids) { + req.setCidsList(cids) + } + return promise( + (cb) => client.listStorageInfo(req, getMeta(), cb), + (res: ListStorageInfoResponse) => res.toObject(), + ) + }, + } +} diff --git a/src/admin/storage-jobs.ts b/src/admin/storage-jobs.ts deleted file mode 100644 index c5b53eab..00000000 --- a/src/admin/storage-jobs.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { grpc } from "@improbable-eng/grpc-web" -import { - ExecutingStorageJobsRequest, - ExecutingStorageJobsResponse, - LatestFinalStorageJobsRequest, - LatestFinalStorageJobsResponse, - LatestSuccessfulStorageJobsRequest, - LatestSuccessfulStorageJobsResponse, - QueuedStorageJobsRequest, - QueuedStorageJobsResponse, - StorageJobsSummaryRequest, - StorageJobsSummaryResponse, -} from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb" -import { AdminServiceClient } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb_service" -import { Config } from "../types" -import { promise } from "../util" - -export interface StorageJobs { - /** - * List queued storgae jobs. - * @param userId The user id to query or an empty string for all users. - * @param cids An optional list of data cids to fileter the results with. - * @returns A list of queued storage jobs. - */ - queued: (userId: string, ...cids: string[]) => Promise - - /** - * List executing storgae jobs. - * @param userId The user id to query or an empty string for all users. - * @param cids An optional list of data cids to fileter the results with. - * @returns A list of executing storage jobs. - */ - executing: (userId: string, ...cids: string[]) => Promise - - /** - * List the latest final storgae jobs. - * @param userId The user id to query or an empty string for all users. - * @param cids An optional list of data cids to fileter the results with. - * @returns A list of the latest final storage jobs. - */ - latestFinal: ( - userId: string, - ...cids: string[] - ) => Promise - - /** - * List the latest successful storgae jobs. - * @param userId The user id to query or an empty string for all users. - * @param cids An optional list of data cids to fileter the results with. - * @returns A list of the latest successful storage jobs. - */ - latestSuccessful: ( - userId: string, - ...cids: string[] - ) => Promise - - /** - * Get a summary of all jobs. - * @param userId The user id to query or an empty string for all users. - * @param cids An optional list of data cids to fileter the results with. - * @returns A summary of all jobs. - */ - summary: (userId: string, ...cids: string[]) => Promise -} - -/** - * @ignore - */ -export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): StorageJobs => { - const client = new AdminServiceClient(config.host, config) - return { - queued: (userId: string, ...cids: string[]) => { - const req = new QueuedStorageJobsRequest() - req.setCidsList(cids) - req.setUserId(userId) - return promise( - (cb) => client.queuedStorageJobs(req, getMeta(), cb), - (resp: QueuedStorageJobsResponse) => resp.toObject(), - ) - }, - - executing: (userId: string, ...cids: string[]) => { - const req = new ExecutingStorageJobsRequest() - req.setCidsList(cids) - req.setUserId(userId) - return promise( - (cb) => client.executingStorageJobs(req, getMeta(), cb), - (resp: ExecutingStorageJobsResponse) => resp.toObject(), - ) - }, - - latestFinal: (userId: string, ...cids: string[]) => { - const req = new LatestFinalStorageJobsRequest() - req.setCidsList(cids) - req.setUserId(userId) - return promise( - (cb) => client.latestFinalStorageJobs(req, getMeta(), cb), - (resp: LatestFinalStorageJobsResponse) => resp.toObject(), - ) - }, - - latestSuccessful: (userId: string, ...cids: string[]) => { - const req = new LatestSuccessfulStorageJobsRequest() - req.setCidsList(cids) - req.setUserId(userId) - return promise( - (cb) => client.latestSuccessfulStorageJobs(req, getMeta(), cb), - (resp: LatestSuccessfulStorageJobsResponse) => resp.toObject(), - ) - }, - - summary: (userId: string, ...cids: string[]) => { - const req = new StorageJobsSummaryRequest() - req.setCidsList(cids) - req.setUserId(userId) - return promise( - (cb) => client.storageJobsSummary(req, getMeta(), cb), - (resp: StorageJobsSummaryResponse) => resp.toObject(), - ) - }, - } -} diff --git a/src/admin/storage-jobs/index.ts b/src/admin/storage-jobs/index.ts new file mode 100644 index 00000000..558e74a4 --- /dev/null +++ b/src/admin/storage-jobs/index.ts @@ -0,0 +1,91 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + ListStorageJobsRequest, + ListStorageJobsResponse, + StorageJobsSummaryRequest, + StorageJobsSummaryResponse, +} from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb" +import { AdminServiceClient } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb_service" +import { StorageJobsSelector } from "@textile/grpc-powergate-client/dist/powergate/user/v1/user_pb" +import { ListSelect } from "../../storage-jobs" +import { Config } from "../../types" +import { promise } from "../../util" +import { AdminListOptions } from "./types" + +export interface StorageJobs { + /** + * Lists StorageJobs according to the provided ListOptions. + * @param opts Optional ListOptions to control the behavior of listing jobs. + * @returns An object containing a list of storage jobs. + */ + list: (opts?: AdminListOptions) => Promise + + /** + * Get a summary of all jobs. + * @param userId The user id to query or undefined for all users. + * @param cids An optional cid to fileter the results with. + * @returns A summary of all jobs. + */ + summary: (userId?: string, cid?: string) => Promise +} + +/** + * @ignore + */ +export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): StorageJobs => { + const client = new AdminServiceClient(config.host, config) + return { + list: (opts?: AdminListOptions) => { + const req = new ListStorageJobsRequest() + if (opts?.ascending) { + req.setAscending(opts.ascending) + } + if (opts?.cidFilter) { + req.setCidFilter(opts.cidFilter) + } + if (opts?.limit) { + req.setLimit(opts.limit) + } + if (opts?.nextPageToken) { + req.setNextPageToken(opts.nextPageToken) + } + if (opts?.select != undefined) { + switch (opts.select) { + case ListSelect.All: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_ALL) + break + case ListSelect.Queued: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_QUEUED) + break + case ListSelect.Executing: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_EXECUTING) + break + case ListSelect.Final: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_FINAL) + break + } + } + if (opts?.userId) { + req.setUserIdFilter(opts.userId) + } + return promise( + (cb) => client.listStorageJobs(req, getMeta(), cb), + (resp: ListStorageJobsResponse) => resp.toObject(), + ) + }, + + summary: (userId?: string, cid?: string) => { + const req = new StorageJobsSummaryRequest() + if (userId) { + req.setUserId(userId) + } + if (cid) { + req.setCid(cid) + } + return promise( + (cb) => client.storageJobsSummary(req, getMeta(), cb), + (resp: StorageJobsSummaryResponse) => resp.toObject(), + ) + }, + } +} diff --git a/src/admin/storage-jobs/types.ts b/src/admin/storage-jobs/types.ts new file mode 100644 index 00000000..28c1a820 --- /dev/null +++ b/src/admin/storage-jobs/types.ts @@ -0,0 +1,5 @@ +import { ListOptions } from "../../storage-jobs" + +export interface AdminListOptions extends ListOptions { + userId?: string +} diff --git a/src/admin/users.ts b/src/admin/users/index.ts similarity index 93% rename from src/admin/users.ts rename to src/admin/users/index.ts index b4e469cb..65bfbc7e 100644 --- a/src/admin/users.ts +++ b/src/admin/users/index.ts @@ -6,8 +6,8 @@ import { UsersResponse, } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb" import { AdminServiceClient } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb_service" -import { Config } from "../types" -import { promise } from "../util" +import { Config } from "../../types" +import { promise } from "../../util" export interface Users { /** diff --git a/src/admin/wallet.ts b/src/admin/wallet/index.ts similarity index 96% rename from src/admin/wallet.ts rename to src/admin/wallet/index.ts index 59f849a6..9e29c670 100644 --- a/src/admin/wallet.ts +++ b/src/admin/wallet/index.ts @@ -8,8 +8,8 @@ import { SendFilResponse, } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb" import { AdminServiceClient } from "@textile/grpc-powergate-client/dist/powergate/admin/v1/admin_pb_service" -import { Config } from "../types" -import { promise } from "../util" +import { Config } from "../../types" +import { promise } from "../../util" export interface Wallet { /** diff --git a/src/data/index.ts b/src/data/index.ts index 7a7040b1..a96b97ef 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -2,6 +2,8 @@ import { grpc } from "@improbable-eng/grpc-web" import { CidInfoRequest, CidInfoResponse, + CidSummaryRequest, + CidSummaryResponse, GetRequest, LogEntry, ReplaceDataRequest, @@ -79,7 +81,19 @@ export interface Data { opts?: WatchLogsOptions, ) => () => void - cidInfo: (...cids: string[]) => Promise + /** + * Get high level information about the current state of cids in Powergate. + * @param cids A list of cids to filter the results by. + * @returns An object containing a list of cid summary info. + */ + cidSummary: (...cids: string[]) => Promise + + /** + * Get detailed information about the current state of a cid in Powergate. + * @param cid The cid to get information for. + * @returns An object with detailed information about the cid. + */ + cidInfo: (cid: string) => Promise } /** @@ -171,7 +185,7 @@ export const createData = ( const stream = fs.createWriteStream(fullFilePath) for await (const chunk of file.content) { const slice = chunk.slice() - await new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { stream.write(slice, (err) => { if (err) { reject(err) @@ -221,9 +235,18 @@ export const createData = ( ) }, - cidInfo: (...cids: string[]) => { - const req = new CidInfoRequest() + cidSummary: (...cids: string[]) => { + const req = new CidSummaryRequest() req.setCidsList(cids) + return promise( + (cb) => client.cidSummary(req, getMeta(), cb), + (res: CidSummaryResponse) => res.toObject(), + ) + }, + + cidInfo: (cid: string) => { + const req = new CidInfoRequest() + req.setCid(cid) return promise( (cb) => client.cidInfo(req, getMeta(), cb), (res: CidInfoResponse) => res.toObject(), diff --git a/src/index.ts b/src/index.ts index 618568fe..13937ef4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { Admin, createAdmin } from "./admin" import { createData, Data, GetFolderOptions, WatchLogsOptions } from "./data" import { createDeals, DealRecordsOptions, Deals } from "./deals" import { ApplyOptions, createStorageConfig, StorageConfig } from "./storage-config" +import { createStorageInfo, StorageInfo } from "./storage-info" import { createStorageJobs, StorageJobs } from "./storage-jobs" import { Config } from "./types" import { getTransport, host, promise, useTokens } from "./util" @@ -75,6 +76,11 @@ export interface Pow { */ deals: Deals + /** + * The StorageInfo API + */ + storageInfo: StorageInfo + /** * The StorageJobs API */ @@ -125,6 +131,8 @@ export const createPow = (config?: Partial): Pow => { deals: createDeals(c, getMeta), + storageInfo: createStorageInfo(c, getMeta), + storageJobs: createStorageJobs(c, getMeta), admin: createAdmin(c, getMeta), diff --git a/src/integration.spec.ts b/src/integration.spec.ts index e1b69923..cb10ca90 100644 --- a/src/integration.spec.ts +++ b/src/integration.spec.ts @@ -3,6 +3,7 @@ import cp from "child_process" import crypto from "crypto" import wait from "wait-on" import { ApplyOptions, createPow, Pow, powTypes } from "." +import { ListSelect } from "./storage-jobs" import { host } from "./util" beforeEach(async function () { @@ -75,43 +76,12 @@ describe("pow", () => { await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - const res = await pow.admin.storageJobs.executing(auth.id, cid) - expect(res.storageJobsList).length(1) - expect(res.storageJobsList[0].id).equals(jobId) - expect(res.storageJobsList[0].cid).equals(cid) - }) - - it("should get latest final", async function () { - this.timeout(180000) - const pow = newPow() - const auth = await expectNewUser(pow) - const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address) - const cid = await expectStage(pow, crypto.randomBytes(1024)) - const jobId = await expectApplyStorageConfig(pow, cid) - let res = await pow.admin.storageJobs.latestFinal(auth.id, cid) - expect(res.storageJobsList).empty - await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - res = await pow.admin.storageJobs.latestFinal(auth.id, cid) - expect(res.storageJobsList).length(1) - expect(res.storageJobsList[0].id).equals(jobId) - expect(res.storageJobsList[0].cid).equals(cid) - }) - - it("should get latest successful", async function () { - this.timeout(180000) - const pow = newPow() - const auth = await expectNewUser(pow) - const addressees = await expectAddresses(pow, 1) - await waitForBalance(pow, addressees[0].address) - const cid = await expectStage(pow, crypto.randomBytes(1024)) - const jobId = await expectApplyStorageConfig(pow, cid) - let res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) - expect(res.storageJobsList).empty - await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) - expect(res.storageJobsList).length(1) - res = await pow.admin.storageJobs.latestSuccessful(auth.id, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_EXECUTING) + const res = await pow.admin.storageJobs.list({ + userId: auth.id, + cidFilter: cid, + select: ListSelect.Executing, + }) expect(res.storageJobsList).length(1) expect(res.storageJobsList[0].id).equals(jobId) expect(res.storageJobsList[0].cid).equals(cid) @@ -124,7 +94,11 @@ describe("pow", () => { await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) await expectApplyStorageConfig(pow, cid) - const res = await pow.admin.storageJobs.queued(auth.id, cid) + const res = await pow.admin.storageJobs.list({ + userId: auth.id, + cidFilter: cid, + select: ListSelect.Queued, + }) expect(res.storageJobsList).length.lessThan(2) }) @@ -139,12 +113,7 @@ describe("pow", () => { expect(res.executingStorageJobsList).length(1) res = await pow.admin.storageJobs.summary(auth.id, cid) expect(res.executingStorageJobsList).length(1) - expect(res.executingStorageJobsList[0].cid).equals(cid) - expect(res.executingStorageJobsList[0].id).equals(jobId) - expect(res.jobCounts?.executing).equals(1) - expect(res.jobCounts?.latestFinal).equals(0) - expect(res.jobCounts?.latestSuccessful).equals(0) - expect(res.jobCounts?.queued).equals(0) + expect(res.executingStorageJobsList[0]).equals(jobId) }) }) @@ -158,11 +127,8 @@ describe("pow", () => { const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - const res = await pow.admin.storageJobs.summary("") - expect(res.jobCounts?.latestFinal).greaterThan(0) - expect(res.jobCounts?.latestSuccessful).greaterThan(0) - expect(res.jobCounts?.queued).equals(0) - expect(res.jobCounts?.executing).equals(0) + const res = await pow.admin.storageJobs.summary() + expect(res.finalStorageJobsList).length.greaterThan(0) }) }) @@ -190,9 +156,100 @@ describe("pow", () => { await waitForBalance(pow, res1.address, bal) }) }) + + describe("data", () => { + it("should gc staged", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewUser(pow) + const cid = await expectStage(pow, crypto.randomBytes(1024)) + const res = await pow.admin.data.gcStaged() + expect(res.unpinnedCidsList).length(1) + expect(res.unpinnedCidsList[0]).equals(cid) + }) + + it("should get pinned cids", async function () { + this.timeout(30000) + const pow = newPow() + const { id } = await expectNewUser(pow) + const cid = await expectStage(pow, crypto.randomBytes(1024)) + const opts: ApplyOptions = { + storageConfig: { + repairable: false, + hot: { + enabled: true, + allowUnfreeze: false, + unfreezeMaxPrice: 0, + ipfs: { + addTimeout: 300, + }, + }, + }, + } + const jobId = await expectApplyStorageConfig(pow, cid, opts) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + const res = await pow.admin.data.pinnedCids() + expect(res.cidsList).length(1) + expect(res.cidsList[0].cid).equals(cid) + expect(res.cidsList[0].usersList).length(1) + expect(res.cidsList[0].usersList[0].userId).equals(id) + }) + }) + + describe("storage info", () => { + it("should get", async function () { + this.timeout(180000) + const pow = newPow() + const { id } = await expectNewUser(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + const res = await pow.admin.storageInfo.get(id, cid) + expect(res?.storageInfo).not.undefined + expect(res?.storageInfo?.cid).equals(cid) + }) + + it("should list", async function () { + this.timeout(180000) + const pow = newPow() + const { id } = await expectNewUser(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + let res = await pow.admin.storageInfo.list() + expect(res.storageInfoList).length(1) + res = await pow.admin.storageInfo.list([id]) + expect(res.storageInfoList).length(1) + res = await pow.admin.storageInfo.list([id], [cid]) + expect(res.storageInfoList).length(1) + res = await pow.admin.storageInfo.list(undefined, [cid]) + expect(res.storageInfoList).length(1) + expect(res?.storageInfoList[0].cid).equals(cid) + }) + }) }) describe("data", () => { + it("should get cid summary", async function () { + this.timeout(180000) + const pow = newPow() + await expectNewUser(pow) + const addressees = await expectAddresses(pow, 1) + await waitForBalance(pow, addressees[0].address) + const cid = await expectStage(pow, crypto.randomBytes(1024)) + const jobId = await expectApplyStorageConfig(pow, cid) + let res = await pow.data.cidSummary() + expect(res.cidSummaryList).length(1) + res = await pow.data.cidSummary(cid) + expect(res.cidSummaryList).length(1) + expect(res.cidSummaryList[0].cid).equals(cid) + expect(res.cidSummaryList[0].executingJob).equals(jobId) + }) + it("should get cid info", async function () { this.timeout(180000) const pow = newPow() @@ -202,28 +259,22 @@ describe("pow", () => { const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) let res = await pow.data.cidInfo(cid) - expect(res.cidInfosList).length(1) - res = await pow.data.cidInfo() - expect(res.cidInfosList).length(1) - expect(res.cidInfosList[0].cid).equals(cid) - expect(res.cidInfosList[0].currentStorageInfo).undefined - expect(res.cidInfosList[0].latestFinalStorageJob).undefined - expect(res.cidInfosList[0].latestSuccessfulStorageJob).undefined - expect(res.cidInfosList[0].latestPushedStorageConfig).not.undefined - expect(res.cidInfosList[0].queuedStorageJobsList).length.lessThan(2) - if (res.cidInfosList[0].executingStorageJob) { - expect(res.cidInfosList[0].executingStorageJob.cid).equals(cid) + expect(res.cidInfo).not.undefined + expect(res.cidInfo?.cid).equals(cid) + expect(res.cidInfo?.currentStorageInfo).undefined + expect(res.cidInfo?.latestPushedStorageConfig).not.undefined + expect(res.cidInfo?.queuedStorageJobsList).length.lessThan(2) + if (res.cidInfo?.executingStorageJob) { + expect(res.cidInfo?.executingStorageJob.cid).equals(cid) } await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - res = await pow.data.cidInfo() - expect(res.cidInfosList).length(1) - expect(res.cidInfosList[0].cid).equals(cid) - expect(res.cidInfosList[0].currentStorageInfo?.cid).equals(cid) - expect(res.cidInfosList[0].latestFinalStorageJob).not.undefined - expect(res.cidInfosList[0].latestSuccessfulStorageJob).not.undefined - expect(res.cidInfosList[0].latestPushedStorageConfig).not.undefined - expect(res.cidInfosList[0].queuedStorageJobsList).length(0) - expect(res.cidInfosList[0].executingStorageJob).undefined + res = await pow.data.cidInfo(cid) + expect(res.cidInfo).not.undefined + expect(res.cidInfo?.cid).equals(cid) + expect(res.cidInfo?.currentStorageInfo?.cid).equals(cid) + expect(res.cidInfo?.latestPushedStorageConfig).not.undefined + expect(res.cidInfo?.queuedStorageJobsList).length(0) + expect(res.cidInfo?.executingStorageJob).undefined }) it("should get", async function () { @@ -361,112 +412,108 @@ describe("pow", () => { await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - let res = await pow.storageJobs.executing() + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_EXECUTING) + let res = await pow.storageJobs.list({ select: ListSelect.Executing }) expect(res.storageJobsList).length(1) - res = await pow.storageJobs.executing(cid) + res = await pow.storageJobs.list({ + cidFilter: cid, + select: ListSelect.Executing, + }) expect(res.storageJobsList).length(1) expect(res.storageJobsList[0].id).equals(jobId) expect(res.storageJobsList[0].cid).equals(cid) }) - it("should get latest final", async function () { - this.timeout(180000) + it("should get queued", async function () { const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) - const jobId = await expectApplyStorageConfig(pow, cid) - let res = await pow.storageJobs.latestFinal() - expect(res.storageJobsList).empty - await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - res = await pow.storageJobs.latestFinal() - expect(res.storageJobsList).length(1) - res = await pow.storageJobs.latestFinal(cid) - expect(res.storageJobsList).length(1) - expect(res.storageJobsList[0].id).equals(jobId) - expect(res.storageJobsList[0].cid).equals(cid) + await expectApplyStorageConfig(pow, cid) + const res = await pow.storageJobs.list({ + cidFilter: cid, + select: ListSelect.Queued, + }) + expect(res.storageJobsList).length.lessThan(2) }) - it("should get latest successful", async function () { - this.timeout(180000) + it("should get storage config for job", async function () { const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - let res = await pow.storageJobs.latestSuccessful() - expect(res.storageJobsList).empty - await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) - res = await pow.storageJobs.latestSuccessful() - expect(res.storageJobsList).length(1) - res = await pow.storageJobs.latestSuccessful(cid) - expect(res.storageJobsList).length(1) - expect(res.storageJobsList[0].id).equals(jobId) - expect(res.storageJobsList[0].cid).equals(cid) + const res = await pow.storageJobs.storageConfig(jobId) + expect(res.storageConfig).not.undefined }) - it("should get queued", async function () { + it("should get storage job", async function () { const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) - await expectApplyStorageConfig(pow, cid) - const res = await pow.storageJobs.queued(cid) - expect(res.storageJobsList).length.lessThan(2) + const jobId = await expectApplyStorageConfig(pow, cid) + const res = await pow.storageJobs.get(jobId) + expect(res.storageJob?.id).equals(jobId) }) - it("should get storage config for job", async function () { + it("should get summary", async function () { const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - const res = await pow.storageJobs.storageConfigForJob(jobId) - expect(res.storageConfig).not.undefined + let res = await pow.storageJobs.summary() + expect(res.executingStorageJobsList).length(1) + res = await pow.storageJobs.summary(cid) + expect(res.executingStorageJobsList).length(1) + expect(res.executingStorageJobsList[0]).equals(jobId) }) - it("should get storage job", async function () { + it("should watch", async function () { const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - const res = await pow.storageJobs.storageJob(jobId) - expect(res.storageJob?.id).equals(jobId) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_EXECUTING) }) + }) - it("should get summary", async function () { + describe("storage info", () => { + it("should get", async function () { + this.timeout(180000) const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - let res = await pow.storageJobs.summary() - expect(res.executingStorageJobsList).length(1) - res = await pow.storageJobs.summary(cid) - expect(res.executingStorageJobsList).length(1) - expect(res.executingStorageJobsList[0].cid).equals(cid) - expect(res.executingStorageJobsList[0].id).equals(jobId) - expect(res.jobCounts?.executing).equals(1) - expect(res.jobCounts?.latestFinal).equals(0) - expect(res.jobCounts?.latestSuccessful).equals(0) - expect(res.jobCounts?.queued).equals(0) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + const res = await pow.storageInfo.get(cid) + expect(res?.storageInfo).not.undefined + expect(res?.storageInfo?.cid).equals(cid) }) - it("should watch", async function () { + it("should list", async function () { + this.timeout(180000) const pow = newPow() await expectNewUser(pow) const addressees = await expectAddresses(pow, 1) await waitForBalance(pow, addressees[0].address) const cid = await expectStage(pow, crypto.randomBytes(1024)) const jobId = await expectApplyStorageConfig(pow, cid) - await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_EXECUTING) + await watchJobUntil(pow, jobId, powTypes.JobStatus.JOB_STATUS_SUCCESS) + let res = await pow.storageInfo.list() + expect(res.storageInfoList).length(1) + res = await pow.storageInfo.list(cid) + expect(res.storageInfoList).length(1) + expect(res?.storageInfoList[0].cid).equals(cid) }) }) diff --git a/src/storage-config/index.ts b/src/storage-config/index.ts index 192b28ea..92e720da 100644 --- a/src/storage-config/index.ts +++ b/src/storage-config/index.ts @@ -93,6 +93,12 @@ export const createStorageConfig = ( req.setConfig(c) req.setHasConfig(true) } + if (opts?.importDealIds) { + req.setImportDealIdsList(opts.importDealIds) + } + if (opts?.noExec) { + req.setNoExec(opts.noExec) + } return promise( (cb) => client.applyStorageConfig(req, getMeta(), cb), (res: ApplyStorageConfigResponse) => res.toObject(), diff --git a/src/storage-config/types.ts b/src/storage-config/types.ts index 83aeb953..afcc013e 100644 --- a/src/storage-config/types.ts +++ b/src/storage-config/types.ts @@ -3,7 +3,7 @@ import { StorageConfig } from "@textile/grpc-powergate-client/dist/powergate/use /** * Options to control the behavior of pushStorageConfig. */ -export type ApplyOptions = { +export interface ApplyOptions { /** * Allows you to override an existing storage configuration */ @@ -13,4 +13,14 @@ export type ApplyOptions = { * Allows you to override the default storage config with a custom one */ storageConfig?: StorageConfig.AsObject + + /** + * Allows to import active on-chain deals to the Cid deals information. + */ + importDealIds?: number[] + + /** + * Allows to configure if a Job should ensure the new storage configuration. + */ + noExec?: boolean } diff --git a/src/storage-info/index.ts b/src/storage-info/index.ts new file mode 100644 index 00000000..81cedd3a --- /dev/null +++ b/src/storage-info/index.ts @@ -0,0 +1,52 @@ +import { grpc } from "@improbable-eng/grpc-web" +import { + ListStorageInfoRequest, + ListStorageInfoResponse, + StorageInfoRequest, + StorageInfoResponse, +} from "@textile/grpc-powergate-client/dist/powergate/user/v1/user_pb" +import { UserServiceClient } from "@textile/grpc-powergate-client/dist/powergate/user/v1/user_pb_service" +import { Config } from "../types" +import { promise } from "../util" + +export interface StorageInfo { + /** + * Get the current storage state of a cid. + * @param cid The cid to query. + * @returns The current storage state of the cid. + */ + get: (cid: string) => Promise + + /** + * Lists the current storage state for many or all cids. + * @param cids Optional list of cids to filter the results by. + * @returns An object containing a list of storage info. + */ + list: (...cids: string[]) => Promise +} + +/** + * @ignore + */ +export const createStorageInfo = (config: Config, getMeta: () => grpc.Metadata): StorageInfo => { + const client = new UserServiceClient(config.host, config) + return { + get: (cid: string) => { + const req = new StorageInfoRequest() + req.setCid(cid) + return promise( + (cb) => client.storageInfo(req, getMeta(), cb), + (res: StorageInfoResponse) => res.toObject(), + ) + }, + + list: (...cids: string[]) => { + const req = new ListStorageInfoRequest() + req.setCidsList(cids) + return promise( + (cb) => client.listStorageInfo(req, getMeta(), cb), + (res: ListStorageInfoResponse) => res.toObject(), + ) + }, + } +} diff --git a/src/storage-jobs/index.ts b/src/storage-jobs/index.ts index 14f5724d..a5f53903 100644 --- a/src/storage-jobs/index.ts +++ b/src/storage-jobs/index.ts @@ -2,19 +2,14 @@ import { grpc } from "@improbable-eng/grpc-web" import { CancelStorageJobRequest, CancelStorageJobResponse, - ExecutingStorageJobsRequest, - ExecutingStorageJobsResponse, - LatestFinalStorageJobsRequest, - LatestFinalStorageJobsResponse, - LatestSuccessfulStorageJobsRequest, - LatestSuccessfulStorageJobsResponse, - QueuedStorageJobsRequest, - QueuedStorageJobsResponse, + ListStorageJobsRequest, + ListStorageJobsResponse, StorageConfigForJobRequest, StorageConfigForJobResponse, StorageJob, StorageJobRequest, StorageJobResponse, + StorageJobsSelector, StorageJobsSummaryRequest, StorageJobsSummaryResponse, WatchStorageJobsRequest, @@ -22,6 +17,9 @@ import { import { UserServiceClient } from "@textile/grpc-powergate-client/dist/powergate/user/v1/user_pb_service" import { Config } from "../types" import { promise } from "../util" +import { ListOptions, ListSelect } from "./types" + +export { ListOptions, ListSelect } export interface StorageJobs { /** @@ -29,49 +27,28 @@ export interface StorageJobs { * @param jobId The job id to query. * @returns The current state of the storage job. */ - storageJob: (jobId: string) => Promise - - /** - * Get the desired storage config for the provided cid, this config may not yet be realized. - * @param cid The cid of the desired storage config. - * @returns The storage config for the provided cid. - */ - storageConfigForJob: (jobId: string) => Promise - - /** - * Get queued jobs in the user for the specified cids or all cids. - * @param cids A list of cids to get jobs for, providing no cids means all cids. - * @returns An object containing a list of jobs. - */ - queued: (...cids: string[]) => Promise - - /** - * Get executing jobs in the user for the specified cids or all cids. - * @param cids A list of cids to get jobs for, providing no cids means all cids. - * @returns An object containing a list of jobs. - */ - executing: (...cids: string[]) => Promise + get: (jobId: string) => Promise /** - * Get the latest final jobs in the user for the specified cids or all cids. - * @param cids A list of cids to get jobs for, providing no cids means all cids. - * @returns An object containing a list of jobs. + * Get the storage config associated with the specified storage job id. + * @param jobId The cid of the desired storage config. + * @returns The storage config associated with the provided job id. */ - latestFinal: (...cids: string[]) => Promise + storageConfig: (jobId: string) => Promise /** - * Get latest successful jobs in the user for the specified cids or all cids. - * @param cids A list of cids to get jobs for, providing no cids means all cids. - * @returns An object containing a list of jobs. + * Lists StorageJobs according to the provided ListOptions. + * @param opts Optional ListOptions to control the behavior of listing jobs. + * @returns An object containing a list of storage jobs. */ - latestSuccessful: (...cids: string[]) => Promise + list: (opts?: ListOptions) => Promise /** * Get a summary of jobs in the user for the specified cids or all cids. - * @param cids A list of cids to get a job summary for, providing no cids means all cids. + * @param cid An optional cid to get a job summary for, providing no cid means all cids. * @returns An object containing a summary of jobs. */ - summary: (...cids: string[]) => Promise + summary: (cid?: string) => Promise /** * Listen for job updates for the provided job ids. @@ -94,7 +71,7 @@ export interface StorageJobs { export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): StorageJobs => { const client = new UserServiceClient(config.host, config) return { - storageJob: (jobId: string) => { + get: (jobId: string) => { const req = new StorageJobRequest() req.setJobId(jobId) return promise( @@ -103,7 +80,7 @@ export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): ) }, - storageConfigForJob: (jobId: string) => { + storageConfig: (jobId: string) => { const req = new StorageConfigForJobRequest() req.setJobId(jobId) return promise( @@ -112,45 +89,47 @@ export const createStorageJobs = (config: Config, getMeta: () => grpc.Metadata): ) }, - queued: (...cids: string[]) => { - const req = new QueuedStorageJobsRequest() - req.setCidsList(cids) - return promise( - (cb) => client.queuedStorageJobs(req, getMeta(), cb), - (res: QueuedStorageJobsResponse) => res.toObject(), - ) - }, - - executing: (...cids: string[]) => { - const req = new ExecutingStorageJobsRequest() - req.setCidsList(cids) - return promise( - (cb) => client.executingStorageJobs(req, getMeta(), cb), - (res: ExecutingStorageJobsResponse) => res.toObject(), - ) - }, - - latestFinal: (...cids: string[]) => { - const req = new LatestFinalStorageJobsRequest() - req.setCidsList(cids) - return promise( - (cb) => client.latestFinalStorageJobs(req, getMeta(), cb), - (res: LatestFinalStorageJobsResponse) => res.toObject(), - ) - }, - - latestSuccessful: (...cids: string[]) => { - const req = new LatestSuccessfulStorageJobsRequest() - req.setCidsList(cids) + list: (opts?: ListOptions) => { + const req = new ListStorageJobsRequest() + if (opts?.ascending) { + req.setAscending(opts.ascending) + } + if (opts?.cidFilter) { + req.setCidFilter(opts.cidFilter) + } + if (opts?.limit) { + req.setLimit(opts.limit) + } + if (opts?.nextPageToken) { + req.setNextPageToken(opts.nextPageToken) + } + if (opts?.select != undefined) { + switch (opts.select) { + case ListSelect.All: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_ALL) + break + case ListSelect.Queued: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_QUEUED) + break + case ListSelect.Executing: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_EXECUTING) + break + case ListSelect.Final: + req.setSelector(StorageJobsSelector.STORAGE_JOBS_SELECTOR_FINAL) + break + } + } return promise( - (cb) => client.latestSuccessfulStorageJobs(req, getMeta(), cb), - (res: LatestSuccessfulStorageJobsResponse) => res.toObject(), + (cb) => client.listStorageJobs(req, getMeta(), cb), + (res: ListStorageJobsResponse) => res.toObject(), ) }, - summary: (...cids: string[]) => { + summary: (cid?: string) => { const req = new StorageJobsSummaryRequest() - req.setCidsList(cids) + if (cid) { + req.setCid(cid) + } return promise( (cb) => client.storageJobsSummary(req, getMeta(), cb), (res: StorageJobsSummaryResponse) => res.toObject(), diff --git a/src/storage-jobs/types.ts b/src/storage-jobs/types.ts new file mode 100644 index 00000000..8fab78f2 --- /dev/null +++ b/src/storage-jobs/types.ts @@ -0,0 +1,47 @@ +/** + * Specifies which StorageJobs to list. + */ +export enum ListSelect { + /** + * Lists all StorageJobs and is the default. + */ + All, + /** + * Lists queued StorageJobs. + */ + Queued, + /** + * Lists executing StorageJobs. + */ + Executing, + /** + * Lists final StorageJobs. + */ + Final, +} + +/** + * Controls the behavior for listing StorageJobs. + */ +export interface ListOptions { + /** + * Filters StorageJobs list to the specified cid. Defaults to no filter. + */ + cidFilter?: string + /** + * Limits the number of StorageJobs returned. Defaults to no limit. + */ + limit?: number + /** + * Returns the StorageJobs ascending by time. Defaults to false, descending. + */ + ascending?: boolean + /** + * Specifies to return StorageJobs in the specified state. + */ + select?: ListSelect + /** + * Sets the slug from which to start building the next page of results. + */ + nextPageToken?: string +}