diff --git a/scripts/js/package-lock.json b/scripts/js/package-lock.json index cd593ff..5428e83 100644 --- a/scripts/js/package-lock.json +++ b/scripts/js/package-lock.json @@ -24,7 +24,7 @@ "json-stable-stringify": "^1.1.1", "p-limit": "^6.1.0", "proper-lockfile": "^4.1.2", - "webflow-api": "^1.3.1", + "webflow-api": "^2.4.2", "yaml": "^2.6.0" }, "devDependencies": { @@ -955,31 +955,6 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, - "node_modules/axios": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", - "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "node_modules/axios/node_modules/form-data": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", - "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -1030,6 +1005,30 @@ "balanced-match": "^1.0.0" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -1422,6 +1421,15 @@ "node": ">=6" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -1462,26 +1470,6 @@ "fxparser": "src/cli/cli.js" } }, - "node_modules/follow-redirects": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", - "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/foreground-child": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", @@ -1522,6 +1510,15 @@ "node": ">= 18" } }, + "node_modules/formdata-node": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", + "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1957,6 +1954,26 @@ "node": ">= 14" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/image-size": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.1.1.tgz", @@ -2047,6 +2064,12 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/js-base64": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz", + "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==", + "license": "BSD-3-Clause" + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -2421,6 +2444,15 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/proper-lockfile": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", @@ -2483,12 +2515,6 @@ "node": ">=12.0.0" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -2958,6 +2984,12 @@ "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", "license": "MIT" }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, "node_modules/url-template": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", @@ -2980,12 +3012,62 @@ } }, "node_modules/webflow-api": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/webflow-api/-/webflow-api-1.3.1.tgz", - "integrity": "sha512-ij/Y7t7VqeS2doOhHaCSToKkZeItFwkgCS003mqbG6d51eUmihcJ2ri4SOiR3zTxmUYZO+sg1sF+aAqsY7tgiA==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/webflow-api/-/webflow-api-2.4.2.tgz", + "integrity": "sha512-+znE6V6E6YULwZIGIk8NLFZaimGFH7xVEAjCeivHz4gV13Zcg4FRXyhWxxTHnOYBwKjcjDoaWl8ZK1H9mUA5mQ==", + "dependencies": { + "form-data": "^4.0.0", + "formdata-node": "^6.0.3", + "js-base64": "3.7.2", + "node-fetch": "2.7.0", + "qs": "6.11.2", + "readable-stream": "^4.5.2", + "url-join": "4.0.1" + } + }, + "node_modules/webflow-api/node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", "license": "MIT", "dependencies": { - "axios": "^1.1.3" + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/webflow-api/node_modules/qs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/webflow-api/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/webidl-conversions": { diff --git a/scripts/js/package.json b/scripts/js/package.json index 76fe207..89bfc5e 100644 --- a/scripts/js/package.json +++ b/scripts/js/package.json @@ -28,7 +28,7 @@ "json-stable-stringify": "^1.1.1", "p-limit": "^6.1.0", "proper-lockfile": "^4.1.2", - "webflow-api": "^1.3.1", + "webflow-api": "^2.4.2", "yaml": "^2.6.0" }, "devDependencies": { diff --git a/scripts/js/src/gen_webflow_types.ts b/scripts/js/src/gen_webflow_types.ts index 48dd6c8..977b456 100644 --- a/scripts/js/src/gen_webflow_types.ts +++ b/scripts/js/src/gen_webflow_types.ts @@ -1,10 +1,10 @@ // Generates the webflow_types.ts based on the collection information returned via API. -import Webflow from 'webflow-api'; +import { WebflowClient } from 'webflow-api'; +import type { Field, FieldType } from 'webflow-api/api/types'; import { compile } from 'json-schema-to-typescript'; import { program } from '@commander-js/extra-typings'; import fs from 'node:fs/promises'; -import assert from 'node:assert'; import util from 'util'; @@ -17,12 +17,14 @@ if (!process.env.WEBFLOW_COLLECTION_ID || !process.env.WEBFLOW_API_TOKEN) { console.error('Missing WEBFLOW_COLLECTION_ID or WEBFLOW_API_TOKEN'); process.exit(1); } -const webflow = new Webflow({ token: process.env.WEBFLOW_API_TOKEN }); +const webflow = new WebflowClient({ accessToken: process.env.WEBFLOW_API_TOKEN }); const rootCollectionId = process.env.WEBFLOW_COLLECTION_ID; async function generateTypes(collectionId: string, baseTypeNames: { [k: string]: string }): Promise { - const collection = await webflow.collection({ collectionId }); - + const collection = await webflow.collections.get(collectionId); + if (!collection.slug) { + throw new Error(`Webflow API: slug was not present for collection ${collection.id}`); + } if (!(collection.slug in baseTypeNames)) { console.warn(`No base type name for collection ${collection.slug}, ignoring`); return {}; @@ -49,8 +51,18 @@ async function generateTypes(collectionId: string, baseTypeNames: { [k: string]: res[schemaRead.title] = schemaRead; res[schemaWrite.title] = schemaWrite; - for (const field of collection.fields) { + // We have to do this because WebFlow OpenAPI Spec is incomplete + // https://github.com/webflow/openapi-spec/issues/3 + interface RealField extends Omit { + validations: any; + type: FieldType | 'Option' | 'MultiReference'; + } + + for (const field of collection.fields as RealField[]) { const desc: any = {}; + if (!field.slug) { + throw new Error(`Webflow API: slug was not present for field ${field.displayName}`); + } if ('helpText' in field) { desc.description = field.helpText; } @@ -58,45 +70,46 @@ async function generateTypes(collectionId: string, baseTypeNames: { [k: string]: let propsWrite: any; switch (field.type) { case 'PlainText': - case 'RichText': - case 'Date': case 'Link': case 'Color': - case 'User': propsRead = { type: 'string', ...desc }; - propsWrite = { type: field.required ? 'string' : ['string', 'null'], ...desc }; + propsWrite = { type: field.isRequired ? 'string' : ['string', 'null'], ...desc }; break; - case 'Bool': + case 'Switch': propsRead = { type: 'boolean', ...desc }; - propsWrite = { type: field.required ? 'boolean' : ['boolean', 'null'], ...desc }; + propsWrite = { type: field.isRequired ? 'boolean' : ['boolean', 'null'], ...desc }; break; - case 'ImageRef': + case 'Image': propsRead = { '$ref': '#/$defs/imageRef' }; - propsWrite = { type: field.required ? 'string' : ['string', 'null'], ...desc }; + propsWrite = { type: field.isRequired ? 'string' : ['string', 'null'], ...desc }; break; + case 'Option': { + const values = field.validations.options.map((o: any) => o.name); + propsRead = { type: 'string', ...desc }; + propsWrite = { type: field.isRequired ? 'string' : ['string', 'null'], enum: values, ...desc }; + break; + } case 'Number': propsRead = { type: 'number', ...desc } - propsWrite = { type: field.required ? 'number' : ['number', 'null'], ...desc }; + propsWrite = { type: field.isRequired ? 'number' : ['number', 'null'], ...desc }; break; - case 'Set': - assert((field as any).innerType === 'ImageRef'); + case 'MultiImage': propsRead = { type: 'array', items: { '$ref': '#/$defs/imageRef' }, ...desc }; propsWrite = { - type: field.required ? 'array' : ['array', 'null'], - items: { type: 'string', minItems: field.required ? 1 : 0 }, + type: field.isRequired ? 'array' : ['array', 'null'], + items: { type: 'string', minItems: field.isRequired ? 1 : 0 }, ...desc }; break; - case 'ItemRefSet': + case 'MultiReference': propsRead = { type: 'array', items: { type: 'string' }, ...desc }; propsWrite = { - type: field.required ? 'array' : ['array', 'null'], - items: { type: 'string', minItems: field.required ? 1 : 0 }, + type: field.isRequired ? 'array' : ['array', 'null'], + items: { type: 'string', minItems: field.isRequired ? 1 : 0 }, ...desc }; - const subType = await generateTypes( - (field.validations as any).collectionId, baseTypeNames); + const subType = await generateTypes(field.validations.collectionId, baseTypeNames); // It might be not optimal if the same collection is referenced multiple times, // but it's not a problem for now. res = { ...res, ...subType }; @@ -106,27 +119,19 @@ async function generateTypes(collectionId: string, baseTypeNames: { [k: string]: } schemaRead.properties[field.slug] = propsRead; - if (field.required) { + if (field.isRequired) { schemaRead.required.push(field.slug); } - - const ignoreForWrite = [ - 'created-on', 'updated-on', 'published-on', - 'created-by', 'updated-by', 'published-by' - ]; - if (!ignoreForWrite.includes(field.slug)) { - schemaWrite.properties[field.slug] = propsWrite; - if (field.required) { - schemaWrite.required.push(field.slug); - } + schemaWrite.properties[field.slug] = propsWrite; + if (field.isRequired) { + schemaWrite.required.push(field.slug); } } - return res; } const types = await generateTypes(rootCollectionId, { - 'maps-v2': 'Map', + 'map': 'Map', 'map-tags-v2': 'MapTag', 'map-terrain-types': 'MapTerrain', }); diff --git a/scripts/js/src/maps_metadata.ts b/scripts/js/src/maps_metadata.ts index 614f358..bab602c 100644 --- a/scripts/js/src/maps_metadata.ts +++ b/scripts/js/src/maps_metadata.ts @@ -4,7 +4,7 @@ import { Storage } from '@google-cloud/storage'; import got from 'got'; import path from 'node:path'; import fs from 'node:fs/promises'; -import { writeFileSync, readFileSync, renameSync } from 'node:fs'; +import { writeFileSync, readFileSync, renameSync, mkdirSync } from 'node:fs'; import { randomUUID } from 'node:crypto'; import stream from 'node:stream/promises'; import process from 'node:process'; @@ -48,6 +48,7 @@ async function downloadFile(bucket: string, filePath: string, outputPath: string function loadMapLocationCache(): Map { const mapLocationCacheVersion = 1; + mkdirSync(mapsCacheDir, { recursive: true }); const locationCachePath = path.join(mapsCacheDir, 'mapLocationCache.json'); process.on('beforeExit', () => { @@ -105,7 +106,6 @@ export async function fetchMapsMetadata(maps: MapList): Promise const limit = pLimit(10); // Don't fetch maps metadata from multiple processes in parallel, which happens // when make is called in parallel. - await fs.mkdir(mapsCacheDir, { recursive: true }); const releaseLock = await lock(mapsCacheDir, { lockfilePath: 'mapsMetadata.lock', retries: { diff --git a/scripts/js/src/sync_to_webflow.ts b/scripts/js/src/sync_to_webflow.ts index 40312a9..ef89f59 100644 --- a/scripts/js/src/sync_to_webflow.ts +++ b/scripts/js/src/sync_to_webflow.ts @@ -6,8 +6,8 @@ import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { createHash } from 'node:crypto'; import got from 'got'; -import Webflow from 'webflow-api'; -import { Item as WebflowItem, Collection as WebflowCollection } from 'webflow-api/dist/api'; +import { WebflowClient } from 'webflow-api'; +import type * as Webflow from 'webflow-api/api/types'; import Bottleneck from 'bottleneck'; import { program } from '@commander-js/extra-typings'; import { readMapList, fetchMapsMetadata } from './maps_metadata.js'; @@ -190,12 +190,22 @@ interface IWebsiteItem { } interface IWebflowItemType { - new(i: WebflowItem): IWebflowItem - generateFields(i: IWebsiteItem): { [x: string]: any }; + new(i: Webflow.CollectionItem): IWebflowItem + generateFields(i: IWebsiteItem): Webflow.CollectionItemFieldData; +} + +function fieldsToItem(fields: Webflow.CollectionItemFieldData): Webflow.CollectionItem { + // Need to cast because no id, that's because of + // https://github.com/webflow/openapi-spec/issues/4 + return { + isDraft: false, + isArchived: false, + fieldData: fields + } as Webflow.CollectionItem; } interface IWebflowItem extends IWebsiteItem { - item: WebflowItem; + item: Webflow.CollectionItem; } // WebsiteMapTag is the internal representation of a map tag used in this script. @@ -210,10 +220,11 @@ interface WebflowMapTag extends WebsiteMapTag { } // WebflowMapTag is the native Webflow representation of a tag as used by the // Webflow API. class WebflowMapTag implements IWebflowItem { - item: WebflowItem & WebflowMapTagFieldsRead; + item: Webflow.CollectionItem; - constructor(item: WebflowItem) { - const o = this.item = item as WebflowItem & WebflowMapTagFieldsRead; + constructor(item: Webflow.CollectionItem) { + this.item = item; + const o = item.fieldData as WebflowMapTagFieldsRead; this.name = o.name; this.slug = o.slug; @@ -223,8 +234,6 @@ class WebflowMapTag implements IWebflowItem { return { name: tag.name, slug: tag.slug, - _archived: false, - _draft: false, }; } } @@ -241,10 +250,11 @@ interface WebflowMapTerrain extends WebsiteMapTerrain { } // WebflowMapTerrain is the native Webflow representation of a Terrain as used by the // Webflow API. class WebflowMapTerrain implements IWebflowItem { - item: WebflowItem & WebflowMapTerrainFieldsRead; + item: Webflow.CollectionItem; - constructor(item: WebflowItem) { - const o = this.item = item as WebflowItem & WebflowMapTerrainFieldsRead; + constructor(item: Webflow.CollectionItem) { + this.item = item; + const o = item.fieldData as WebflowMapTerrainFieldsRead; this.name = o.name; this.slug = o.slug; @@ -254,8 +264,6 @@ class WebflowMapTerrain implements IWebflowItem { return { name: terrain.name, slug: terrain.slug, - _archived: false, - _draft: false, }; } } @@ -323,13 +331,19 @@ async function isWebflowMapInfoEqual(a: WebsiteMapInfo, b: WebsiteMapInfo): Prom interface WebflowMapInfo extends WebsiteMapInfo { } +// fieldData is marked as possibly not set for some reason. +type CollectionItemWithData = Webflow.CollectionItem & { fieldData: Webflow.CollectionItemFieldData }; + // WebflowMap is the native Webflow representation of data as used by the // Webflow API. class WebflowMapInfo { - item: WebflowItem & WebflowMapFieldsRead; + // fieldData is always set. + item: CollectionItemWithData; - constructor(item: WebflowItem) { - const o = this.item = item as WebflowItem & WebflowMapFieldsRead; + constructor(item: Webflow.CollectionItem) { + assert(item.fieldData); + this.item = item as CollectionItemWithData; + const o = item.fieldData as WebflowMapFieldsRead; this.name = o.name; this.rowyId = reqRStr(o.rowyid); @@ -361,11 +375,9 @@ class WebflowMapInfo { return { name: info.name, slug: slugFromName(info.name), - _archived: false, - _draft: false, rowyid: info.rowyId, - minimap: await pickImage(info.minimapUrl, base?.item.minimap), - 'minimap-photo-thumb': await pickImage(info.minimapThumbUrl, base?.item['minimap-photo-thumb']), + minimap: await pickImage(info.minimapUrl, base?.item.fieldData.minimap), + 'minimap-photo-thumb': await pickImage(info.minimapThumbUrl, base?.item.fieldData['minimap-photo-thumb']), downloadurl: info.downloadUrl, width: info.width, height: info.height, @@ -373,17 +385,17 @@ class WebflowMapInfo { title: info.title, description: info.description, author: info.author, - 'bg-image': info.bgImageUrl ? await pickImage(info.bgImageUrl, base?.item['bg-image']) : null, - 'perspective-shot': info.perspectiveShotUrl ? await pickImage(info.perspectiveShotUrl, base?.item['perspective-shot']) : null, - 'more-images': await pickImages(info.moreImagesUrl, base?.item['more-images']), + 'bg-image': info.bgImageUrl ? await pickImage(info.bgImageUrl, base?.item.fieldData['bg-image']) : null, + 'perspective-shot': info.perspectiveShotUrl ? await pickImage(info.perspectiveShotUrl, base?.item.fieldData['perspective-shot']) : null, + 'more-images': await pickImages(info.moreImagesUrl, base?.item.fieldData['more-images']), 'wind-min': info.windMin, 'wind-max': info.windMax, 'tidal-strength': info.tidalStrength, 'team-count': info.teamCount, 'max-players': info.maxPlayers, - 'mini-map': await pickImage(info.textureMapUrl, base?.item['mini-map']), - 'height-map': await pickImage(info.heightMapUrl, base?.item['height-map']), - 'metal-map': await pickImage(info.metalMapUrl, base?.item['metal-map']), + 'mini-map': await pickImage(info.textureMapUrl, base?.item.fieldData['mini-map']), + 'height-map': await pickImage(info.heightMapUrl, base?.item.fieldData['height-map']), + 'metal-map': await pickImage(info.metalMapUrl, base?.item.fieldData['metal-map']), 'game-tags-ref-2': info.mapTags, 'terrain-types': info.mapTerrains, }; @@ -461,12 +473,15 @@ async function buildWebflowInfo( return [mapInfo, allMapTags]; } -async function getFieldCollection(field: keyof WebflowMapFieldsRead, mapCollection: WebflowCollection, webflow: Webflow): Promise { - const fields = mapCollection.fields.filter(f => f.slug === field); +async function getFieldCollection(field: keyof WebflowMapFieldsRead, mapCollection: Webflow.Collection, webflow: WebflowClient): Promise { + // We have to do this because WebFlow OpenAPI Spec is incomplete + // https://github.com/webflow/openapi-spec/issues/3 + type RealWebFlowCollectionField = Webflow.Field & { validations: any }; + const fields = mapCollection.fields.filter(f => f.slug === field) as RealWebFlowCollectionField[]; if (fields.length !== 1) { throw new Error(`Expected one field with slug '${field}' in ${mapCollection.slug}, got ${fields.length}`); } - return await webflow.collection({ collectionId: fields[0].validations!.collectionId }); + return await webflow.collections.get(fields[0].validations!.collectionId); } @@ -480,26 +495,26 @@ function resolveItemRefsInMapInfos(mapInfos: Map, field: } throw new Error(`Missing ${field} ${ref}`); } - return t.item._id; + return t.item.id; }); } } -async function getAllWebflowItems(collection: WebflowCollection): Promise { - const items: WebflowItem[] = []; +async function getAllWebflowItems(collection: Webflow.Collection): Promise { + const items: Webflow.CollectionItem[] = []; const limit = 100; for (let offset = 0; true; offset += limit) { - const response = await limiter.schedule(() => collection.items({ limit, offset })); - if (response.length === 0) { + const response = await limiter.schedule(() => webflow.collections.items.listItems(collection.id, { limit, offset })); + if (!response.items || response.items.length === 0) { break; } - items.push(...response); + items.push(...response.items); } return items; } // getAllWebflowMaps returns all maps from the Webflow collection mapped by rowyId. -async function getAllWebflowMaps(mapsCollection: WebflowCollection): Promise> { +async function getAllWebflowMaps(mapsCollection: Webflow.Collection): Promise> { const items = await getAllWebflowItems(mapsCollection); const maps = items.map(item => new WebflowMapInfo(item)); const res = new Map(); @@ -515,14 +530,14 @@ async function getAllWebflowMaps(mapsCollection: WebflowCollection): Promise> { +async function getAllWebflowMapTags(mapTagsCollection: Webflow.Collection): Promise> { const items = await getAllWebflowItems(mapTagsCollection); const tags = items.map(item => new WebflowMapTag(item)); return new Map(tags.map(tag => [tag.slug, tag])); } // getAllWebflowMapTerrains returns all map tags from the Webflow collection mapped by map tag slug. -async function getAllWebflowMapTerrains(mapTagsCollection: WebflowCollection): Promise> { +async function getAllWebflowMapTerrains(mapTagsCollection: Webflow.Collection): Promise> { const items = await getAllWebflowItems(mapTagsCollection); const terrains = items.map(item => new WebflowMapTerrain(item)); return new Map(terrains.map(t => [t.slug, t])); @@ -534,7 +549,7 @@ async function syncCollectionToWebflowAdditions( typeName: string, src: Map, dest: Map, - collection: WebflowCollection, + collection: Webflow.Collection, dryRun: boolean, ) { for (const item of src.values()) { @@ -543,8 +558,11 @@ async function syncCollectionToWebflowAdditions( const fields = webflowItemType.generateFields(item); console.log(`Adding ${typeName} ${item.name}`); if (!dryRun) { - const item = await limiter.schedule(() => collection.createItem(fields)); - dest.set(item.slug, new webflowItemType(item)); + const item = await limiter.schedule( + () => webflow.collections.items.createItem( + collection.id, fieldsToItem(fields))); + assert(item.fieldData!.slug!); + dest.set(item.fieldData!.slug!, new webflowItemType(item)); } else { console.log(fields); } @@ -552,8 +570,13 @@ async function syncCollectionToWebflowAdditions( console.log(`Updating ${typeName} ${item.name}`); const fields = webflowItemType.generateFields(item); if (!dryRun) { - const item = await limiter.schedule(() => webflowTag.item.update(fields)); - dest.set(item.slug, new webflowItemType(item)); + const itemPatch = fieldsToItem(fields); + itemPatch.id = webflowTag.item.id; + const item = await limiter.schedule( + () => webflow.collections.items.updateItem( + collection.id, webflowTag.item.id, itemPatch)); + assert(item.fieldData!.slug!); + dest.set(item.fieldData!.slug!, new webflowItemType(item)); } else { console.log(webflowTag); console.log(fields); @@ -563,6 +586,7 @@ async function syncCollectionToWebflowAdditions( } async function syncCollectionToWebflowRemovals( + collection: Webflow.Collection, typeName: string, src: Map, dest: Map, @@ -572,7 +596,7 @@ async function syncCollectionToWebflowRemovals( if (!src.has(item.slug)) { console.log(`Removing ${typeName} ${item.name}`); if (!dryRun) { - await limiter.schedule(() => item.item.remove()); + await limiter.schedule(() => webflow.collections.items.deleteItem(collection.id, item.item.id)); dest.delete(item.slug); } } @@ -582,35 +606,37 @@ async function syncCollectionToWebflowRemovals( async function syncMapTagsToWebflowAdditions( src: Map, dest: Map, - mapTagsCollection: WebflowCollection, + mapTagsCollection: Webflow.Collection, dryRun: boolean ) { return syncCollectionToWebflowAdditions(WebflowMapTag, isWebsiteMapTagEqual, 'tag', src, dest, mapTagsCollection, dryRun); } async function syncMapTagsToWebflowRemovals( + collection: Webflow.Collection, src: Map, dest: Map, dryRun: boolean ) { - return syncCollectionToWebflowRemovals('tag', src, dest, dryRun); + return syncCollectionToWebflowRemovals(collection, 'tag', src, dest, dryRun); } async function syncMapTerrainsToWebflowAdditions( src: Map, dest: Map, - mapTerrainsCollection: WebflowCollection, + mapTerrainsCollection: Webflow.Collection, dryRun: boolean ) { return syncCollectionToWebflowAdditions(WebflowMapTag, isWebsiteMapTerrainEqual, 'terrain', src, dest, mapTerrainsCollection, dryRun); } async function syncMapTerrainsToWebflowRemovals( + collection: Webflow.Collection, src: Map, dest: Map, dryRun: boolean ) { - return syncCollectionToWebflowRemovals('terrain', src, dest, dryRun); + return syncCollectionToWebflowRemovals(collection, 'terrain', src, dest, dryRun); } function getRowyMapTerrains(): Map { @@ -621,7 +647,7 @@ function getRowyMapTerrains(): Map { async function syncMapsToWebflow( src: Map, dest: Map, - mapsCollection: WebflowCollection, + mapsCollection: Webflow.Collection, dryRun: boolean ) { const updatesP: Promise<[boolean, WebsiteMapInfo, WebflowMapInfo]>[] = []; @@ -631,7 +657,9 @@ async function syncMapsToWebflow( const fields = await WebflowMapInfo.generateFields(map); console.log(`Adding ${map.name}`); if (!dryRun) { - const item = await limiter.schedule(() => mapsCollection.createItem(fields)); + const item = await limiter.schedule( + () => webflow.collections.items.createItem( + mapsCollection.id, fieldsToItem(fields))); dest.set(map.rowyId, new WebflowMapInfo(item)); } else { console.log(fields); @@ -644,7 +672,7 @@ async function syncMapsToWebflow( if (!src.has(map.rowyId)) { console.log(`Removing ${map.name}`); if (!dryRun) { - await limiter.schedule(() => map.item.remove()); + await limiter.schedule(() => webflow.collections.items.deleteItem(mapsCollection.id, map.item.id)); dest.delete(map.rowyId); } } @@ -654,7 +682,11 @@ async function syncMapsToWebflow( console.log(`Updating ${map.name}`); const fields = await WebflowMapInfo.generateFields(map, webflowMap); if (!dryRun) { - const item = await limiter.schedule(() => webflowMap.item.update(fields)); + const itemPatch = fieldsToItem(fields); + itemPatch.id = webflowMap.item.id; + const item = await limiter.schedule( + () => webflow.collections.items.updateItem( + mapsCollection.id, webflowMap.item.id, itemPatch)); dest.set(map.rowyId, new WebflowMapInfo(item)); } else { console.log(webflowMap); @@ -663,17 +695,17 @@ async function syncMapsToWebflow( } } -async function publishUpdatedWebflowItems(mapsCollection: WebflowCollection, items: Map, dryRun: boolean) { +async function publishUpdatedWebflowItems(collection: Webflow.Collection, items: Map, dryRun: boolean) { const itemIds = Array.from(items.values()) .map(i => i.item) - .filter(i => !i['published-on'] || Date.parse(i['published-on']) < Date.parse(i['updated-on'])) - .map(i => i._id); + .filter(i => !i.lastPublished || Date.parse(i.lastPublished) < Date.parse(i.lastUpdated!)) + .map(i => i.id); console.log(`Publishing ${itemIds.length} items`); if (!dryRun) { const chunkSize = 100; for (let i = 0; i < itemIds.length; i += chunkSize) { const itemIdsChunk = itemIds.slice(i, i + chunkSize); - await limiter.schedule(() => webflow.publishItems({ collectionId: mapsCollection._id, itemIds: itemIdsChunk })); + await limiter.schedule(() => webflow.collections.items.publishItem(collection.id, { itemIds: itemIdsChunk })); } } } @@ -684,12 +716,12 @@ if (!process.env.WEBFLOW_COLLECTION_ID || !process.env.WEBFLOW_API_TOKEN) { console.error('Missing WEBFLOW_COLLECTION_ID or WEBFLOW_API_TOKEN'); process.exit(1); } -const webflow = new Webflow({ token: process.env.WEBFLOW_API_TOKEN }); +const webflow = new WebflowClient({ accessToken: process.env.WEBFLOW_API_TOKEN }); const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 600 }); const mapsCollectionId = process.env.WEBFLOW_COLLECTION_ID; async function syncCommand(dryRun: boolean) { - const mapsCollection = await limiter.schedule(() => webflow.collection({ collectionId: mapsCollectionId })); + const mapsCollection = await limiter.schedule(() => webflow.collections.get(mapsCollectionId)); const webflowMaps = await getAllWebflowMaps(mapsCollection); const mapTagsCollection = await getFieldCollection('game-tags-ref-2', mapsCollection, webflow); const webflowMapTags = await getAllWebflowMapTags(mapTagsCollection); @@ -710,8 +742,8 @@ async function syncCommand(dryRun: boolean) { await publishUpdatedWebflowItems(mapTerrainsCollection, webflowMapTerrains, dryRun); await publishUpdatedWebflowItems(mapTagsCollection, webflowMapTags, dryRun); await publishUpdatedWebflowItems(mapsCollection, webflowMaps, dryRun); - await syncMapTagsToWebflowRemovals(rowyMapTagsInfo, webflowMapTags, dryRun); - await syncMapTerrainsToWebflowRemovals(rowyMapTerrainsInfo, webflowMapTerrains, dryRun); + await syncMapTagsToWebflowRemovals(mapTagsCollection, rowyMapTagsInfo, webflowMapTags, dryRun); + await syncMapTerrainsToWebflowRemovals(mapTerrainsCollection, rowyMapTerrainsInfo, webflowMapTerrains, dryRun); } catch (e: any) { // To make sure we will get full info from inside of the response. if ('message' in e) { @@ -734,7 +766,7 @@ program.command('sync') program.command('dump-data') .description('Dumps Webflow collection data.') .action(async () => { - const mapsCollection = await limiter.schedule(() => webflow.collection({ collectionId: mapsCollectionId })); + const mapsCollection = await limiter.schedule(() => webflow.collections.get(mapsCollectionId)); const webflowMaps = await getAllWebflowMaps(mapsCollection); console.log(util.inspect(webflowMaps, { showHidden: false, depth: null, colors: true })); diff --git a/scripts/js/src/webflow_types.ts b/scripts/js/src/webflow_types.ts index eec0c65..5c893b9 100644 --- a/scripts/js/src/webflow_types.ts +++ b/scripts/js/src/webflow_types.ts @@ -87,16 +87,9 @@ export interface WebflowMapFieldsRead { * Surface of the map (W * H) */ mapsize?: number; + "startpos-code"?: string; name: string; slug: string; - _archived: boolean; - _draft: boolean; - "created-on"?: string; - "updated-on"?: string; - "published-on"?: string; - "created-by"?: string; - "updated-by"?: string; - "published-by"?: string; } /** * This interface was referenced by `undefined`'s JSON-Schema @@ -190,10 +183,9 @@ export interface WebflowMapFieldsWrite { * Surface of the map (W * H) */ mapsize?: number | null; + "startpos-code"?: string | null; name: string; slug: string; - _archived: boolean; - _draft: boolean; } /** * This interface was referenced by `undefined`'s JSON-Schema @@ -205,14 +197,6 @@ export interface WebflowMapTagFieldsRead { icon?: WebflowImageRef; name: string; slug: string; - _archived: boolean; - _draft: boolean; - "created-on"?: string; - "updated-on"?: string; - "published-on"?: string; - "created-by"?: string; - "updated-by"?: string; - "published-by"?: string; } /** * This interface was referenced by `undefined`'s JSON-Schema @@ -224,32 +208,36 @@ export interface WebflowMapTagFieldsWrite { icon?: string | null; name: string; slug: string; - _archived: boolean; - _draft: boolean; } /** * This interface was referenced by `undefined`'s JSON-Schema * via the `definition` "WebflowMapTerrainFieldsRead". */ export interface WebflowMapTerrainFieldsRead { + icon?: WebflowImageRef; + description?: string; + glow?: string; + /** + * Glow with set color + */ + "show-glow"?: boolean; + category?: string; name: string; slug: string; - _archived: boolean; - _draft: boolean; - "created-on"?: string; - "updated-on"?: string; - "published-on"?: string; - "created-by"?: string; - "updated-by"?: string; - "published-by"?: string; } /** * This interface was referenced by `undefined`'s JSON-Schema * via the `definition` "WebflowMapTerrainFieldsWrite". */ export interface WebflowMapTerrainFieldsWrite { + icon?: string | null; + description?: string | null; + glow?: string | null; + /** + * Glow with set color + */ + "show-glow"?: boolean | null; + category?: ("Global Biome" | "Specific Feature" | "Water" | "Layout") | null; name: string; slug: string; - _archived: boolean; - _draft: boolean; }