From 4855a05e99e8471ec91d57cf874fd99a5994b82c Mon Sep 17 00:00:00 2001 From: mmeigs Date: Tue, 6 Feb 2024 12:05:14 -0500 Subject: [PATCH] DOP-4247: Build Artifacts (#1000) --- .github/workflows/stage-docs-site.yml | 40 ++++++++++++++++++ .../gatsby-source-snooty-prod/gatsby-node.js | 2 +- src/init/DocumentDatabase.js | 41 +++++++++++++++---- src/utils/setup/fetch-manifest-metadata.js | 12 +++++- 4 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/stage-docs-site.yml diff --git a/.github/workflows/stage-docs-site.yml b/.github/workflows/stage-docs-site.yml new file mode 100644 index 000000000..1fed581f2 --- /dev/null +++ b/.github/workflows/stage-docs-site.yml @@ -0,0 +1,40 @@ +on: + push: + branches: + - "main" + pull_request: + types: [opened, synchronize] +name: Build docs site artifact +jobs: + staging: + permissions: write-all + runs-on: ubuntu-latest + strategy: + matrix: + project: ['cloud-docs', 'docs'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '18.x' + - name: Access Build Data + uses: mongodb/docs-worker-actions/build-artifact@main + env: + PROJECT_TO_BUILD: ${{ matrix.project }} + - name: Build Snooty + env: + NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE_64_AUTH }} + NPM_EMAIL: ${{ secrets.NPM_EMAIL }} + GATSBY_BUILD_FROM_JSON: true + GATSBY_PARSER_USER: docsworker-xlarge + GATSBY_SITE: ${{ matrix.project }} + GATSBY_PARSER_BRANCH: main + run: | + npm ci --legacy-peer-deps + npm run build + - name: Save Artifact + uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.project }} + path: | + public diff --git a/plugins/gatsby-source-snooty-prod/gatsby-node.js b/plugins/gatsby-source-snooty-prod/gatsby-node.js index 974b2ec6e..96014cbf2 100644 --- a/plugins/gatsby-source-snooty-prod/gatsby-node.js +++ b/plugins/gatsby-source-snooty-prod/gatsby-node.js @@ -101,7 +101,7 @@ exports.sourceNodes = async ({ actions, createContentDigest, createNodeId }) => // wait to connect to Realm - if (siteMetadata.manifestPath) { + if (siteMetadata.manifestPath || process.env.GATSBY_BUILD_FROM_JSON === 'true') { console.log('Loading documents from manifest'); db = manifestDocumentDatabase; } else { diff --git a/src/init/DocumentDatabase.js b/src/init/DocumentDatabase.js index d776f06a7..258113213 100644 --- a/src/init/DocumentDatabase.js +++ b/src/init/DocumentDatabase.js @@ -1,5 +1,7 @@ const AdmZip = require('adm-zip'); const BSON = require('bson'); +const fs = require('fs'); +const { promisify } = require('util'); const { initRealm } = require('../utils/setup/init-realm'); const { DOCUMENTS_COLLECTION, METADATA_COLLECTION, ASSETS_COLLECTION } = require('../build-constants'); const { manifestMetadata, siteMetadata } = require('../utils/site-metadata'); @@ -8,6 +10,8 @@ const { constructBuildFilter } = require('../utils/setup/construct-build-filter' const DB = siteMetadata.database; const buildFilter = constructBuildFilter(siteMetadata); +const readFileAsync = promisify(fs.readFile); + class RealmInterface { constructor() { this.realmClient = null; @@ -55,7 +59,8 @@ class RealmInterface { class ManifestDocumentDatabase { constructor(path) { - this.zip = new AdmZip(path); + // Allow no zip if building artifact through Github Action + this.zip = process.env.GATSBY_BUILD_FROM_JSON !== 'true' ? new AdmZip(path) : null; this.realmInterface = new RealmInterface(); } @@ -65,11 +70,22 @@ class ManifestDocumentDatabase { async getDocuments() { const result = []; - const zipEntries = this.zip.getEntries(); - for (const entry of zipEntries) { - if (entry.entryName.startsWith('documents/')) { - const doc = BSON.deserialize(entry.getData()); - result.push(doc); + if (!this.zip && process.env.GATSBY_BUILD_FROM_JSON === 'true') { + // Read documents from Gatsby Action download + try { + const documents = JSON.parse(await readFileAsync('snooty-documents.json')); + return documents; + } catch (err) { + console.error('No Manifest Path was found.'); + return result; + } + } else { + const zipEntries = this.zip.getEntries(); + for (const entry of zipEntries) { + if (entry.entryName.startsWith('documents/')) { + const doc = BSON.deserialize(entry.getData()); + result.push(doc); + } } } return result; @@ -80,9 +96,20 @@ class ManifestDocumentDatabase { } async getAsset(checksum) { + if (!this.zip && process.env.GATSBY_BUILD_FROM_JSON === 'true') { + // Read assets from Gatsby Action download + try { + const asset = await readFileAsync(`assets/${checksum}`, { encoding: 'base64' }); + return Buffer.from(asset, 'base64'); + } catch (err) { + console.error('No Manifest Path was found.'); + return null; + } + } const result = this.zip.getEntry(`assets/${checksum}`); if (result) { - return result.getData(); + const buffer = result.getData(); + return buffer; } return null; } diff --git a/src/utils/setup/fetch-manifest-metadata.js b/src/utils/setup/fetch-manifest-metadata.js index 2c5457bf8..d6a32520a 100644 --- a/src/utils/setup/fetch-manifest-metadata.js +++ b/src/utils/setup/fetch-manifest-metadata.js @@ -1,10 +1,20 @@ const AdmZip = require('adm-zip'); const BSON = require('bson'); +const fs = require('fs'); // Returns the metadata from the manifest file if provided const fetchManifestMetadata = () => { let metadata = {}; - if (process.env.GATSBY_MANIFEST_PATH) { + if (!process.env.GATSBY_MANIFEST_PATH && process.env.GATSBY_BUILD_FROM_JSON === 'true') { + // Read metadata from Gatsby Action download + try { + metadata = JSON.parse(fs.readFileSync('snooty-metadata.json')); + return metadata; + } catch (err) { + console.error('No metadata was found.'); + return metadata; + } + } else if (process.env.GATSBY_MANIFEST_PATH) { const zip = new AdmZip(process.env.GATSBY_MANIFEST_PATH); const zipEntries = zip.getEntries(); for (const entry of zipEntries) {