diff --git a/ext/system-resources/README.md b/ext/system-resources/README.md index 5652a7382c..d5cf220fe2 100644 --- a/ext/system-resources/README.md +++ b/ext/system-resources/README.md @@ -1,3 +1,28 @@ # `system-resources` -Build scripts for bundled system resources. \ No newline at end of file +Build scripts for bundled system resources. + +## Index of resources + +1. Chat - basic chat functionality + - Source-code-based +2. Monitor - server manager, txAdmin + - Artifact-based + - Source code: https://github.com/tabarra/txAdmin + - Using ready-built artifacts from the source code repository + +## Updating artifact-based resources + +Update artifact URL, version and SHA256 hash using this command: + +``` +$ node manager.js update --name=%RESOURCE_NAME% --url=%RESOURCE_ARTIFACT_URL% --version=%RESOURCE_VERSION% +``` + +## Adding artifact-based resource + +``` +$ node manager.js add --name=%RESOURCE_NAME% --url=%RESOURCE_ARTIFACT_URL% --version=%RESOURCE_VERSION% +``` + +> Requires adding extra build steps to `build.cmd` and `build.sh` scripts. diff --git a/ext/system-resources/build.cmd b/ext/system-resources/build.cmd index f191bc6bc9..90246bbd0d 100644 --- a/ext/system-resources/build.cmd +++ b/ext/system-resources/build.cmd @@ -1,18 +1,20 @@ @echo off where /q node - if errorlevel 1 ( - exit /B 1 + goto :error ) -if "%1" == "chat" goto chat +if "%1" == "chat" ( + goto :chat +) set SRRoot=%~dp0\data pushd %~dp0 + :: txAdmin -set MonitorArtifactURL="https://github.com/tabarra/txAdmin/releases/download/v7.0.0/monitor.zip" +echo Adding monitor set MonitorPath=%SRRoot%\monitor set MonitorArtifactPath=%SRRoot%\monitor.zip @@ -20,39 +22,55 @@ set MonitorArtifactPath=%SRRoot%\monitor.zip rmdir /s /q %MonitorPath% mkdir %MonitorPath% -curl -Lo %MonitorArtifactPath% %MonitorArtifactURL% -tar -C %MonitorPath%\ -xf %MonitorArtifactPath% +node manager.js download --name=monitor --file=%MonitorArtifactPath% || goto :error + +tar -C %MonitorPath%\ -xf %MonitorArtifactPath% || goto :error del %MonitorArtifactPath% +echo Done adding monitor +:: /txAdmin + + :: chat :chat +echo Adding chat pushd resources\chat rmdir /s /q dist -node %~dp0\..\native-doc-gen\yarn_cli.js +node %~dp0\..\native-doc-gen\yarn_cli.js || goto :error set NODE_OPTIONS="" FOR /F %%g IN ('node -v') do (set NODE_VERSION_STRING=%%g) set /a NODE_VERSION="%NODE_VERSION_STRING:~1,2%" IF %NODE_VERSION% GEQ 18 (set NODE_OPTIONS=--openssl-legacy-provider) -call node_modules\.bin\webpack.cmd +call node_modules\.bin\webpack.cmd || goto :error popd rmdir /s /q %SRRoot%\chat\ rmdir /s /q resources\chat\node_modules\ -xcopy /y /e resources\chat\ %SRRoot%\chat\ +xcopy /y /e resources\chat\ %SRRoot%\chat\ || goto :error del %SRRoot%\chat\yarn.lock del %SRRoot%\chat\package.json rmdir /s /q %SRRoot%\chat\html\ mkdir %SRRoot%\chat\html\vendor\ -xcopy /y /e resources\chat\html\vendor %SRRoot%\chat\html\vendor +xcopy /y /e resources\chat\html\vendor %SRRoot%\chat\html\vendor || goto :error popd -:: done! -exit /B 0 \ No newline at end of file +echo Done adding chat +goto :success +:: /chat + + +:error +echo Failed to build system resources +exit /b %errorlevel% + + +:success +exit /b 0 diff --git a/ext/system-resources/build.sh b/ext/system-resources/build.sh index 17c300ca4b..51121c1204 100644 --- a/ext/system-resources/build.sh +++ b/ext/system-resources/build.sh @@ -7,15 +7,14 @@ npm install -g npm@7.19.1 mkdir -p data # txAdmin -MONITOR_ARTIFACT_URL="https://github.com/tabarra/txAdmin/releases/download/v7.0.0/monitor.zip" - MONITOR_PATH=data/monitor MONITOR_ARTIFACT_PATH=data/monitor.zip rm -rf $MONITOR_PATH mkdir -p $MONITOR_PATH -curl -Lo $MONITOR_ARTIFACT_PATH $MONITOR_ARTIFACT_URL +node manager.js download --name=monitor --file=$MONITOR_ARTIFACT_PATH + unzip $MONITOR_ARTIFACT_PATH -d $MONITOR_PATH rm $MONITOR_ARTIFACT_PATH diff --git a/ext/system-resources/manager.js b/ext/system-resources/manager.js new file mode 100644 index 0000000000..78b7a2d15a --- /dev/null +++ b/ext/system-resources/manager.js @@ -0,0 +1,283 @@ +/** + * System Resources artifacts and manifest manager + */ +const fs = require('fs'); +const http = require('http'); +const https = require('https'); +const crypto = require('crypto'); + +const MANIFEST_FILE = './manifest.json'; + +main(parseArgs(process.argv.slice(2))).catch((error) => { + console.error(error); + process.exit(1); +}); + +/** + * @param {{ args: string[], options: Record }} param0 + * @returns + */ +async function main({ args, options }) { + const [cmd] = args; + + const resourceName = check(options.name, 'No --name provided'); + const resourceFile = check(options.file, 'No --file provided'); + + switch (cmd) { + case 'download': { + const skipHashVerification = Boolean(options.noverify); + + // Get manifest and check that we have passed resource name in it + const manifest = require(MANIFEST_FILE); + if (!manifest[resourceName]) { + return exitError('Resource name unknown'); + } + + // Get resource manifest + const resourceManifest = manifest[resourceName]; + + // Download resource file + await download(resourceManifest.url, resourceFile); + + // Check if resource file at least isn't empty + const resourceFileStat = await getFileStatSafe(resourceFile); + if (resourceFileStat && resourceFileStat.size === 0) { + return exitError(`Downloaded resource ${resourceName} file is empty`); + } + + if (skipHashVerification) { + console.log(`Resource file ${resourceFile} for ${resourceName} has been downloaded`); + } else { + // Verify resource file SHA256 hashsum against the one in resource' manifest + const actualSHA256 = await getSHA256(resourceFile); + + if (resourceManifest.sha256 !== actualSHA256) { + console.log(`Invalid SHA256 of resource ${resourceName} file ${resourceFile}`); + console.log('Expected:', resourceManifest.sha256); + console.log('Actual:', actualSHA256); + + return exitError('Resource file integrity check failed'); + } + + console.log(`Resource file ${resourceFile} for ${resourceName} has been downloaded and verified`); + } + + break; + } + + case 'gen-sha256': { + if (!(await fileExists(resourceFile))) { + const errorMessage = `Resource file does not exist, try "node manager.js download --name=${resourceName} --file=${resourceFile} --noverify" it first`; + + return exitError(errorMessage); + } + + console.log(await getSHA256(resourceFile)); + + break; + } + + case 'add': + case 'update': { + const resourceUrl = check(options.url, 'No --url provided'); + const resourceVersion = check(options.version, 'No --version provided'); + + const manifest = require(MANIFEST_FILE); + const resourceManifest = manifest[resourceName] || {}; + const isResourceManifestNew = Object.keys(resourceManifest).length === 0; + + if (cmd === 'add') { + if (!isResourceManifestNew) { + return exitError('Cannot add resource as it already exists, try updating instead'); + } + } else { + if (isResourceManifestNew) { + return exitError('Resource name unknown, see manifest.json for existing resources'); + } + } + + // Add resourceManifest to manifest in case it's new + manifest[resourceName] = resourceManifest; + + // Download temporary resource' artifact + await download(resourceUrl, resourceFile); + + // Compute artifact hash + const sha256 = await getSHA256(resourceFile); + + // Delete temporary resource' artifact + await fs.promises.unlink(resourceFile); + + // Update resource manifest + resourceManifest.url = resourceUrl; + resourceManifest.version = resourceVersion; + resourceManifest.sha256 = sha256; + + // Write updated manifest + await fs.promises.writeFile(MANIFEST_FILE, JSON.stringify(manifest, null, 2)); + + console.log(`Updated resource ${resourceName} manifest:`, resourceManifest); + + break; + } + } +} + +/** + * Checks if input is a string and it is not empty + * + * @param {string | boolean} inp + * @param {string} error + * @returns {string} + */ +function check(inp, error) { + if (typeof inp !== 'string') { + return exitError('Expected string value, got', typeof inp); + } + + if (!inp) { + return exitError(error); + } + + return inp; +} + +function exitError(message, code = 1) { + console.error(message); + process.exit(code); +} + +/** + * Returns hex-encoded SHA256 hashsum of the file content + * + * @param {string} filename + * @returns {Promise} + */ +async function getSHA256(filename) { + const fileStat = await getFileStatSafe(filename); + if (!fileStat || fileStat.isDirectory()) { + return exitError(`${filename} does not exists or is a directory`); + } + + return new Promise((resolve, reject) => { + const hash = crypto.createHash('sha256'); + const fileStream = fs.createReadStream(filename); + + fileStream.on('data', (data) => { + hash.update(data); + }) + fileStream.on('end', () => { + resolve(hash.digest('hex')); + }); + fileStream.on('error', (err) => { + reject(err); + }); + }); +} + +/** + * Returns fs.Stats object if file/directory exists, null otherwise + * + * Does the same thing as fs.promises.stat, but doesn't throw if stat operation was not successfull + * + * @param {string} filename + * @returns {Promise} + */ +async function getFileStatSafe(filename) { + return fs.promises.stat(filename).catch(() => null); +} + +/** + * Checks if file/directory exists + * + * @param {string} filename + * @returns {Promise} + */ +async function fileExists(filename) { + const fileStat = await getFileStatSafe(filename); + + return fileStat !== null; +} + +/** + * Simple downloader, will follow redirects + * + * @param {string} url + * @param {string} filename + */ +async function download(url, filename, redirectsCount = 0) { + const DOWNLOAD_MAX_REDIRECTS = 50; + + // If not redirect - make sure file doesn't exist + if (redirectsCount === 0 && await fileExists(filename)) { + await fs.promises.unlink(filename); + } + + if (redirectsCount > DOWNLOAD_MAX_REDIRECTS) { + throw new Error(`Failed to download, maximum amount of redirects (${DOWNLOAD_MAX_REDIRECTS}) exceeded`); + } + + const isHttp = url.startsWith('http://'); + const isHttps = url.startsWith('https://'); + + if (!isHttp && !isHttps) { + throw new Error('Unknown download url protocol, only http and https is supported'); + } + + const protocol = isHttp ? http : https; + + await new Promise((resolve, reject) => { + protocol.get(url, (res) => { + if (res.statusCode >= 200 && res.statusCode < 300) { + const fileStream = fs.createWriteStream(filename); + + fileStream.on('error', reject); + fileStream.on('finish', () => { + fileStream.close(); + resolve(); + }); + + res.pipe(fileStream); + } else if (res.headers.location) { + // If redirect - recurse with a given location + download(res.headers.location, filename, redirectsCount + 1).then(resolve, reject); + } else { + reject(new Error(res.statusCode + ' ' + res.statusMessage)); + } + }); + }); +} + +/** + * Parse args array + * + * @param {string[]} args + * @returns {{ args: string[], options: Record }} + */ +function parseArgs(args) { + const map = { + args: [], + options: {}, + }; + + for (const arg of args) { + // Doesn't start with `--` - arbitrary arg + if (!arg.startsWith('--')) { + map.args.push(arg); + continue; + } + + const argNormalized = arg.substring(2); + + // If arg has no value specified - treat it like a boolean flag + if (!argNormalized.includes('=')) { + map.options[argNormalized] = true; + continue; + } + + const [argName, argValue] = argNormalized.split('='); + map.options[argName] = argValue; + } + + return map; +} diff --git a/ext/system-resources/manifest.json b/ext/system-resources/manifest.json new file mode 100644 index 0000000000..88c88c1b6f --- /dev/null +++ b/ext/system-resources/manifest.json @@ -0,0 +1,7 @@ +{ + "monitor": { + "url": "https://github.com/tabarra/txAdmin/releases/download/v7.0.0/monitor.zip", + "version": "7.0.0", + "sha256": "cedd202dc90f8d75d82160528c57c79d21dcd8828ac73f4c43148f36526ea84b" + } +} \ No newline at end of file