From 2411e188ee3400eb68a136526644d9cb76622e5e Mon Sep 17 00:00:00 2001 From: Max Klein Date: Sat, 7 Sep 2024 16:21:47 -0400 Subject: [PATCH] robustify python environment installation (#13) * creates an INSTALLER record on correct installation of python env * run linux and windows debug build CI on all pull requests and pushes to main * add optional .env file to assets * use cpu-only assets build on linux and windows CI * include everything in assets/ in resources of built app * removed redundant `assets.sh` script * trim trailing whitespace * refactor `promise.then()` syntax to use `await` --- .github/actions/build/linux/app/action.yml | 3 +- .github/actions/build/windows/app/action.yml | 9 +- .github/workflows/debug_linux.yml | 7 +- .github/workflows/debug_windows.yml | 9 +- forge.config.ts | 2 +- package.json | 6 +- scripts/assets.sh | 6 - scripts/env.mjs | 7 ++ src/main.ts | 109 ++++++++++--------- yarn.lock | 8 ++ 10 files changed, 94 insertions(+), 72 deletions(-) delete mode 100755 scripts/assets.sh create mode 100644 scripts/env.mjs diff --git a/.github/actions/build/linux/app/action.yml b/.github/actions/build/linux/app/action.yml index fef71cf7..1ab506a3 100644 --- a/.github/actions/build/linux/app/action.yml +++ b/.github/actions/build/linux/app/action.yml @@ -52,7 +52,8 @@ runs: run: | set -x pip install comfy-cli - yarn make:assets:nvidia + yarn make:assets:cpu + yarn clean:assets:dev shell: bash - name: Make app shell: bash diff --git a/.github/actions/build/windows/app/action.yml b/.github/actions/build/windows/app/action.yml index 367b47cd..9eded064 100644 --- a/.github/actions/build/windows/app/action.yml +++ b/.github/actions/build/windows/app/action.yml @@ -35,12 +35,11 @@ runs: python-version: '3.12' - name: Install ComfyUI and create standalone package run: | + set -x pip install comfy-cli - cd assets - comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --nvidia --cuda-version 12.1 - comfy --workspace ./ComfyUI standalone --platform windows --proc x86_64 - rm -rf python cpython*.tar.gz - shell: cmd + yarn make:assets:cpu + yarn clean:assets:dev + shell: bash - name: Make app shell: powershell env: diff --git a/.github/workflows/debug_linux.yml b/.github/workflows/debug_linux.yml index d49e468b..fe023596 100644 --- a/.github/workflows/debug_linux.yml +++ b/.github/workflows/debug_linux.yml @@ -1,11 +1,12 @@ name: Build App Linux Debug on: - workflow_dispatch: - workflow_call: push: branches: - - fix-python-server-startup # just for testing + - main + pull_request: + branches: + - main jobs: build-linux-debug: diff --git a/.github/workflows/debug_windows.yml b/.github/workflows/debug_windows.yml index 2b7cc4a5..9e3353fd 100644 --- a/.github/workflows/debug_windows.yml +++ b/.github/workflows/debug_windows.yml @@ -1,11 +1,12 @@ name: Build Windows Debug on: - workflow_dispatch: - workflow_call: push: branches: - - fix-python-server-startup # just for testing + - main + pull_request: + branches: + - main jobs: build-windows-debug: @@ -27,4 +28,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: comfyui-electron-debug-win32-${{env.sha_short}} - path: out/make/zip/win32/x64/*.zip \ No newline at end of file + path: out/make/zip/win32/x64/*.zip diff --git a/forge.config.ts b/forge.config.ts index b1002e56..840348cf 100644 --- a/forge.config.ts +++ b/forge.config.ts @@ -31,7 +31,7 @@ const config: ForgeConfig = { teamId: process.env.APPLE_TEAM_ID }, }, - extraResource: ['./assets/UI', './assets/ComfyUI', './assets/python.tgz'], + extraResource: ['./assets'], }, rebuildConfig: {}, diff --git a/package.json b/package.json index 56acc03f..d0640a2d 100644 --- a/package.json +++ b/package.json @@ -7,13 +7,14 @@ "packageManager": "yarn@4.4.1", "scripts": { "clean": "rimraf .vite dist out", - "clean:assets": "rimraf assets/ComfyUI assets/python assets/override.txt assets/python.tgz || rimraf assets/cpython*.tar.gz || rimraf assets/requirements.*", + "clean:assets": "rimraf assets/.env assets/ComfyUI assets/python.tgz && yarn run clean:assets:dev", + "clean:assets:dev": "rimraf assets/python assets/override.txt | rimraf assets/cpython*.tar.gz | rimraf assets/requirements.*", "clean:slate": "yarn run clean && yarn run clean:assets && rimraf node_modules", "lint": "eslint --ext .ts,.tsx .", "lint:fix": "eslint --fix --ext .ts,.tsx .", "make": "electron-forge make", "make:assets:amd": "cd assets && comfy-cli --skip-prompt --here install --fast-deps --amd && comfy-cli --here standalone", - "make:assets:cpu": "cd assets && comfy-cli --skip-prompt --here install --fast-deps --cpu && comfy-cli --here standalone", + "make:assets:cpu": "cd assets && comfy-cli --skip-prompt --here install --fast-deps --cpu && comfy-cli --here standalone && node ../scripts/env.mjs", "make:assets:nvidia": "cd assets && comfy-cli --skip-prompt --here install --fast-deps --nvidia && comfy-cli --here standalone", "notarize": "node debug/notarize.js", "package": "electron-forge package", @@ -60,6 +61,7 @@ "dependencies": { "adm-zip": "^0.5.15", "axios": "^1.7.3", + "dotenv": "^16.4.5", "electron-squirrel-startup": "^1.0.1", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/scripts/assets.sh b/scripts/assets.sh deleted file mode 100755 index 0a25abbf..00000000 --- a/scripts/assets.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -comfy-cli --skip-prompt --workspace ./ComfyUI install --fast-deps --amd -comfy-cli --workspace ./ComfyUI standalone -rm -rf python cpython*.tar.gz diff --git a/scripts/env.mjs b/scripts/env.mjs new file mode 100644 index 00000000..22f67377 --- /dev/null +++ b/scripts/env.mjs @@ -0,0 +1,7 @@ +import * as fs from "node:fs/promises"; + +const envContent = `# env vars picked up by the ComfyUI executable on startup +COMFYUI_CPU_ONLY=true +` + +fs.writeFile(".env", envContent); diff --git a/src/main.ts b/src/main.ts index fd8a1ded..841b373c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,8 +1,9 @@ import { spawn, ChildProcess } from 'node:child_process'; -import { access, mkdir, readdir, rm } from 'node:fs/promises'; +import * as fs from 'node:fs/promises'; import net from 'node:net'; import path from 'node:path'; +import dotenv from "dotenv"; import { app, BrowserWindow } from 'electron'; // Handle creating/removing shortcuts on Windows when installing/uninstalling. import('electron-squirrel-startup').then(ess => { @@ -64,7 +65,9 @@ const isPortInUse = (host: string, port: number): Promise => { }; -const launchPythonServer = async () => { +const launchPythonServer = async (args: {userResourcesPath: string, appResourcesPath: string}) => { + const {userResourcesPath, appResourcesPath} = args; + const isServerRunning = await isPortInUse(host, port); if (isServerRunning) { console.log('Python server is already running'); @@ -74,37 +77,14 @@ const launchPythonServer = async () => { console.log('Launching Python server...'); return new Promise(async (resolve, reject) => { - const {userResourcesPath, appResourcesPath} = app.isPackaged ? { - // production: install python to per-user application data dir - userResourcesPath: app.getPath('appData'), - appResourcesPath: process.resourcesPath, - } : { - // development: install python to in-tree assets dir - userResourcesPath: path.join(app.getAppPath(), 'assets'), - appResourcesPath: path.join(app.getAppPath(), 'assets'), - } - - try { - await mkdir(userResourcesPath); - } catch { - null; - } - console.log(`userResourcesPath: ${userResourcesPath}`); - console.log(`appResourcesPath: ${appResourcesPath}`); - - const {pythonPath, scriptPath} = process.platform==='win32' ? { - pythonPath: path.join(userResourcesPath, 'python', 'python.exe'), - scriptPath: path.join(appResourcesPath, 'ComfyUI', 'main.py'), - } : { - pythonPath: path.join(userResourcesPath, 'python', 'bin', 'python'), - scriptPath: path.join(appResourcesPath, 'ComfyUI', 'main.py'), - }; - - console.log('Python Path:', pythonPath); - console.log('Script Path:', scriptPath); - - access(pythonPath).then(async () => { - pythonProcess = spawn(pythonPath, [scriptPath], { + const pythonRootPath = path.join(userResourcesPath, 'python'); + const pythonInterpreterPath = process.platform==='win32' ? path.join(pythonRootPath, 'python.exe') : path.join(pythonRootPath, 'bin', 'python'); + const pythonRecordPath = path.join(pythonRootPath, "INSTALLER"); + const scriptPath = path.join(appResourcesPath, 'ComfyUI', 'main.py'); + const comfyMainCmd = [scriptPath, ...(process.env.COMFYUI_CPU_ONLY === "true" ? ["--cpu"] : [])]; + + const spawnPython = async () => { + pythonProcess = spawn(pythonInterpreterPath, comfyMainCmd, { cwd: path.dirname(scriptPath) }); @@ -114,37 +94,41 @@ const launchPythonServer = async () => { pythonProcess.stdout.on('data', (data) => { console.log(`stdout: ${data}`); }); - }).catch(async () => { + } + + try { + // check for existence of both interpreter and INSTALLER record to ensure a correctly installed python env + await Promise.all([fs.access(pythonInterpreterPath), fs.access(pythonRecordPath)]); + spawnPython(); + } catch { console.log('Running one-time python installation on first startup...'); + // clean up any possible existing non-functional python env + try { + await fs.rm(pythonRootPath, {recursive: true}); + } catch {null;} + const pythonTarPath = path.join(appResourcesPath, 'python.tgz'); await tar.extract({file: pythonTarPath, cwd: userResourcesPath, strict: true}); - const pythonRootPath = path.join(userResourcesPath, 'python'); const wheelsPath = path.join(pythonRootPath, 'wheels'); - const rehydrateCmd = ['-m', 'uv', 'pip', 'install', '--no-index', '--no-deps', ...(await readdir(wheelsPath)).map(x => path.join(wheelsPath, x))]; - const rehydrateProc = spawn(pythonPath, rehydrateCmd, {cwd: wheelsPath}); + const rehydrateCmd = ['-m', 'uv', 'pip', 'install', '--no-index', '--no-deps', ...(await fs.readdir(wheelsPath)).map(x => path.join(wheelsPath, x))]; + const rehydrateProc = spawn(pythonInterpreterPath, rehydrateCmd, {cwd: wheelsPath}); rehydrateProc.on("exit", code => { + // write an INSTALLER record on sucessful completion of rehydration + fs.writeFile(pythonRecordPath, "ComfyUI"); + if (code===0) { // remove the now installed wheels - rm(wheelsPath, {recursive: true}); + fs.rm(wheelsPath, {recursive: true}); console.log(`Python successfully installed to ${pythonRootPath}`); - pythonProcess = spawn(pythonPath, [scriptPath], { - cwd: path.dirname(scriptPath) - }); - - pythonProcess.stderr.on('data', (data) => { - console.error(`stderr: ${data}`); - }); - pythonProcess.stdout.on('data', (data) => { - console.log(`stdout: ${data}`); - }); + spawnPython(); } else { console.log(`Rehydration of python bundle exited with code ${code}`); } }); - }); + } const checkInterval = 1000; // Check every 1 second @@ -167,8 +151,33 @@ const launchPythonServer = async () => { // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. app.on('ready', async () => { + const {userResourcesPath, appResourcesPath} = app.isPackaged ? { + // production: install python to per-user application data dir + userResourcesPath: app.getPath('appData'), + appResourcesPath: process.resourcesPath, + } : { + // development: install python to in-tree assets dir + userResourcesPath: path.join(app.getAppPath(), 'assets'), + appResourcesPath: path.join(app.getAppPath(), 'assets'), + } + + console.log(`userResourcesPath: ${userResourcesPath}`); + console.log(`appResourcesPath: ${appResourcesPath}`); + + try { + dotenv.config({path: path.join(appResourcesPath, ".env")}); + } catch { + // if no .env file, skip it + } + + try { + await fs.mkdir(userResourcesPath); + } catch { + // if user-specific resources dir already exists, that is fine + } + try { - await launchPythonServer(); + await launchPythonServer({userResourcesPath, appResourcesPath}); createWindow(); } catch (error) { diff --git a/yarn.lock b/yarn.lock index 46d7542d..73deb8eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2232,6 +2232,7 @@ __metadata: "@typescript-eslint/parser": "npm:^5.0.0" adm-zip: "npm:^0.5.15" axios: "npm:^1.7.3" + dotenv: "npm:^16.4.5" electron: "npm:31.3.1" electron-squirrel-startup: "npm:^1.0.1" eslint: "npm:^8.0.1" @@ -2580,6 +2581,13 @@ __metadata: languageName: node linkType: hard +"dotenv@npm:^16.4.5": + version: 16.4.5 + resolution: "dotenv@npm:16.4.5" + checksum: 10c0/48d92870076832af0418b13acd6e5a5a3e83bb00df690d9812e94b24aff62b88ade955ac99a05501305b8dc8f1b0ee7638b18493deb6fe93d680e5220936292f + languageName: node + linkType: hard + "eastasianwidth@npm:^0.2.0": version: 0.2.0 resolution: "eastasianwidth@npm:0.2.0"