From 71523d4c6f1df0234797dd00240ae8419467899c Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 13:32:59 -0700 Subject: [PATCH 01/35] Current path. --- action.yml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/action.yml b/action.yml index f290796..7175f9d 100644 --- a/action.yml +++ b/action.yml @@ -87,10 +87,11 @@ runs: with: credentials_json: "${{ inputs.google_credentials }}" - - name: Cache conda + - name: '[Unix] Cache Conda' + if: ${{ inputs.os != 'windows' }} uses: actions/cache@v4 env: - # Increase this value to reset cache if etc/environment.yml has not changed + # Increase this value to reset cache if the environment file has not changed CACHE_NUMBER: 0 with: path: ~/miniconda3/envs/comfyui @@ -209,21 +210,24 @@ runs: ## ## ##################################################################################### + - name: '[Windows] Cache Conda' + if: ${{ inputs.os != 'windows' }} + uses: actions/cache@v4 + env: + # Increase this value to reset cache if the environment file has not changed + CACHE_NUMBER: 0 + with: + path: C:\Miniconda3\envs\comfyui + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('{0}', inputs.conda_env_file)) }} + - name: '[Win] Setup Conda' uses: conda-incubator/setup-miniconda@v3.0.3 if: ${{ inputs.os == 'windows' }} with: - auto-update-conda: true - miniconda-version: latest activate-environment: comfyui - python-version: ${{ inputs.python_version }} - continue-on-error: true - - - name: '[Win-Only] Install Pytorch' - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - conda install pytorch torchvision torchaudio pytorch-cuda=${{ inputs.cuda_version }} -c pytorch -c nvidia --yes + auto-activate-base: false + environment-file: ${{ inputs.conda_env_file }} + use-only-tar-bz2: true - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} @@ -232,12 +236,7 @@ runs: conda list shell: powershell - - name: '[Win] Install dependencies' - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - Get-Command pip - pip install -r requirements.txt + # Keep in mind the self runner must be setup with a model in C:\actions-runner\ - name: '[Win] Download models' From d4a482ab710d2a08eb38bd7474fde4e05bde5bff Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 13:49:03 -0700 Subject: [PATCH 02/35] blah --- action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 7175f9d..c1cf89e 100644 --- a/action.yml +++ b/action.yml @@ -236,7 +236,13 @@ runs: conda list shell: powershell - + - name: Generate hash from models-json + id: generate_hash_windows + run: | + $hash = (Get-FileHash -Algorithm SHA256 -InputStream ([System.IO.MemoryStream][System.Text.Encoding]::UTF8.GetBytes("${{ inputs.models-json }}"))).Hash + echo "::set-output name=hash::$hash" + shell: powershell + # Keep in mind the self runner must be setup with a model in C:\actions-runner\ - name: '[Win] Download models' From eccbf6d18e6941df3736dc776051d0e27e49882a Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:06:13 -0700 Subject: [PATCH 03/35] Update. --- action.yml | 158 ++++++++++++++--------------------------------------- 1 file changed, 41 insertions(+), 117 deletions(-) diff --git a/action.yml b/action.yml index c1cf89e..8e536bc 100644 --- a/action.yml +++ b/action.yml @@ -237,27 +237,35 @@ runs: shell: powershell - name: Generate hash from models-json + if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: | $hash = (Get-FileHash -Algorithm SHA256 -InputStream ([System.IO.MemoryStream][System.Text.Encoding]::UTF8.GetBytes("${{ inputs.models-json }}"))).Hash - echo "::set-output name=hash::$hash" + echo "hash=$hash" >> $GITHUB_OUTPUT shell: powershell - - # Keep in mind the self runner must be setup with a model in C:\actions-runner\ - - name: '[Win] Download models' + - name: '[Windows] Cache models' if: ${{ inputs.os == 'windows' }} - run: | - cd $Env:GITHUB_ACTION_PATH - ls "$Env:GITHUB_WORKSPACE" - cp "C:\actions-runner\v1-5-pruned-emaonly.ckpt" "$Env:GITHUB_WORKSPACE/models/checkpoints" - shell: powershell + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/models + key: ${{ runner.os }}-models-${{ steps.generate_hash_windows.outputs.hash }} - - name: '[Win] Run Python application quick test' + # Keep in mind the self runner must be setup with a model in C:\actions-runner\ + # - name: '[Win] Download models' + # if: ${{ inputs.os == 'windows' }} + # run: | + # cd $Env:GITHUB_ACTION_PATH + # ls "$Env:GITHUB_WORKSPACE" + # cp "C:\actions-runner\v1-5-pruned-emaonly.ckpt" "$Env:GITHUB_WORKSPACE/models/checkpoints" + # shell: powershell + + - name: '[Windows] Download models' if: ${{ inputs.os == 'windows' }} shell: powershell run: | - python $Env:GITHUB_WORKSPACE/main.py --quick-test-for-ci + Set-Location ${{ env.GITHUB_ACTION_PATH }} + python download-models.py raw "${{ inputs.models-json }}" "${{ env.GITHUB_WORKSPACE }}\models\" - name: '[Win] Run Python application' if: ${{ inputs.os == 'windows' }} @@ -276,76 +284,6 @@ runs: python poll_server_start.py shell: powershell - - name: '[Win] Queue Prompt' - id: win_queue_prompt - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - cd $Env:GITHUB_ACTION_PATH - $FULL_PATH="${Env:GITHUB_WORKSPACE}/${{ inputs.workflow_json_path }}" - Write-Output "Full path to the JSON file: $FULL_PATH" - $PROMPT_ID = python queue_prompt.py "$FULL_PATH" - Write-Output "prompt_id=$PROMPT_ID" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "Script output: " - Write-Output "$PROMPT_ID" - - - name: '[Win] Get start time' - id: win_start_time - if: ${{ inputs.os == 'windows' }} - run: | - $currentTime = [int][double]::Parse((Get-Date -UFormat "%s")) - Write-Output "start_time=$currentTime" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "Start time: " - Write-Output "$currentTime" - shell: powershell - - - name: '[Win] Check Prompt Status and Get Output Files' - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - cd $Env:GITHUB_ACTION_PATH - Write-Output "Prompt ID: ${{ steps.win_queue_prompt.outputs.prompt_id }}" - python check_prompt_status.py ${{ steps.win_queue_prompt.outputs.prompt_id }} http://localhost:8188/history ${{ inputs.timeout }} - - - name: '[Win] Get end time' - id: win_end_tim - if: ${{ inputs.os == 'windows' }} - run: | - $currentTime = [int][double]::Parse((Get-Date -UFormat "%s")) - Write-Output "end_time=$currentTime" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "End time: " - Write-Output "$currentTime" - shell: powershell - - - name: '[Win] Auth to GCS' - uses: "google-github-actions/auth@v2" - if: ${{ inputs.os == 'windows' }} - with: - credentials_json: "${{ inputs.google_credentials }}" - - - name: '[Win] Upload Output Files to GCS' - if: ${{ success() && inputs.os == 'windows'}} - id: win_upload-output-files - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: ${{ github.workspace }}/output - destination: ${{ inputs.gcs_bucket_name }}/output-files/${{ github.job }}-${{ inputs.os }}-${{ inputs.workflow_name }}-run${{ github.run_id }} - glob: "${{ inputs.output_prefix }}*" - - - name: '[Win] Upload log file to GCS' - if: ${{ ( success() || failure() ) && inputs.os == 'windows'}} - id: win_upload-log-files - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: ${{ github.workspace }}/application.log - destination: ${{ inputs.gcs_bucket_name }}/logs/${{ github.job }}-${{ inputs.os }}-${{ inputs.workflow_name }}-run${{ github.run_id }} - - - name: '[Win] Debug print out commit timestamp and commit message' - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - Write-Output "Event : ${{ github.event }}" - - name: '[Win] Get Commit Details' id: win_get_commit_details if: ${{ inputs.os == 'windows' }} @@ -354,40 +292,31 @@ runs: $timestamp = git show -s --format=%cI HEAD^ $message = git show -s --format=%s HEAD^ $commit_hash = git rev-parse HEAD^ - Write-Output "Commit time: $timestamp" - Write-Output "Commit title: $message" - Write-Output "Commit hash: $commit_hash" - Write-Output "commit_time=$timestamp" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "commit_title=$message" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "commit_hash=$commit_hash" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append + "commit_time=$timestamp" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append + "commit_title=$message" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append + "commit_hash=$commit_hash" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - - name: '[Win] Call API to upload artifact details' - if: ${{ inputs.os == 'windows' && success() }} + - name: '[Windows] Queue Prompts' + id: windows_queue_prompt + if: ${{ inputs.os == 'windows' }} shell: powershell run: | - $response = Invoke-WebRequest -Uri "${{ inputs.api_endpoint}}" ` - -Method POST ` - -Headers @{"Content-Type"="application/json"} ` - -Body (@{ - "repo" = "${{ github.repository }}" - "run_id" = "${{ github.run_id }}" - "os" = "${{ inputs.os }}" - "cuda_version" = "${{ inputs.cuda_version }}" - "output_files_gcs_paths" = "${{ steps.win_upload-output-files.outputs.uploaded }}" - "commit_hash" = "${{ steps.win_get_commit_details.outputs.commit_hash }}" - "commit_time" = "${{ steps.win_get_commit_details.outputs.commit_time }}" - "commit_message" = "${{ steps.win_get_commit_details.outputs.commit_title }}" - "branch_name" = "${{ github.ref_name }}" - "bucket_name" = "${{ inputs.gcs_bucket_name }}" - "workflow_name" = "${{ inputs.workflow_name }}" - "start_time" = ${{ steps.win_start_time.outputs.start_time }} - "end_time" = ${{ steps.win_end_time.outputs.end_time }} - } | ConvertTo-Json) - - if ($response.StatusCode -ne 200) { - Write-Error "API request failed with status code $($response.StatusCode)" - exit 1 - } + Set-Location ${{ env.GITHUB_ACTION_PATH }} + Write-Host "Running workflows: ${{ inputs.workflow_filenames }}" + python queue_prompt.py ` + --comfy-workflow-names "${{ inputs.workflow_filenames }}" ` + --github-action-workflow-name "${{ github.workflow }}" ` + --os "${{ inputs.os }}" ` + --run-id "${{ github.run_id }}" ` + --gsc-bucket-name "${{ inputs.gcs_bucket_name }}" ` + --workspace-path "${{ github.workspace }}" ` + --output-file-prefix "${{ inputs.output_prefix }}" ` + --repo "${{ github.repository }}" ` + --commit-hash "${{ steps.win_get_commit_details.outputs.commit_hash }}" ` + --commit-time "${{ steps.win_get_commit_details.outputs.commit_time }}" ` + --commit-message "${{ steps.win_get_commit_details.outputs.commit_title }}" ` + --branch-name "${{ github.ref_name }}" ` + --api-endpoint "${{ inputs.api_endpoint }}" - name: '[Win] Upload Output Files' uses: actions/upload-artifact@v4 @@ -402,8 +331,3 @@ runs: with: name: app-logs-${{ github.job }}-${{ inputs.os }}-${{inputs.workflow_name}}-${{ github.run_id }} path: ${{ github.workspace }}/application.log - - - name: '[Win] Cleanup output files only' - if: ${{ inputs.os == 'windows' && ( success() || failure() ) }} - shell: powershell - run: Remove-Item -Path "${{ github.workspace }}/output/*" -Recurse -Force From 9a22487e598dae94c8deb2f8808f9247bf0be262 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:13:28 -0700 Subject: [PATCH 04/35] latest. --- action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/action.yml b/action.yml index 8e536bc..ad72b2f 100644 --- a/action.yml +++ b/action.yml @@ -226,6 +226,7 @@ runs: with: activate-environment: comfyui auto-activate-base: false + miniconda-version: "latest" environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From fabd20e590a1aac240de662260fbea7e810fa1ee Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:17:59 -0700 Subject: [PATCH 05/35] Fix conda. --- action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index ad72b2f..8f70814 100644 --- a/action.yml +++ b/action.yml @@ -226,7 +226,8 @@ runs: with: activate-environment: comfyui auto-activate-base: false - miniconda-version: "latest" + auto-update-conda: true + miniconda-version: latest environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From 29359963137b3ccd3afdf726dcdc8af27b15207e Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:20:25 -0700 Subject: [PATCH 06/35] Add unix check. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 8f70814..ed135b8 100644 --- a/action.yml +++ b/action.yml @@ -172,8 +172,8 @@ runs: python3 queue_prompt.py --comfy-workflow-names ${{ inputs.workflow_filenames }} --github-action-workflow-name "${{ github.workflow }}" --os "${{ inputs.os }}" --run-id "${{ github.run_id }}" --gsc-bucket-name "${{ inputs.gcs_bucket_name }}" --workspace-path "${{ github.workspace }}" --output-file-prefix ${{ inputs.output_prefix }} --repo "${{ github.repository }}" --commit-hash "${{ steps.unix_get_commit_details.outputs.commit_hash }}" --commit-time "${{ steps.unix_get_commit_details.outputs.commit_time }}" --commit-message "${{ steps.unix_get_commit_details.outputs.commit_title }}" --branch-name "${{ github.ref_name }}" --api-endpoint "${{ inputs.api_endpoint }}" - name: '[Unix] Upload log file' + if: ${{ inputs.os != 'windows' }} uses: actions/upload-artifact@v4 - if: ${{ success() || failure() }} with: name: app-logs-${{ github.job }}-${{ inputs.os }}-${{inputs.workflow_name}}-${{ github.run_id }} path: ${{ github.workspace }}/application.log From 0f905c5078c5851b53addbfdcf47bababf31b644 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:23:34 -0700 Subject: [PATCH 07/35] Use latest conda. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index ed135b8..43007c2 100644 --- a/action.yml +++ b/action.yml @@ -221,7 +221,7 @@ runs: key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('{0}', inputs.conda_env_file)) }} - name: '[Win] Setup Conda' - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3 if: ${{ inputs.os == 'windows' }} with: activate-environment: comfyui From a78f9a10a4ebc53fd875d2156c375024919b30bf Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:31:06 -0700 Subject: [PATCH 08/35] update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 43007c2..8b0415f 100644 --- a/action.yml +++ b/action.yml @@ -227,7 +227,7 @@ runs: activate-environment: comfyui auto-activate-base: false auto-update-conda: true - miniconda-version: latest + miniconda-version: 24.5.0 environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From 3773b44f827301edece6bfc1a070a86c2c3c163e Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:33:37 -0700 Subject: [PATCH 09/35] Update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 8b0415f..030f64b 100644 --- a/action.yml +++ b/action.yml @@ -227,7 +227,7 @@ runs: activate-environment: comfyui auto-activate-base: false auto-update-conda: true - miniconda-version: 24.5.0 + miniconda-version: "Miniconda3-py312_24.4.0-0-Windows-x86_64.exe" environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From ead4aa849ba6067ee5e23bb015e068fdf179bfff Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:37:30 -0700 Subject: [PATCH 10/35] Update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 030f64b..5ab7134 100644 --- a/action.yml +++ b/action.yml @@ -227,7 +227,7 @@ runs: activate-environment: comfyui auto-activate-base: false auto-update-conda: true - miniconda-version: "Miniconda3-py312_24.4.0-0-Windows-x86_64.exe" + miniconda-version: "Miniconda3-py312_24.4.0-0" environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From 41b86e89fafb5fa5eb6d3d05c7bbf88e446a3e93 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 14:40:17 -0700 Subject: [PATCH 11/35] Update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 5ab7134..99d1bbb 100644 --- a/action.yml +++ b/action.yml @@ -227,7 +227,7 @@ runs: activate-environment: comfyui auto-activate-base: false auto-update-conda: true - miniconda-version: "Miniconda3-py312_24.4.0-0" + miniconda-version: "py312_24.4.0-0" environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From b57d93024fff59a873bb3fb3e5df96c032a37db4 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 15:16:03 -0700 Subject: [PATCH 12/35] Latest. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 99d1bbb..43007c2 100644 --- a/action.yml +++ b/action.yml @@ -227,7 +227,7 @@ runs: activate-environment: comfyui auto-activate-base: false auto-update-conda: true - miniconda-version: "py312_24.4.0-0" + miniconda-version: latest environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From 899a62a3c2bca7e501b0523f59bb3e0685873132 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 15:29:17 -0700 Subject: [PATCH 13/35] Update. --- action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 43007c2..978c53b 100644 --- a/action.yml +++ b/action.yml @@ -226,8 +226,7 @@ runs: with: activate-environment: comfyui auto-activate-base: false - auto-update-conda: true - miniconda-version: latest + auto-update-conda: false environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true From e3d404d07ecb8599b02f8273f8fedf19968950e9 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 15:34:46 -0700 Subject: [PATCH 14/35] Update. --- action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 978c53b..1aade8d 100644 --- a/action.yml +++ b/action.yml @@ -224,11 +224,13 @@ runs: uses: conda-incubator/setup-miniconda@v3 if: ${{ inputs.os == 'windows' }} with: + auto-update-conda: true + miniconda-version: latest activate-environment: comfyui auto-activate-base: false - auto-update-conda: false environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true + continue-on-error: true - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} From e0467bd01c54570113e2fcd410d55f4d61194703 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 15:41:02 -0700 Subject: [PATCH 15/35] Run --- action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 1aade8d..2bab5fd 100644 --- a/action.yml +++ b/action.yml @@ -243,7 +243,10 @@ runs: if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: | - $hash = (Get-FileHash -Algorithm SHA256 -InputStream ([System.IO.MemoryStream][System.Text.Encoding]::UTF8.GetBytes("${{ inputs.models-json }}"))).Hash + $jsonString = '{"v1-5-pruned-emaonly.ckpt": {"url": "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt", "directory": "checkpoints"}}' + $bytes = [System.Text.Encoding]::UTF8.GetBytes($jsonString) + $stream = [System.IO.MemoryStream]::new($bytes) + $hash = (Get-FileHash -Algorithm SHA256 -InputStream $stream).Hash echo "hash=$hash" >> $GITHUB_OUTPUT shell: powershell From 1c505152931f555ef8aa2e86b128e5f50d099251 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 15:42:02 -0700 Subject: [PATCH 16/35] d --- action.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 2bab5fd..1eeea4e 100644 --- a/action.yml +++ b/action.yml @@ -243,10 +243,20 @@ runs: if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: | - $jsonString = '{"v1-5-pruned-emaonly.ckpt": {"url": "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt", "directory": "checkpoints"}}' + # Escape the JSON string from the GitHub Actions input + $jsonString = "${{ inputs.models-json }}" + $jsonString = $jsonString -replace '"', '\"' + + # Convert the JSON string to bytes $bytes = [System.Text.Encoding]::UTF8.GetBytes($jsonString) + + # Create a memory stream from the bytes $stream = [System.IO.MemoryStream]::new($bytes) + + # Calculate the SHA256 hash of the data in the memory stream $hash = (Get-FileHash -Algorithm SHA256 -InputStream $stream).Hash + + # Output the hash to the GitHub Actions environment echo "hash=$hash" >> $GITHUB_OUTPUT shell: powershell From 268330f03c6464894255eeacc3a3d712a3ddd358 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:07:30 -0700 Subject: [PATCH 17/35] d --- action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/action.yml b/action.yml index 1eeea4e..93f9754 100644 --- a/action.yml +++ b/action.yml @@ -347,3 +347,8 @@ runs: with: name: app-logs-${{ github.job }}-${{ inputs.os }}-${{inputs.workflow_name}}-${{ github.run_id }} path: ${{ github.workspace }}/application.log + + - name: '[Win] Cleanup output files only' + if: ${{ inputs.os == 'windows' && ( success() || failure() ) }} + shell: powershell + run: Remove-Item -Path "${{ github.workspace }}/output/*" -Recurse -Force From 4995056df9e3ded3d341c1f9d5239013acc69190 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:20:30 -0700 Subject: [PATCH 18/35] Update. --- action.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/action.yml b/action.yml index 93f9754..4551cba 100644 --- a/action.yml +++ b/action.yml @@ -243,23 +243,14 @@ runs: if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: | - # Escape the JSON string from the GitHub Actions input - $jsonString = "${{ inputs.models-json }}" - $jsonString = $jsonString -replace '"', '\"' - - # Convert the JSON string to bytes + $jsonString = @'${{ inputs.models-json }}'@ $bytes = [System.Text.Encoding]::UTF8.GetBytes($jsonString) - - # Create a memory stream from the bytes $stream = [System.IO.MemoryStream]::new($bytes) - - # Calculate the SHA256 hash of the data in the memory stream $hash = (Get-FileHash -Algorithm SHA256 -InputStream $stream).Hash - - # Output the hash to the GitHub Actions environment - echo "hash=$hash" >> $GITHUB_OUTPUT + echo "hash=$hash" >> $env:GITHUB_OUTPUT shell: powershell + - name: '[Windows] Cache models' if: ${{ inputs.os == 'windows' }} uses: actions/cache@v4 From 1c7cb91eb60ffda9d29026027009365e65c8909e Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:30:23 -0700 Subject: [PATCH 19/35] Hash string. --- action.yml | 10 ++-------- hash_string.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 hash_string.py diff --git a/action.yml b/action.yml index 4551cba..aa84857 100644 --- a/action.yml +++ b/action.yml @@ -225,10 +225,9 @@ runs: if: ${{ inputs.os == 'windows' }} with: auto-update-conda: true - miniconda-version: latest + miniconda-version: "latest" activate-environment: comfyui auto-activate-base: false - environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true continue-on-error: true @@ -242,12 +241,7 @@ runs: - name: Generate hash from models-json if: ${{ inputs.os == 'windows' }} id: generate_hash_windows - run: | - $jsonString = @'${{ inputs.models-json }}'@ - $bytes = [System.Text.Encoding]::UTF8.GetBytes($jsonString) - $stream = [System.IO.MemoryStream]::new($bytes) - $hash = (Get-FileHash -Algorithm SHA256 -InputStream $stream).Hash - echo "hash=$hash" >> $env:GITHUB_OUTPUT + run: python hash_string.py '${{ inputs.models-json }}' shell: powershell diff --git a/hash_string.py b/hash_string.py new file mode 100644 index 0000000..0f6c0ea --- /dev/null +++ b/hash_string.py @@ -0,0 +1,31 @@ +import hashlib +import sys +import json + + +def main(json_string): + # Ensure the JSON string is valid + try: + json_data = json.loads(json_string) + except json.JSONDecodeError: + print("Invalid JSON string") + sys.exit(1) + + # Convert JSON string to bytes + json_bytes = json_string.encode("utf-8") + + # Calculate SHA256 hash + hash_object = hashlib.sha256(json_bytes) + hash_hex = hash_object.hexdigest() + + # Output the hash + print(f"hash={hash_hex}") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python generate_hash.py ''") + sys.exit(1) + + json_string = sys.argv[1] + main(json_string) From 6cc7c3e123b3d0674abcc7755cfc1e6b85cf4a49 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:38:39 -0700 Subject: [PATCH 20/35] Update. --- action.yml | 5 ++++- hash_string.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index aa84857..57ec56e 100644 --- a/action.yml +++ b/action.yml @@ -241,7 +241,10 @@ runs: - name: Generate hash from models-json if: ${{ inputs.os == 'windows' }} id: generate_hash_windows - run: python hash_string.py '${{ inputs.models-json }}' + run: | + Set-Location ${{ env.GITHUB_ACTION_PATH }} + $hash = python hash_string.py '${{ inputs.models-json }}' + echo "##[set-output name=hash;]$hash" shell: powershell diff --git a/hash_string.py b/hash_string.py index 0f6c0ea..36c4955 100644 --- a/hash_string.py +++ b/hash_string.py @@ -19,7 +19,7 @@ def main(json_string): hash_hex = hash_object.hexdigest() # Output the hash - print(f"hash={hash_hex}") + print(hash_hex) if __name__ == "__main__": From d2c11ecfe686dd32f55efc01cf04a84e493d7081 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:39:13 -0700 Subject: [PATCH 21/35] Update. --- action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 57ec56e..2993cc6 100644 --- a/action.yml +++ b/action.yml @@ -224,10 +224,10 @@ runs: uses: conda-incubator/setup-miniconda@v3 if: ${{ inputs.os == 'windows' }} with: - auto-update-conda: true - miniconda-version: "latest" + auto-update-conda: false activate-environment: comfyui auto-activate-base: false + environment-file: ${{ inputs.conda_env_file }} use-only-tar-bz2: true continue-on-error: true From dac12cc25013aa5389b930d093e0521fd8f6e5dc Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:43:59 -0700 Subject: [PATCH 22/35] update. --- action.yml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/action.yml b/action.yml index 2993cc6..7c6bbc0 100644 --- a/action.yml +++ b/action.yml @@ -210,26 +210,11 @@ runs: ## ## ##################################################################################### - - name: '[Windows] Cache Conda' - if: ${{ inputs.os != 'windows' }} - uses: actions/cache@v4 - env: - # Increase this value to reset cache if the environment file has not changed - CACHE_NUMBER: 0 - with: - path: C:\Miniconda3\envs\comfyui - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('{0}', inputs.conda_env_file)) }} - - - name: '[Win] Setup Conda' - uses: conda-incubator/setup-miniconda@v3 + - name: '[Win-Only] Install Pytorch' if: ${{ inputs.os == 'windows' }} - with: - auto-update-conda: false - activate-environment: comfyui - auto-activate-base: false - environment-file: ${{ inputs.conda_env_file }} - use-only-tar-bz2: true - continue-on-error: true + shell: powershell + run: | + conda env create -f ${{ inputs.conda_env_file }} - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} From 89302e6fc56b41e255b3a8fc0223de7863fa6097 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 16:48:53 -0700 Subject: [PATCH 23/35] d --- action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 7c6bbc0..02956f7 100644 --- a/action.yml +++ b/action.yml @@ -215,6 +215,7 @@ runs: shell: powershell run: | conda env create -f ${{ inputs.conda_env_file }} + conda activate comfyui - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} @@ -228,7 +229,7 @@ runs: id: generate_hash_windows run: | Set-Location ${{ env.GITHUB_ACTION_PATH }} - $hash = python hash_string.py '${{ inputs.models-json }}' + $hash = python3 hash_string.py '${{ inputs.models-json }}' echo "##[set-output name=hash;]$hash" shell: powershell From 31501f16c3bc301c10cd9d63b83debf1e8325bfa Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:07:54 -0700 Subject: [PATCH 24/35] Activate. --- action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 02956f7..7b7201a 100644 --- a/action.yml +++ b/action.yml @@ -215,7 +215,6 @@ runs: shell: powershell run: | conda env create -f ${{ inputs.conda_env_file }} - conda activate comfyui - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} @@ -228,6 +227,7 @@ runs: if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: | + conda activate comfyui Set-Location ${{ env.GITHUB_ACTION_PATH }} $hash = python3 hash_string.py '${{ inputs.models-json }}' echo "##[set-output name=hash;]$hash" @@ -255,6 +255,7 @@ runs: shell: powershell run: | Set-Location ${{ env.GITHUB_ACTION_PATH }} + conda activate comfyui python download-models.py raw "${{ inputs.models-json }}" "${{ env.GITHUB_WORKSPACE }}\models\" - name: '[Win] Run Python application' @@ -271,6 +272,7 @@ runs: if: ${{ inputs.os == 'windows' }} run: | cd $Env:GITHUB_ACTION_PATH + conda activate comfyui python poll_server_start.py shell: powershell @@ -292,6 +294,7 @@ runs: shell: powershell run: | Set-Location ${{ env.GITHUB_ACTION_PATH }} + conda activate comfyui Write-Host "Running workflows: ${{ inputs.workflow_filenames }}" python queue_prompt.py ` --comfy-workflow-names "${{ inputs.workflow_filenames }}" ` From 650bba49d04c3e70d30035926696732e4e72982b Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:12:42 -0700 Subject: [PATCH 25/35] conda update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 7b7201a..3f7272f 100644 --- a/action.yml +++ b/action.yml @@ -214,7 +214,7 @@ runs: if: ${{ inputs.os == 'windows' }} shell: powershell run: | - conda env create -f ${{ inputs.conda_env_file }} + conda env update -f ${{ inputs.conda_env_file }} - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} From b9a6c3bd9dab6a96fc3ac6e3bd903161eb32e31d Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:15:40 -0700 Subject: [PATCH 26/35] update. --- action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 3f7272f..a81bc6c 100644 --- a/action.yml +++ b/action.yml @@ -219,6 +219,7 @@ runs: - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} run: | + conda activate comfyui conda info conda list shell: powershell @@ -229,7 +230,7 @@ runs: run: | conda activate comfyui Set-Location ${{ env.GITHUB_ACTION_PATH }} - $hash = python3 hash_string.py '${{ inputs.models-json }}' + $hash = python hash_string.py '${{ inputs.models-json }}' echo "##[set-output name=hash;]$hash" shell: powershell From 49ff6a7d273a3b35b65beafc74f00b510cd6fe1a Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:23:38 -0700 Subject: [PATCH 27/35] update. --- action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index a81bc6c..3896a59 100644 --- a/action.yml +++ b/action.yml @@ -228,10 +228,12 @@ runs: if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: | + Set-Location $Env:GITHUB_ACTION_PATH conda activate comfyui - Set-Location ${{ env.GITHUB_ACTION_PATH }} - $hash = python hash_string.py '${{ inputs.models-json }}' - echo "##[set-output name=hash;]$hash" + ls + python hash_string.py '${{ inputs.models-json }}' > hash_output.txt + $hash = Get-Content hash_output.txt + echo "hash=$hash" >> $Env:GITHUB_ENV shell: powershell @@ -256,6 +258,7 @@ runs: shell: powershell run: | Set-Location ${{ env.GITHUB_ACTION_PATH }} + ls conda activate comfyui python download-models.py raw "${{ inputs.models-json }}" "${{ env.GITHUB_WORKSPACE }}\models\" From 2dd8bd91e83f4dfb47645035e469ea5a496c3e25 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:26:49 -0700 Subject: [PATCH 28/35] update. --- action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/action.yml b/action.yml index 3896a59..6dfd040 100644 --- a/action.yml +++ b/action.yml @@ -231,8 +231,10 @@ runs: Set-Location $Env:GITHUB_ACTION_PATH conda activate comfyui ls + Write-Host "Running hash_string.py..." python hash_string.py '${{ inputs.models-json }}' > hash_output.txt $hash = Get-Content hash_output.txt + Write-Host "Hash value: $hash" echo "hash=$hash" >> $Env:GITHUB_ENV shell: powershell From 450dda2997d85e2ad071085c69e27079be43eec9 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:28:36 -0700 Subject: [PATCH 29/35] update. --- hash_string.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hash_string.py b/hash_string.py index 36c4955..5629be4 100644 --- a/hash_string.py +++ b/hash_string.py @@ -4,13 +4,6 @@ def main(json_string): - # Ensure the JSON string is valid - try: - json_data = json.loads(json_string) - except json.JSONDecodeError: - print("Invalid JSON string") - sys.exit(1) - # Convert JSON string to bytes json_bytes = json_string.encode("utf-8") From 4d5c3ac849fb61563b079943189ef2c37619b906 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 17:30:26 -0700 Subject: [PATCH 30/35] update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 6dfd040..e2e6456 100644 --- a/action.yml +++ b/action.yml @@ -259,7 +259,7 @@ runs: if: ${{ inputs.os == 'windows' }} shell: powershell run: | - Set-Location ${{ env.GITHUB_ACTION_PATH }} + Set-Location $Env:GITHUB_ACTION_PATH ls conda activate comfyui python download-models.py raw "${{ inputs.models-json }}" "${{ env.GITHUB_WORKSPACE }}\models\" From 6d7694a17c8907f4f72061cfda419e60f0d33b64 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 18:28:08 -0700 Subject: [PATCH 31/35] Update model path. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index e2e6456..c4c7b14 100644 --- a/action.yml +++ b/action.yml @@ -262,7 +262,7 @@ runs: Set-Location $Env:GITHUB_ACTION_PATH ls conda activate comfyui - python download-models.py raw "${{ inputs.models-json }}" "${{ env.GITHUB_WORKSPACE }}\models\" + python download-models.py raw "${{ inputs.models-json }}" "$Env:GITHUB_WORKSPACE\models\" - name: '[Win] Run Python application' if: ${{ inputs.os == 'windows' }} From 954271c2322947fcbad5e25e4b4802d4143b5e47 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 18:34:37 -0700 Subject: [PATCH 32/35] d --- action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index c4c7b14..1eea421 100644 --- a/action.yml +++ b/action.yml @@ -262,7 +262,10 @@ runs: Set-Location $Env:GITHUB_ACTION_PATH ls conda activate comfyui - python download-models.py raw "${{ inputs.models-json }}" "$Env:GITHUB_WORKSPACE\models\" + $jsonInput = '${{ inputs.models-json }}' + $outputDir = "$Env:GITHUB_WORKSPACE\models\" + Write-Host "Running download-models.py with input: $jsonInput and output directory: $outputDir" + python download-models.py raw "$jsonInput" "$outputDir" - name: '[Win] Run Python application' if: ${{ inputs.os == 'windows' }} From 1f103928a6bd05685fcb1e1c38d138729577088d Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 18:37:21 -0700 Subject: [PATCH 33/35] dsad --- action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 1eea421..0974138 100644 --- a/action.yml +++ b/action.yml @@ -263,9 +263,10 @@ runs: ls conda activate comfyui $jsonInput = '${{ inputs.models-json }}' + $jsonInputEscaped = $jsonInput -replace '"', '\"' $outputDir = "$Env:GITHUB_WORKSPACE\models\" - Write-Host "Running download-models.py with input: $jsonInput and output directory: $outputDir" - python download-models.py raw "$jsonInput" "$outputDir" + Write-Host "Running download-models.py with input: $jsonInputEscaped and output directory: $outputDir" + python download-models.py raw "$jsonInputEscaped" "$outputDir" - name: '[Win] Run Python application' if: ${{ inputs.os == 'windows' }} From 9d7a78146dd8ceb7e4312a2331c012219aa451fc Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 20:44:01 -0700 Subject: [PATCH 34/35] Fix queue prompt. --- action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 0974138..bd25cec 100644 --- a/action.yml +++ b/action.yml @@ -126,6 +126,7 @@ runs: with: path: ${{ github.workspace }}/models key: ${{ runner.os }}-models-${{ steps.generate_hash.outputs.hash }} + save-always: true - name: '[Unix] Download models' if: ${{ inputs.os != 'windows' }} @@ -245,6 +246,7 @@ runs: with: path: ${{ github.workspace }}/models key: ${{ runner.os }}-models-${{ steps.generate_hash_windows.outputs.hash }} + save-always: true # Keep in mind the self runner must be setup with a model in C:\actions-runner\ # - name: '[Win] Download models' @@ -303,7 +305,7 @@ runs: if: ${{ inputs.os == 'windows' }} shell: powershell run: | - Set-Location ${{ env.GITHUB_ACTION_PATH }} + cd $Env:GITHUB_ACTION_PATH conda activate comfyui Write-Host "Running workflows: ${{ inputs.workflow_filenames }}" python queue_prompt.py ` From eb67668461ca49394bccc43957d21a98f16d29b8 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 21:47:05 -0700 Subject: [PATCH 35/35] update. --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index bd25cec..1c48492 100644 --- a/action.yml +++ b/action.yml @@ -225,7 +225,7 @@ runs: conda list shell: powershell - - name: Generate hash from models-json + - name: '[Win] Generate hash from models-json' if: ${{ inputs.os == 'windows' }} id: generate_hash_windows run: |