From 4098231ac9a55e3fdc2f6f790f02330493e3d116 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 21:47:57 -0700 Subject: [PATCH] Windows support to call multiple workflows in queue_prompt. (#9) * Current path. * blah * Update. * latest. * Fix conda. * Add unix check. * Use latest conda. * update. * Update. * Update. * Update. * Latest. * Update. * Update. * Run * d * d * Update. * Hash string. * Update. * Update. * update. * d * Activate. * conda update. * update. * update. * update. * update. * update. * Update model path. * d * dsad * Fix queue prompt. * update. --- action.yml | 197 +++++++++++++++++-------------------------------- hash_string.py | 24 ++++++ 2 files changed, 92 insertions(+), 129 deletions(-) create mode 100644 hash_string.py diff --git a/action.yml b/action.yml index f290796..1c48492 100644 --- a/action.yml +++ b/action.yml @@ -87,10 +87,11 @@ runs: with: credentials_json: "${{ inputs.google_credentials }}" - - name: Cache conda + - name: '[Unix] Cache Conda' + if: ${{ inputs.os != 'windows' }} uses: actions/cache@v4 env: - # Increase this value to reset cache if etc/environment.yml has not changed + # Increase this value to reset cache if the environment file has not changed CACHE_NUMBER: 0 with: path: ~/miniconda3/envs/comfyui @@ -125,6 +126,7 @@ runs: with: path: ${{ github.workspace }}/models key: ${{ runner.os }}-models-${{ steps.generate_hash.outputs.hash }} + save-always: true - name: '[Unix] Download models' if: ${{ inputs.os != 'windows' }} @@ -171,8 +173,8 @@ runs: python3 queue_prompt.py --comfy-workflow-names ${{ inputs.workflow_filenames }} --github-action-workflow-name "${{ github.workflow }}" --os "${{ inputs.os }}" --run-id "${{ github.run_id }}" --gsc-bucket-name "${{ inputs.gcs_bucket_name }}" --workspace-path "${{ github.workspace }}" --output-file-prefix ${{ inputs.output_prefix }} --repo "${{ github.repository }}" --commit-hash "${{ steps.unix_get_commit_details.outputs.commit_hash }}" --commit-time "${{ steps.unix_get_commit_details.outputs.commit_time }}" --commit-message "${{ steps.unix_get_commit_details.outputs.commit_title }}" --branch-name "${{ github.ref_name }}" --api-endpoint "${{ inputs.api_endpoint }}" - name: '[Unix] Upload log file' + if: ${{ inputs.os != 'windows' }} uses: actions/upload-artifact@v4 - if: ${{ success() || failure() }} with: name: app-logs-${{ github.job }}-${{ inputs.os }}-${{inputs.workflow_name}}-${{ github.run_id }} path: ${{ github.workspace }}/application.log @@ -209,50 +211,64 @@ runs: ## ## ##################################################################################### - - name: '[Win] Setup Conda' - uses: conda-incubator/setup-miniconda@v3.0.3 - if: ${{ inputs.os == 'windows' }} - with: - auto-update-conda: true - miniconda-version: latest - activate-environment: comfyui - python-version: ${{ inputs.python_version }} - continue-on-error: true - - name: '[Win-Only] Install Pytorch' if: ${{ inputs.os == 'windows' }} shell: powershell run: | - conda install pytorch torchvision torchaudio pytorch-cuda=${{ inputs.cuda_version }} -c pytorch -c nvidia --yes + conda env update -f ${{ inputs.conda_env_file }} - name: "[Win] Check conda environment" if: ${{ inputs.os == 'windows' }} run: | + conda activate comfyui conda info conda list shell: powershell - - name: '[Win] Install dependencies' + - name: '[Win] Generate hash from models-json' if: ${{ inputs.os == 'windows' }} - shell: powershell + id: generate_hash_windows run: | - Get-Command pip - pip install -r requirements.txt + Set-Location $Env:GITHUB_ACTION_PATH + conda activate comfyui + ls + Write-Host "Running hash_string.py..." + python hash_string.py '${{ inputs.models-json }}' > hash_output.txt + $hash = Get-Content hash_output.txt + Write-Host "Hash value: $hash" + echo "hash=$hash" >> $Env:GITHUB_ENV + shell: powershell - # Keep in mind the self runner must be setup with a model in C:\actions-runner\ - - name: '[Win] Download models' + + - name: '[Windows] Cache models' if: ${{ inputs.os == 'windows' }} - run: | - cd $Env:GITHUB_ACTION_PATH - ls "$Env:GITHUB_WORKSPACE" - cp "C:\actions-runner\v1-5-pruned-emaonly.ckpt" "$Env:GITHUB_WORKSPACE/models/checkpoints" - shell: powershell + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/models + key: ${{ runner.os }}-models-${{ steps.generate_hash_windows.outputs.hash }} + save-always: true - - name: '[Win] Run Python application quick test' + # Keep in mind the self runner must be setup with a model in C:\actions-runner\ + # - name: '[Win] Download models' + # if: ${{ inputs.os == 'windows' }} + # run: | + # cd $Env:GITHUB_ACTION_PATH + # ls "$Env:GITHUB_WORKSPACE" + # cp "C:\actions-runner\v1-5-pruned-emaonly.ckpt" "$Env:GITHUB_WORKSPACE/models/checkpoints" + # shell: powershell + + - name: '[Windows] Download models' if: ${{ inputs.os == 'windows' }} shell: powershell run: | - python $Env:GITHUB_WORKSPACE/main.py --quick-test-for-ci + Set-Location $Env:GITHUB_ACTION_PATH + ls + conda activate comfyui + $jsonInput = '${{ inputs.models-json }}' + $jsonInputEscaped = $jsonInput -replace '"', '\"' + $outputDir = "$Env:GITHUB_WORKSPACE\models\" + Write-Host "Running download-models.py with input: $jsonInputEscaped and output directory: $outputDir" + python download-models.py raw "$jsonInputEscaped" "$outputDir" - name: '[Win] Run Python application' if: ${{ inputs.os == 'windows' }} @@ -268,79 +284,10 @@ runs: if: ${{ inputs.os == 'windows' }} run: | cd $Env:GITHUB_ACTION_PATH + conda activate comfyui python poll_server_start.py shell: powershell - - name: '[Win] Queue Prompt' - id: win_queue_prompt - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - cd $Env:GITHUB_ACTION_PATH - $FULL_PATH="${Env:GITHUB_WORKSPACE}/${{ inputs.workflow_json_path }}" - Write-Output "Full path to the JSON file: $FULL_PATH" - $PROMPT_ID = python queue_prompt.py "$FULL_PATH" - Write-Output "prompt_id=$PROMPT_ID" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "Script output: " - Write-Output "$PROMPT_ID" - - - name: '[Win] Get start time' - id: win_start_time - if: ${{ inputs.os == 'windows' }} - run: | - $currentTime = [int][double]::Parse((Get-Date -UFormat "%s")) - Write-Output "start_time=$currentTime" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "Start time: " - Write-Output "$currentTime" - shell: powershell - - - name: '[Win] Check Prompt Status and Get Output Files' - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - cd $Env:GITHUB_ACTION_PATH - Write-Output "Prompt ID: ${{ steps.win_queue_prompt.outputs.prompt_id }}" - python check_prompt_status.py ${{ steps.win_queue_prompt.outputs.prompt_id }} http://localhost:8188/history ${{ inputs.timeout }} - - - name: '[Win] Get end time' - id: win_end_tim - if: ${{ inputs.os == 'windows' }} - run: | - $currentTime = [int][double]::Parse((Get-Date -UFormat "%s")) - Write-Output "end_time=$currentTime" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "End time: " - Write-Output "$currentTime" - shell: powershell - - - name: '[Win] Auth to GCS' - uses: "google-github-actions/auth@v2" - if: ${{ inputs.os == 'windows' }} - with: - credentials_json: "${{ inputs.google_credentials }}" - - - name: '[Win] Upload Output Files to GCS' - if: ${{ success() && inputs.os == 'windows'}} - id: win_upload-output-files - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: ${{ github.workspace }}/output - destination: ${{ inputs.gcs_bucket_name }}/output-files/${{ github.job }}-${{ inputs.os }}-${{ inputs.workflow_name }}-run${{ github.run_id }} - glob: "${{ inputs.output_prefix }}*" - - - name: '[Win] Upload log file to GCS' - if: ${{ ( success() || failure() ) && inputs.os == 'windows'}} - id: win_upload-log-files - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: ${{ github.workspace }}/application.log - destination: ${{ inputs.gcs_bucket_name }}/logs/${{ github.job }}-${{ inputs.os }}-${{ inputs.workflow_name }}-run${{ github.run_id }} - - - name: '[Win] Debug print out commit timestamp and commit message' - if: ${{ inputs.os == 'windows' }} - shell: powershell - run: | - Write-Output "Event : ${{ github.event }}" - - name: '[Win] Get Commit Details' id: win_get_commit_details if: ${{ inputs.os == 'windows' }} @@ -349,40 +296,32 @@ runs: $timestamp = git show -s --format=%cI HEAD^ $message = git show -s --format=%s HEAD^ $commit_hash = git rev-parse HEAD^ - Write-Output "Commit time: $timestamp" - Write-Output "Commit title: $message" - Write-Output "Commit hash: $commit_hash" - Write-Output "commit_time=$timestamp" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "commit_title=$message" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - Write-Output "commit_hash=$commit_hash" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append + "commit_time=$timestamp" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append + "commit_title=$message" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append + "commit_hash=$commit_hash" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append - - name: '[Win] Call API to upload artifact details' - if: ${{ inputs.os == 'windows' && success() }} + - name: '[Windows] Queue Prompts' + id: windows_queue_prompt + if: ${{ inputs.os == 'windows' }} shell: powershell run: | - $response = Invoke-WebRequest -Uri "${{ inputs.api_endpoint}}" ` - -Method POST ` - -Headers @{"Content-Type"="application/json"} ` - -Body (@{ - "repo" = "${{ github.repository }}" - "run_id" = "${{ github.run_id }}" - "os" = "${{ inputs.os }}" - "cuda_version" = "${{ inputs.cuda_version }}" - "output_files_gcs_paths" = "${{ steps.win_upload-output-files.outputs.uploaded }}" - "commit_hash" = "${{ steps.win_get_commit_details.outputs.commit_hash }}" - "commit_time" = "${{ steps.win_get_commit_details.outputs.commit_time }}" - "commit_message" = "${{ steps.win_get_commit_details.outputs.commit_title }}" - "branch_name" = "${{ github.ref_name }}" - "bucket_name" = "${{ inputs.gcs_bucket_name }}" - "workflow_name" = "${{ inputs.workflow_name }}" - "start_time" = ${{ steps.win_start_time.outputs.start_time }} - "end_time" = ${{ steps.win_end_time.outputs.end_time }} - } | ConvertTo-Json) - - if ($response.StatusCode -ne 200) { - Write-Error "API request failed with status code $($response.StatusCode)" - exit 1 - } + cd $Env:GITHUB_ACTION_PATH + conda activate comfyui + Write-Host "Running workflows: ${{ inputs.workflow_filenames }}" + python queue_prompt.py ` + --comfy-workflow-names "${{ inputs.workflow_filenames }}" ` + --github-action-workflow-name "${{ github.workflow }}" ` + --os "${{ inputs.os }}" ` + --run-id "${{ github.run_id }}" ` + --gsc-bucket-name "${{ inputs.gcs_bucket_name }}" ` + --workspace-path "${{ github.workspace }}" ` + --output-file-prefix "${{ inputs.output_prefix }}" ` + --repo "${{ github.repository }}" ` + --commit-hash "${{ steps.win_get_commit_details.outputs.commit_hash }}" ` + --commit-time "${{ steps.win_get_commit_details.outputs.commit_time }}" ` + --commit-message "${{ steps.win_get_commit_details.outputs.commit_title }}" ` + --branch-name "${{ github.ref_name }}" ` + --api-endpoint "${{ inputs.api_endpoint }}" - name: '[Win] Upload Output Files' uses: actions/upload-artifact@v4 @@ -397,7 +336,7 @@ runs: with: name: app-logs-${{ github.job }}-${{ inputs.os }}-${{inputs.workflow_name}}-${{ github.run_id }} path: ${{ github.workspace }}/application.log - + - name: '[Win] Cleanup output files only' if: ${{ inputs.os == 'windows' && ( success() || failure() ) }} shell: powershell diff --git a/hash_string.py b/hash_string.py new file mode 100644 index 0000000..5629be4 --- /dev/null +++ b/hash_string.py @@ -0,0 +1,24 @@ +import hashlib +import sys +import json + + +def main(json_string): + # Convert JSON string to bytes + json_bytes = json_string.encode("utf-8") + + # Calculate SHA256 hash + hash_object = hashlib.sha256(json_bytes) + hash_hex = hash_object.hexdigest() + + # Output the hash + print(hash_hex) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python generate_hash.py ''") + sys.exit(1) + + json_string = sys.argv[1] + main(json_string)