green-coding-solutions · ArneTR · Jun 4, 2024 · Jun 4, 2024 · Jun 4, 2024 · Jun 4, 2024
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -31,22 +31,42 @@ jobs:
         with:
           task: start-measurement
 
-      - name: Wait before the next step
+      - name: Sleep step
         run: sleep 2
 
       - name: Test measurement 1
         uses: ./
         with:
           task: get-measurement
+          label: "Sleep 3s"
 
-      - name: Wait before the next step
-        run: sleep 2
+      - name: Filesystem
+        run: ls -alhR /usr
 
       - name: Test measurement 2
         uses: ./
         with:
           task: get-measurement
-          label: "Sleep #2"
+          label: "ls -alhR /usr"
+
+      - name: Sleep 3
+        run: sleep 3
+
+      - name: Test measurement 2
+        uses: ./
+        with:
+          task: get-measurement
+          label: "Sleep 3s"
+
+
+      - name: Dump ECO-CI CPU
+        run: |
+          cat /tmp/eco-ci/cpu-util-total.txt
+
+      - name: Dump ECO-CI Energy
+        run: |
+          cat /tmp/eco-ci//tmp/eco-ci/energy.txt
+
 
       - name: Eco CI Energy Estimation
         uses: ./

diff --git a/.gitlab-ci.yml.example b/.gitlab-ci.yml.example
@@ -12,6 +12,7 @@ test-job:
     # Generate one here for example: https://www.freecodeformat.com/validate-uuid-guid.php
     #- export ECO_CI_COMPANY_UUID="YOUR COMPANY UUID"
     #- export ECO_CI_PROJECT_UUID="YOUR PROJECT UUID"
+    #- export ECO_CI_MACHINE_UUID="YOUR MACHINE UUID"
 
     - !reference [.initialize_energy_estimator, script]
     - !reference [.start_measurement, script]

diff --git a/action.yml b/action.yml
@@ -75,71 +75,15 @@ runs:
       name: Setup
       shell: bash
       run: |
-        if command -v python3 &>/dev/null; then
-            echo "Python is already installed."
-        else
-            echo "Python is not installed. Installing..."
-            apt-get update
-            apt-get install -y python3.12 python3.12-venv || apt-get install -y python3.10 python3.10-venv
-            echo "Python has been installed."
-        fi
-
-        python_version=$(python3 --version 2>&1)
-        python_major_version=$(python3 -c 'import sys; print(sys.version_info[0])')
-        python_minor_version=$(python3 -c 'import sys; print(sys.version_info[1])')
-        python_cache_path="/tmp/eco-ci/venv/lib/python${python_major_version}.${python_minor_version}/site-packages"
-        echo "python_cache_path=$python_cache_path" >> $GITHUB_OUTPUT
-
-
         # call the initialize function of setup.sh
-        ${{github.action_path}}/scripts/setup.sh initialize -g ${{inputs.display-graph}}
-
-      # To identify the hash for our cache we cannot use the classic mechansim of
-      # hashFiles('/tmp/eco-ci/spec-power-model/requirements.txt')
-      # hashFiles is restricted to ONLY work in the GITHUB_WORKSPACE which is for the calling action
-      # therefore we need to construct the hash ourselfs beforehand and save it to an output variable
-    - if:  inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE'
-      name: Hash requirements file
-      id: hash-requirements
-      shell: bash
-      run: echo "myhash=$(md5sum /tmp/eco-ci/spec-power-model/requirements.txt | cut -d ' ' -f1)" >> $GITHUB_OUTPUT;
-
-    - if:  inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE'
-      name: Cache pip packages
-      id: cache-pip
-      uses: actions/cache@v4
-      env:
-        cache-name: cache-pip-packages
-      with:
-        # npm cache files are stored in `~/.npm` on Linux/macOS
-        path: ${{ steps.initialize.outputs.python_cache_path }}
-        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.hash-requirements.outputs.myhash }}
-        restore-keys: |
-          ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.hash-requirements.outputs.myhash }}
-
-    - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' && steps.cache-pip.outputs.cache-hit == 'true'
-      name: Inform about cache hit
-      continue-on-error: true
-      shell: bash
-      run: |
-        echo "Cache hit succeeded! 😀"
-
-    - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' && steps.cache-pip.outputs.cache-hit != 'true'
-      name: Inform about cache hit
-      continue-on-error: true
-      shell: bash
-      run: |
-        echo "Cache hit failed! ❌"
-
+        ${{github.action_path}}/scripts/setup.sh initialize
 
     - if:  inputs.task == 'start-measurement'
       name: Starting measurement
       shell: bash
       # if measurement is started first time the reporter might not have run already
       # we prefer this over manual startint / stopping as it is less error prone for users
       run: |
-        ${{github.action_path}}/scripts/setup.sh setup_python
-
         if ${{inputs.send-data}}; then
           curl_response=$(curl -s -H "Authorization: Bearer ${{github.token}}" ${{ github.api_url }}/repos/${{ github.repository }}/actions/workflows)
           workflow_id=$(echo $curl_response | jq '.workflows[] | select(.name == "${{ github.workflow }}") | .id')

diff --git a/eco-ci-gitlab.yml b/eco-ci-gitlab.yml
@@ -15,15 +15,9 @@ variables:
 .initialize_energy_estimator:
     script:
         - |
-            apt-get update
-            apt-get install git uuid-runtime python3.10 python3.10-venv gcc bc jq curl golang-go psmisc -y
             if [[ -d /tmp/eco-ci ]]; then
                 rm -rf /tmp/eco-ci
             fi
-            git clone --depth 1 --single-branch --branch main https://github.com/green-coding-solutions/eco-ci-energy-estimation /tmp/eco-ci/main
-            /tmp/eco-ci/main/scripts/setup.sh initialize -r false -g false
-            /tmp/eco-ci/main/scripts/setup.sh setup_python
-
 .start_measurement:
     script:
         - |

diff --git a/scripts/display_results.sh b/scripts/display_results.sh
@@ -18,7 +18,6 @@ function display_results {
     CPU_FREQ=${CPU_FREQ:-}
     CPU_CHIPS=${CPU_CHIPS:-}
     VHOST_RATIO=${VHOST_RATIO:-}
-    PREVIOUS_VENV=${PREVIOUS_VENV:-}
     MEASUREMENT_COUNT=${MEASUREMENT_COUNT:-}
     WORKFLOW_ID=${WORKFLOW_ID:-}
     API_BASE=${API_BASE:-}
@@ -29,30 +28,25 @@ function display_results {
 
     if [[ $MEASUREMENT_RAN != true ]]; then
         echo "Running a measurement to have at least one result to display."
-        source /tmp/eco-ci/venv/bin/activate
 
         if [[ "$MODEL_NAME" == "unknown" ]]; then
-            cat /tmp/eco-ci/cpu-util.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
         elif [[ -n "$VHOST_RATIO" ]]; then
-            cat /tmp/eco-ci/cpu-util.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \
             --tdp $TDP --cpu-threads $CPU_THREADS \
             --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \
             --release-year $RELEASE_YEAR --ram $RAM \
             --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \
-            --vhost-ratio $VHOST_RATIO --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
+            --vhost-ratio $VHOST_RATIO --silent < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
         else
-            cat /tmp/eco-ci/cpu-util.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \
             --tdp $TDP --cpu-threads $CPU_THREADS \
             --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \
             --release-year $RELEASE_YEAR --ram $RAM \
             --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \
-            --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
+            --silent < /tmp/eco-ci/cpu-util.txt  | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
         fi
 
-        # reactivate the old venv, if it was present
-        if [[ $PREVIOUS_VENV != '' ]]; then
-          source $PREVIOUS_VENV/bin/activate
-        fi
         max_measurement_number=1
     fi
 
@@ -116,21 +110,28 @@ function display_results {
             echo '📈 Energy graph:' | tee -a $output $output_pr
             echo '```bash' | tee -a $output $output_pr
             echo ' ' | tee -a $output $output_pr
-            cat /tmp/eco-ci/energy-total.txt | /home/runner/go/bin/asciigraph -h 10 -c "Watts over time" | tee -a $output $output_pr
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time" < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr
             echo ' ```' | tee -a $output $output_pr
         elif [[ $source == 'gitlab' ]]; then
             echo '📈 Energy graph:' >> $output
-            cat /tmp/eco-ci/energy-total.txt | /home/runner/go/bin/asciigraph -h 10 -c "Watts over time" >> $output
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time" < /tmp/eco-ci/energy-total.txt >> $output
         fi
     fi
     repo_enc=$( echo ${repo} | jq -Rr @uri)
     branch_enc=$( echo ${branch} | jq -Rr @uri)
 
     if [[ ${show_carbon} == 'true' ]]; then
-        source "$(dirname "$0")/vars.sh" get_co2 "$total_energy"
-        if [ -n "${CO2EQ-}" ]; then # We only check for co2 as if this is set the others should be set too
+        source "$(dirname "$0")/vars.sh" get_energy_co2 "$total_energy"
+        source "$(dirname "$0")/vars.sh" get_embodied_co2 "$time"
+
+
+        if [ -n "$CO2EQ_EMBODIED" ] && [ -n "$CO2EQ_ENERGY" ]; then # We only check for co2 as if this is set the others should be set too
+            CO2EQ=$(echo "$CO2EQ_EMBODIED + $CO2EQ_ENERGY" | bc -l)
+
             echo '🌳 CO2 Data:' | tee -a $output $output_pr
             echo "City: <b>$CITY</b>, Lat: <b>$LAT</b>, Lon: <b>$LON</b>" | tee -a $output $output_pr
+            echo "CO₂ from energy is: $CO2EQ_ENERGY" | tee -a $output $output_pr
+            echo "CO₂ from manufacturing (embodied carbon) is: $CO2EQ_EMBODIED" | tee -a $output $output_pr
             echo "<a href='https://www.electricitymaps.com/methodology#carbon-intensity-and-emission-factors' target=_blank rel=noopener>Carbon Intensity</a> for this location: <b>$CO2I gCO₂eq/kWh</b>" | tee -a $output $output_pr
             printf "<a href='https://sci-guide.greensoftware.foundation/'  target=_blank rel=noopener>SCI</a>: <b>%.6f gCO₂eq / pipeline run</b> emitted\n" $CO2EQ | tee -a $output $output_pr
         else

diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh
@@ -22,47 +22,39 @@ function make_measurement() {
     WORKFLOW_ID=${WORKFLOW_ID:-}
     API_BASE=${API_BASE:-}
 
-    # check wc -l of cpu-util is greater than 0
-    if [[ $(wc -l < /tmp/eco-ci/cpu-util.txt) -gt 0 ]]; then
-        # capture time
-        time=$(($(date +%s) - $(cat /tmp/eco-ci/timer.txt)))
 
-        # capture cpu util
-        cat /tmp/eco-ci/cpu-util.txt > /tmp/eco-ci/cpu-util-temp.txt
 
-        # if a previous venv is already active,
-        if type deactivate &>/dev/null
-        then
-           deactivate
-        fi
-        # then activate our venv
-        source /tmp/eco-ci/venv/bin/activate
+    # capture time
+    time=$(($(date +%s) - $(cat /tmp/eco-ci/timer.txt)))
+
+    # reset timer and cpu capturing (lap)
+    source "$(dirname "$0")/setup.sh" lap_measurement
+
+    # capture cpu util - can only be done after docker container has written
+    cat /tmp/eco-ci/cpu-util.txt > /tmp/eco-ci/cpu-util-temp.txt
+
+
+    # check wc -l of cpu-util is greater than 0
+    if [[ $(wc -l < /tmp/eco-ci/cpu-util.txt) -gt 0 ]]; then
 
         ## make a note that we cannot use --energy, skew the result as we do not have an input delay.
         # this works because demo-reporter is 1/second
         if [[ "$MODEL_NAME" == "unknown" ]]; then
-            cat /tmp/eco-ci/cpu-util-temp.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
         elif [[ -n "$VHOST_RATIO" ]]; then
-            cat /tmp/eco-ci/cpu-util-temp.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \
             --tdp $TDP --cpu-threads $CPU_THREADS \
             --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \
             --release-year $RELEASE_YEAR --ram $RAM \
             --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \
-            --vhost-ratio $VHOST_RATIO --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
+            --vhost-ratio $VHOST_RATIO --silent < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
         else
-            cat /tmp/eco-ci/cpu-util-temp.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \
+            docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \
             --tdp $TDP --cpu-threads $CPU_THREADS \
             --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \
             --release-year $RELEASE_YEAR --ram $RAM \
             --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \
-            --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
-        fi
-
-        # now reset to old venv
-        deactivate
-        # reactivate the old venv, if it was present
-        if [[ $PREVIOUS_VENV != '' ]]; then
-          source $PREVIOUS_VENV/bin/activate
+            --silent < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt
         fi
 
         if [[ $MEASUREMENT_COUNT == '' ]]; then
@@ -88,13 +80,12 @@ function make_measurement() {
         echo $total_energy >> /tmp/eco-ci/energy-values.txt
         source "$(dirname "$0")/vars.sh" add_var MEASUREMENT_RAN true
 
-        if [ -z "$cb_machine_uuid" ]; then
-             cb_machine_uuid=$(uuidgen)
-        fi
-
         if [[ $send_data == 'true' ]]; then
 
-            source "$(dirname "$0")/vars.sh" get_co2 "$total_energy"
+            source "$(dirname "$0")/vars.sh" get_energy_co2 "$total_energy"
+            source "$(dirname "$0")/vars.sh" get_embodied_co2 "$time"
+
+            CO2EQ=$(echo "$CO2EQ_EMBODIED +  $CO2EQ_ENERGY" | bc -l)
 
             add_endpoint=$API_BASE"/v1/ci/measurement/add"
             value_mJ=$(echo "$total_energy*1000" | bc -l | cut -d '.' -f 1)
@@ -127,8 +118,6 @@ function make_measurement() {
             }"
         fi
 
-
-
         # write data to output
         lap_data_file="/tmp/eco-ci/lap-data.json"
         repo_enc=$( echo ${repo} | jq -Rr @uri)
@@ -141,10 +130,9 @@ function make_measurement() {
         source "$(dirname "$0")/create-and-add-meta.sh" --file "${lap_data_file}" --repository "${repo_enc}" --branch "${branch_enc}" --workflow "$WORKFLOW_ID" --run_id "${run_id_enc}"
         source "$(dirname "$0")/add-data.sh" --file "${lap_data_file}" --label "$label" --cpu "${cpu_avg}" --energy "${total_energy}" --power "${power_avg}" --time "${time}"
 
-        # reset timer and cpu capturing
-        killall -9 -q /tmp/eco-ci/demo-reporter || true
-        /tmp/eco-ci/demo-reporter | tee -a /tmp/eco-ci/cpu-util-total.txt > /tmp/eco-ci/cpu-util.txt &
-        date +%s > /tmp/eco-ci/timer.txt
+        # Reset the timers again, so we do not capture the overhead per step
+        # we want to only caputure the overhead in the totals
+        source "$(dirname "$0")/setup.sh" lap_measurement
 
     else
         echo "Skipping measurement as no data was collected since last call"