Skip to content

Commit 809ad17

Browse files
lacraig2lacraig2
committed
CI updates
1 parent cc0a43e commit 809ad17

File tree

3 files changed

+197
-75
lines changed

3 files changed

+197
-75
lines changed

‎.github/workflows/parallel_tests.yml‎

Lines changed: 71 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -18,59 +18,94 @@ on:
1818
# code into it, and then rebuilding
1919

2020
jobs:
21+
lint:
22+
runs-on: panda-arc
23+
steps:
24+
- uses: actions/checkout@v4
25+
if: github.event_name == 'pull_request'
26+
- name: Set up Python
27+
if: github.event_name == 'pull_request'
28+
uses: actions/setup-python@v2
29+
with:
30+
python-version: "3.10"
31+
- name: Install dependencies
32+
if: github.event_name == 'pull_request'
33+
run: pip install flake8 lintly markupsafe==2.0.1
34+
- name: Lint with flake8
35+
if: github.event_name == 'pull_request'
36+
run: |
37+
set -o pipefail
38+
(flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics | lintly) 2>lintly.err || {
39+
if grep -q 'diff exceeded the maximum number of lines' lintly.err; then
40+
echo "Bypassing lint failure due to large diff."
41+
exit 0
42+
else
43+
cat lintly.err
44+
exit 1
45+
fi
46+
}
47+
env:
48+
LINTLY_API_KEY: ${{ secrets.GITHUB_TOKEN }}
49+
2150

2251
test_installer: # test install_ubuntu.sh
2352
runs-on: panda-arc # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet)
2453
container:
25-
image: ubuntu:20.04
54+
image: ubuntu:22.04
2655
steps:
2756
- name: Update
2857
run: apt-get -qq update -y
29-
- name: Install ssl
30-
run: apt-get -qq install -y libssl-dev
31-
- name: Set up Python
32-
uses: actions/setup-python@v5
33-
with:
34-
python-version: 3.9
35-
- name: Install Python dev headers
36-
run: apt-get -qq install -y libpython3.9-dev
37-
- uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
38-
- name: Lint PyPANDA with flake8
39-
run: |
40-
pip install --upgrade pip
41-
pip install flake8
42-
flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics
43-
# python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
4458
- name: Run install_ubuntu.sh
4559
run: cd $GITHUB_WORKSPACE && ./panda/scripts/install_ubuntu.sh
4660

47-
4861
build_container:
4962
if: github.repository == 'panda-re/panda'
5063
runs-on: panda-arc
5164
steps:
52-
- name: Install git
53-
run: sudo apt-get -qq update -y && sudo apt-get -qq install git -y
54-
- uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
65+
- uses: actions/checkout@v4
5566
with:
5667
fetch-depth: 0
57-
- name: 'Login to Github Container Registry'
58-
uses: docker/login-action@v3
59-
with:
60-
registry: ghcr.io
61-
username: ${{ github.actor }}
62-
password: ${{ secrets.GITHUB_TOKEN }}
6368
- name: Set up Docker Buildx
6469
uses: docker/setup-buildx-action@v3
65-
- name: Build and push
66-
uses: docker/build-push-action@v5
6770
with:
71+
driver-opts: |
72+
image=moby/buildkit:master
73+
network=host
74+
buildkitd-config-inline: |
75+
[registry."${{ secrets.PANDA_ARC_REGISTRY }}"]
76+
insecure = true
77+
http = true
78+
79+
- name: Trust Harbor's self-signed certificate
80+
run: |
81+
echo "Fetching certificate from registry"
82+
openssl s_client -showcerts -connect ${{ secrets.PANDA_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
83+
sudo update-ca-certificates
84+
85+
- name: Log in to Rehosting Arc Registry
86+
uses: docker/login-action@v3
87+
with:
88+
registry: ${{ secrets.PANDA_ARC_REGISTRY }}
89+
username: ${{ secrets.PANDA_ARC_REGISTRY_USER }}
90+
password: ${{ secrets.PANDA_ARC_REGISTRY_PASSWORD }}
91+
92+
- name: Build panda:latest
93+
uses: docker/[email protected]
94+
with:
95+
push: true
96+
load: true
6897
context: ${{ github.workspace }}
69-
tags: ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}
7098
target: developer
71-
push: true
99+
tags: |
100+
${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:${{ github.sha }}
101+
cache-from: |
102+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:cache,mode=max
103+
cache-to: |
104+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:cache,mode=max
105+
build-args: |
106+
REGISTRY=${{ secrets.PANDA_ARC_REGISTRY }}/proxy
72107
- name: Minimal test of built container # Just test to see if one of our binaries is built
73-
run: docker run --rm "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")'
108+
run: docker run --rm "${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")'
74109

75110
tests:
76111
if: github.repository == 'panda-re/panda'
@@ -103,7 +138,7 @@ jobs:
103138
docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID}
104139
--mount type=bind,source=$(pwd)/wheezy_panda2.qcow2,target=/home/panda/regdir/qcows/wheezy_panda2.qcow2
105140
--mount type=bind,source=$(pwd)/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2,target=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2
106-
--rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c
141+
--rm -t "${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:${{ github.sha }}" bash -c
107142
"cd /tmp; git clone https://github.com/panda-re/panda_test;
108143
cd ./panda_test/tests/taint2;
109144
echo 'Running Record:';
@@ -122,11 +157,11 @@ jobs:
122157
docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID}
123158
--mount type=bind,source=$(pwd)/ubuntu_1604_x86.qcow,target=/root/.panda/ubuntu_1604_x86.qcow
124159
-e PANDA_TEST=yes --cap-add SYS_NICE
125-
--rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c
160+
--rm -t "${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:${{ github.sha }}" bash -c
126161
"cd /panda/panda/python/tests/ && make && pip3 install -r requirements.txt && chmod +x ./run_all_tests.sh && ./run_all_tests.sh";
127162
128163
docker run --name panda_sym_test_${{ matrix.target }}_${GITHUB_RUN_ID}
129-
--rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c
164+
--rm -t "${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:${{ github.sha }}" bash -c
130165
"pip3 install capstone keystone-engine z3-solver; python3 /panda/panda/python/examples/unicorn/taint_sym_x86_64.py;
131166
if [ $? -eq 0 ]; then echo -e 'TEST PASSED!' && exit 0; else echo 'TEST FAILED!' && exit 1; fi"
132167
@@ -135,26 +170,10 @@ jobs:
135170
run: >-
136171
docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID}
137172
-e PANDA_TEST=yes --cap-add SYS_NICE
138-
--rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c
173+
--rm -t "${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:${{ github.sha }}" bash -c
139174
"cd /panda/build && make check"
140175
141-
cleanup:
142-
# Cleanup after prior jobs finish - even if they fail
143-
needs: [tests]
144-
runs-on: panda-arc
145-
if: always()
146-
147-
steps:
148-
# Note we leave the last 72hrs because caching is nice (first few panda image layers won't change often)
149-
# docker system prune -> Remove all unused containers, networks, images (both dangling and unreferenced)
150-
# docker builder prune -> Remove build cache
151-
- name: Cleanup images
152-
run: |
153-
docker system prune -af --filter "until=72h"
154-
docker image prune --all -f --filter "until=72h"
155-
docker builder prune -af --filter "until=72h"
156-
157-
build_and_check_fork: # Forked repos can't use panda-arc test suite - just checkout and run make check
176+
build_and_check_fork: # Forked repos can't use panda-arc test suite - just checkout and run make check
158177
if: github.repository != 'panda-re/panda'
159178
runs-on: panda-arc
160179

‎.github/workflows/publish_docker.yml‎

Lines changed: 71 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,78 @@ jobs:
4444
with:
4545
fetch-depth: 0
4646

47+
- name: Set up Docker Buildx
48+
uses: docker/setup-buildx-action@v3
49+
with:
50+
driver-opts: |
51+
image=moby/buildkit:master
52+
network=host
53+
buildkitd-config-inline: |
54+
[registry."${{ secrets.PANDA_ARC_REGISTRY }}"]
55+
insecure = true
56+
http = true
57+
58+
- name: 'Login to Docker Registry'
59+
if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
60+
uses: docker/login-action@v3
61+
with:
62+
username: pandare
63+
password: ${{secrets.pandare_dockerhub}}
64+
65+
- name: Trust Harbor's self-signed certificate
66+
run: |
67+
echo "Fetching certificate from registry"
68+
openssl s_client -showcerts -connect ${{ secrets.PANDA_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
69+
sudo update-ca-certificates
70+
71+
- name: Log in to Rehosting Arc Registry
72+
uses: docker/login-action@v3
73+
with:
74+
registry: ${{ secrets.PANDA_ARC_REGISTRY }}
75+
username: ${{ secrets.PANDA_ARC_REGISTRY_USER }}
76+
password: ${{ secrets.PANDA_ARC_REGISTRY_PASSWORD }}
77+
78+
- name: Build panda:latest
79+
uses: docker/[email protected]
80+
with:
81+
push: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
82+
load: true
83+
context: ${{ github.workspace }}
84+
tags: |
85+
pandare/panda:${{ github.sha }}
86+
pandare/panda:${{ needs.create_release.outputs.v-version }}
87+
pandare/panda:latest
88+
panda
89+
cache-from: |
90+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:cache,mode=max
91+
cache-to: |
92+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda:cache,mode=max
93+
build-args: |
94+
REGISTRY=${{ secrets.PANDA_ARC_REGISTRY }}/proxy
95+
96+
- name: Build panda packager
97+
uses: docker/[email protected]
98+
with:
99+
push: true
100+
load: true
101+
target: packager
102+
context: ${{ github.workspace }}
103+
tags: |
104+
packager
105+
cache-from: |
106+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda-ng-plugins:cache,mode=max
107+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda-ng-plugins:packagecache,mode=max
108+
cache-to: |
109+
type=registry,ref=${{secrets.PANDA_ARC_REGISTRY}}/pandare/panda-ng-plugins:packagecache,mode=max
110+
build-args: |
111+
REGISTRY=${{ secrets.PANDA_ARC_REGISTRY }}/proxy
112+
47113
- name: Build package
48-
working-directory: panda/debian
49-
run: ./setup.sh Ubuntu ${{ matrix.ubuntu_version }}
114+
working-directory: debian
115+
run: |
116+
docker run --rm -v $(pwd):/out panda bash -c "cp /panda/panda/python/core/dist/*.whl /out"
117+
docker run --rm -v $(pwd):/out packager bash -c "cp /pandare.deb /out"
118+
mv pandare.deb pandare_${version}.deb
50119
51120
- name: Upload wheel and debian packages to release
52121
uses: softprops/action-gh-release@v2
@@ -65,14 +134,6 @@ jobs:
65134
name: pypanda
66135
path: panda/debian/pandare*.whl
67136
if-no-files-found: error
68-
69-
- name: 'Login to Docker Registry'
70-
if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
71-
uses: docker/login-action@v3
72-
with:
73-
username: pandare
74-
password: ${{secrets.pandare_dockerhub}}
75-
76137

77138
#- name: 'Login to GHCR Registry'
78139
# if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}

0 commit comments

Comments
 (0)