Merge pull request #3837 from gizmoguy/ci-improvements

gizmoguy · web-flow · commit 9a1c90fc0ccd · 2021-05-21T09:51:28.000+12:00
Rework faucet release and test CI jobs
diff --git a/.github/workflows/release-debian.yml b/.github/workflows/release-debian.yml
@@ -12,6 +12,8 @@ jobs:
   debian-package:
     name: "Build debian packages"
     runs-on: ubuntu-latest
+    environment:
+      name: "release"
     container:
       image: "ubuntu:bionic"
     steps:
@@ -54,6 +56,8 @@ jobs:
       image: "ubuntu:focal"
       options: --privileged
     needs: debian-package
+    permissions:
+      contents: write
     steps:
       - name: Install dependencies
         run: |
@@ -68,54 +72,52 @@ jobs:
           cd images/vm/
           ./build-faucet-vm.sh
       - name: Upload release asset
-        id: upload-release-asset
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: svenstaro/upload-release-action@v2
         with:
-          upload_url: ${{ github.event.release.upload_url }}
-          asset_path: ./images/vm/faucet-amd64-${{ github.event.release.tag_name }}.qcow2
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref }}
+          overwrite: true
+          file: ./images/vm/faucet-amd64-${{ github.event.release.tag_name }}.qcow2
           asset_name: faucet-amd64-${{ github.event.release.tag_name }}.qcow2
-          asset_content_type: application/octet-stream
 
   rpi-image:
     name: "Build raspberry pi image"
     runs-on: ubuntu-latest
-    container:
-      image: "ubuntu:bionic"
-      options: --privileged
     needs: debian-package
+    permissions:
+      contents: write
     steps:
       - name: Install dependencies
         run: |
-          dpkg --add-architecture i386
-          apt-get update
-          apt-get -y upgrade
-          apt-get install -y coreutils quilt parted qemu-user-static:i386 debootstrap zerofree zip dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc xxd kmod binfmt-support
-          /usr/sbin/update-binfmts --enable
+          sudo apt-get update
+          sudo apt-get -y upgrade
+          sudo apt-get install -y binfmt-support
+      - name: Prepare build environment
+        run: |
+          sudo /usr/sbin/update-binfmts --enable
+          sudo modprobe nbd
       - name: Checkout repo
         uses: actions/checkout@v2
       - name: Install pi-gen
         run: |
           cd images/raspbian/
           git clone https://github.com/RPi-Distro/pi-gen.git
-          cd pi-gen
-          mv ../install-faucet.sh ./stage2/01-sys-tweaks/02-install-faucet.sh
       - name: Build image
         run: |
           cd images/raspbian/pi-gen/
           echo "IMG_NAME='faucet_${{ github.event.release.tag_name }}_raspbian'" > config
+          mkdir ./stage2/04-install-faucet/
+          mv ../install-faucet.sh ./stage2/04-install-faucet/02-run.sh
+          rm ./stage2/EXPORT_NOOBS
           touch ./stage3/SKIP ./stage4/SKIP ./stage5/SKIP
           touch ./stage4/SKIP_IMAGES ./stage5/SKIP_IMAGES
-          ./build.sh
+          ./build-docker.sh
           mv ./deploy/image_*_raspbian-lite.zip ./deploy/faucet_${{ github.event.release.tag_name }}_raspbian-lite.zip
       - name: Upload release asset
-        id: upload-release-asset
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: svenstaro/upload-release-action@v2
         with:
-          upload_url: ${{ github.event.release.upload_url }}
-          asset_path: ./images/raspbian/pi-gen/deploy/faucet_${{ github.event.release.tag_name }}_raspbian-lite.zip
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref }}
+          overwrite: true
+          file: ./images/raspbian/pi-gen/deploy/faucet_${{ github.event.release.tag_name }}_raspbian-lite.zip
           asset_name: faucet_${{ github.event.release.tag_name }}_raspbian-lite.zip
-          asset_content_type: application/zip
diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml
@@ -9,6 +9,8 @@ jobs:
   faucet-docker-image:
     name: "Build faucet docker image"
     runs-on: ubuntu-latest
+    environment:
+      name: "release"
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
@@ -37,6 +39,8 @@ jobs:
   gauge-docker-image:
     name: "Build gauge docker image"
     runs-on: ubuntu-latest
+    environment:
+      name: "release"
     needs: faucet-docker-image
     steps:
       - name: Checkout repo
@@ -66,6 +70,8 @@ jobs:
   rabbitmq-adapter-docker-image:
     name: "Build rabbitmq adapter docker image"
     runs-on: ubuntu-latest
+    environment:
+      name: "release"
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
@@ -95,6 +101,8 @@ jobs:
   faucetagent-adapter-docker-image:
     name: "Build faucetagent adapter docker image"
     runs-on: ubuntu-latest
+    environment:
+      name: "release"
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
@@ -9,6 +9,8 @@ jobs:
   python-package:
     name: "Build and publish python packages"
     runs-on: ubuntu-latest
+    environment:
+      name: "release"
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
@@ -22,4 +24,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@v1.4.2
         with:
           user: __token__
-          password: ${{ secrets.PYPI_PASSWORD }}
+          password: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml
@@ -58,7 +58,6 @@ jobs:
           modprobe openvswitch
           modprobe ebtables
           ln -s "$(pwd)" /faucet-src
-          cd /faucet-src/
       - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }}
         name: Run sanity tests
         run: |
@@ -127,7 +126,6 @@ jobs:
           modprobe openvswitch
           modprobe ebtables
           ln -s "$(pwd)" /faucet-src
-          cd /faucet-src/
       - if: ${{ env.FILES_CHANGED == 'all' || env.PY_FILES_CHANGED || env.RQ_FILES_CHANGED }}
         name: Shard tests
         run: |
diff --git a/.github/workflows/tests-unit.yml b/.github/workflows/tests-unit.yml
@@ -55,6 +55,10 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - if: matrix.python-version == 3.5
+        name: Upgrade setuptools for python-3.5
+        run: |
+          pip3 install --upgrade setuptools
       - name: Install dependencies
         run: |
           ./docker/pip_deps.sh
@@ -66,8 +70,7 @@ jobs:
       - if: ${{ matrix.python-version == env.CODECOV_PY_VER }}
         name: Upload codecov
         uses: codecov/codecov-action@v1
-        # TODO: pytype support for 3.9 coming soon: https://github.com/google/pytype/issues/749
-      - if: ${{ matrix.python-version != 3.5 && matrix.python-version != 3.9 && (env.FILES_CHANGED == 'all' || env.RQ_FILES_CHANGED || env.PY_FILES_CHANGED) }}
+      - if: ${{ matrix.python-version != 3.5 && (env.FILES_CHANGED == 'all' || env.RQ_FILES_CHANGED || env.PY_FILES_CHANGED) }}
         name: Pytype
         run: |
           ./docker/pip_deps.sh --extra-requirements="codecheck-requirements.txt"
diff --git a/Dockerfile.fuzz-config b/Dockerfile.fuzz-config
@@ -11,17 +11,13 @@ WORKDIR /faucet-src
 RUN \
   apt-get update && \
   apt-get install -y afl cython3 && \
-  $PIP3 -r pip-requirements.txt && \
-  $PIP3 setuptools && \
   $PIP3 -r requirements.txt && \
   $PIP3 -r fuzz-requirements.txt && \
   $PIP3 . && \
   git clone https://github.com/mininet/mininet && \
   cd mininet && \
   pip3 install -q .
 
-
-
 VOLUME ["/var/log/afl/"]
 
 CMD ["docker/fuzz_config.sh"]
diff --git a/Dockerfile.fuzz-packet b/Dockerfile.fuzz-packet
@@ -2,7 +2,7 @@
 
 FROM faucet/test-base:9.0.0
 
-ENV PIP3="pip3 -q --no-cache-dir install --upgrade"
+ENV PIP3="pip3 --no-cache-dir install --upgrade"
 ENV PATH="/venv/bin:$PATH"
 
 COPY ./ /faucet-src/
@@ -11,8 +11,6 @@ WORKDIR /faucet-src
 RUN \
   apt-get update && \
   apt-get install -y afl cython3 && \
-  $PIP3 -r pip-requirements.txt && \
-  $PIP3 setuptools && \
   $PIP3 -r requirements.txt && \
   $PIP3 -r fuzz-requirements.txt && \
   $PIP3 .
diff --git a/adapters/vendors/faucetagent/Dockerfile b/adapters/vendors/faucetagent/Dockerfile
@@ -23,14 +23,12 @@ ENV GOPATH /go
 ENV PATH /go/bin:$PATH
 
 COPY requirements.txt /src/requirements.txt
-COPY pip-requirements.txt /src/pip-requirements.txt
 
 RUN for module in gnmi_capabilities gnmi_get gnmi_set gnmi_subscribe gnmi_target; do \
     go get github.com/google/gnxi/${module} \
     ; go install github.com/google/gnxi/${module} \
     ; done \
     && cd /src \
-    && pip3 install --no-cache-dir -r pip-requirements.txt \
     && pip3 install --no-cache-dir -r requirements.txt \
     && git clone https://github.com/faucetsdn/faucetagent \
     && cd faucetagent \
diff --git a/adapters/vendors/faucetagent/pip-requirements.txt b/adapters/vendors/faucetagent/pip-requirements.txt
diff --git a/adapters/vendors/rabbitmq/Dockerfile b/adapters/vendors/rabbitmq/Dockerfile
@@ -8,7 +8,6 @@ ENV PYTHONUNBUFFERED=0
 WORKDIR /src
 
 COPY requirements.txt requirements.txt
-COPY pip-requirements.txt pip-requirements.txt
 COPY rabbit.py rabbit.py
 COPY test_rabbit.py test_rabbit.py
 
@@ -17,8 +16,6 @@ RUN apk add --update \
       python3-dev \
       gcc \
       musl-dev \
-    && pip3 install --no-cache-dir -r pip-requirements.txt \
-    && pip3 install --no-cache-dir --upgrade wheel setuptools  \
     && pip3 install --no-cache-dir -r requirements.txt \
 # run tests
     && apk add --update pytest \
diff --git a/adapters/vendors/rabbitmq/pip-requirements.txt b/adapters/vendors/rabbitmq/pip-requirements.txt
diff --git a/docker/install-faucet.sh b/docker/install-faucet.sh
@@ -11,8 +11,7 @@ FROOT="/faucet-src"
 dir=$(dirname "$0")
 
 ${APK} add -U git ${BUILDDEPS}
-"${dir}/retrycmd.sh" "${PIP3} -r ${FROOT}/pip-requirements.txt"
-"${dir}/retrycmd.sh" "${PIP3} setuptools ${TESTDEPS}"
+"${dir}/retrycmd.sh" "${PIP3} ${TESTDEPS}"
 "${dir}/retrycmd.sh" "${PIP3} -r ${FROOT}/requirements.txt"
 ${PIP3} ${FROOT}
 
diff --git a/docker/pip_deps.sh b/docker/pip_deps.sh
@@ -22,19 +22,13 @@ done
 
 pip3="pip3 install -q --upgrade ${pip_args}"
 
-# Install pip pre-dependencies.
-"${BASEDIR}/docker/retrycmd.sh" "${pip3} -r pip-requirements.txt"
-"${BASEDIR}/docker/retrycmd.sh" "${pip3} wheel cython setuptools"
+"${BASEDIR}/docker/retrycmd.sh" "${pip3} wheel"
 
 for req in ${reqs}; do
   "${BASEDIR}/docker/retrycmd.sh" "${pip3} -r ${BASEDIR}/${req}"
 done
 
 # Topo unit test needs mininet in user python environment
 if ! python -c 'import mininet.net' 2> /dev/null; then
-  TMPDIR=$(mktemp -d) && pushd "${TMPDIR}"
-  git clone https://github.com/mininet/mininet
-  cd mininet
-  pip3 install -q .
-  popd && rm -rf "${TMPDIR}"
+  pip3 install -q git+https://github.com/mininet/mininet
 fi
diff --git a/docs/installation.rst b/docs/installation.rst
@@ -256,13 +256,11 @@ You can install the latest pip package, or you can install directly from git via
 Installing faucet
 ~~~~~~~~~~~~~~~~~
 
-First, install some python dependencies:
+First, ensure python3 is installed:
 
 .. code:: console
 
-  apt-get install python3-dev python3-pip
-  pip3 install setuptools
-  pip3 install wheel
+  apt-get install python3 python3-pip
 
 Then install the latest stable release of faucet from pypi, via pip:
 
@@ -274,6 +272,7 @@ Or, install the latest development code from git, via pip:
 
 .. code:: console
 
+  apt-get install git
   pip3 install git+https://github.com/faucetsdn/faucet.git
 
 Starting faucet manually
diff --git a/images/raspbian/install-faucet.sh b/images/raspbian/install-faucet.sh
@@ -1,12 +1,13 @@
 #!/bin/bash -e
 echo "Installing Faucet"
 
-ID="$(grep -Po '(?<=^ID=).*' ${ROOTFS_DIR}/etc/os-release | awk '{print tolower($0)}')"
-CODE="$(grep -Po -m 1 '(?<=\()[^\)]+' ${ROOTFS_DIR}/etc/os-release | awk '{print tolower($0)}')"
+ID="$(grep -Po '(?<=^ID=).*' "${ROOTFS_DIR}/etc/os-release" | awk '{print tolower($0)}')"
+CODE="$(grep -Po -m 1 '(?<=\()[^\)]+' "${ROOTFS_DIR}/etc/os-release" | awk '{print tolower($0)}')"
 on_chroot << EOF
 apt-get install -y curl apt-transport-https lsb-release tcpdump
-echo "deb https://packagecloud.io/faucetsdn/faucet/${ID}/ ${CODE} main" | sudo tee /etc/apt/sources.list.d/faucet.list
-curl -L https://packagecloud.io/faucetsdn/faucet/gpgkey | sudo apt-key add -
+echo "deb https://packagecloud.io/faucetsdn/faucet/${ID}/ ${CODE} main" | tee /etc/apt/sources.list.d/faucet.list
+curl -L https://packagecloud.io/faucetsdn/faucet/gpgkey | apt-key add -
 apt-get update
 apt-get install -y --fix-missing faucet-all-in-one
+apt-get clean
 EOF
diff --git a/images/vm/elements/faucet-all-in-one/install.d/02-install-faucet-gauge-prometheus-grafana b/images/vm/elements/faucet-all-in-one/install.d/02-install-faucet-gauge-prometheus-grafana
@@ -1,12 +1,10 @@
 #!/bin/bash
 
-if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+if [ "${DIB_DEBUG_TRACE:-1}" -gt 0 ]; then
     set -x
 fi
 set -euo pipefail
 
-SCRIPTDIR=$(dirname $0)
-
 # Add a `faucet' user
 adduser --disabled-password --gecos "faucet user" faucet
 echo faucet:faucet | chpasswd
diff --git a/pip-requirements.txt b/pip-requirements.txt
diff --git a/tests/unit/gauge/test_gauge.py b/tests/unit/gauge/test_gauge.py
@@ -393,6 +393,8 @@ def test_ship_success(self):
         """Checks that the shipper successsfully connects
         to a HTTP server when the points are shipped"""
 
+        server = None
+
         try:
             server = start_server(PretendInflux)
             shipper = gauge_influx.InfluxShipper()
@@ -402,8 +404,9 @@ def test_ship_success(self):
         except (ConnectionError, ReadTimeout) as err:
             self.fail("Code threw an exception: {}".format(err))
         finally:
-            server.socket.close()
-            server.shutdown()
+            if server:
+                server.socket.close()
+                server.shutdown()
 
     def test_ship_connection_err(self):
         """Checks that even when there is a connection error,
