Build: Multi-platform Docker Builds and Integration Test (#428)

* build: publish docker image to ghcr

* build: test cache

* build: build standard zgrab image as runner

* build: Integration Test

use more reliable way to determine project root

* build: zgrab should not be built in host

* build: attach a tty

* build: fix "the input device is not a TTY"

* test: remove auto env setup

* build: setup python2

* chore: comments

* style: format python files with black

* fix(test): mongodb schema

* fix: pop3 banner checker

according to rfc1939, merely +OK is a valid response in the AUTHORIZATION state

* build: remove the naive build CI

* docs: fix path and instructions in integrated test documents

* build: docker image only on main branch push or tagging

* build: add linter workflow for Python and Go

* patch: template py file validity

* build: only run linter on active PRs or main branch

* patch: condense case

.dockerignore

@@ -1,2 +1,5 @@
 Dockerfile
-
+.github
+integration_tests
+zgrab2_schemas
+*.md

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: "/docker-runner"
+    schedule:
+      interval: weekly

.github/workflows/docker-publish.yml

@@ -0,0 +1,153 @@
+# See also:
+# https://docs.docker.com/build/ci/github-actions/multi-platform/
+# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+
+name: Publish Docker image
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    strategy:
+      fail-fast: true
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm/v6
+          - linux/arm/v7
+          - linux/arm64
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.build.outputs.digest }}
+          push-to-registry: true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish:
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=ref,event=tag
+            type=ref,event=branch
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest list and push
+        id: push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect image
+        id: inspect
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+          IMAGE_DIGEST=$(docker buildx imagetools inspect --format "{{json .Manifest}}" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} | jq -r '.digest')
+          echo "IMAGE_DIGEST=$IMAGE_DIGEST" >> $GITHUB_OUTPUT
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.inspect.outputs.IMAGE_DIGEST }}
+          push-to-registry: true

.github/workflows/integration-test.yml

@@ -0,0 +1,37 @@
+name: Integration Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+
+      - name: Check out source
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          set -e
+          sudo wget https://github.com/jmespath/jp/releases/download/0.2.1/jp-linux-amd64 -O /usr/local/bin/jp
+          sudo chmod +x /usr/local/bin/jp
+          # Install Python 2.7
+          sudo apt update
+          sudo apt install -y python2
+          curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+          sudo python2 get-pip.py
+          # Install Python dependencies
+          pip2 install --user zschema
+          pip2 install --user -r requirements.txt
+
+      - name: Run tests
+        run: |
+          make integration-test

.github/workflows/linter.yml

@@ -0,0 +1,28 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+
+jobs:
+  py-black:
+    name: Python Formatter
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable
+  go-fmt:
+    name: Go Formatter
+    runs-on: ubuntu-latest
+    if: false # Pending activation
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+      - name: Run go fmt
+        run: go fmt ./...

Makefile

@@ -9,7 +9,7 @@ TEST_MODULES ?=
 
 all: zgrab2
 
-.PHONY: all clean integration-test integration-test-clean docker-runner container-clean gofmt test
+.PHONY: all clean integration-test integration-test-clean docker-runner gofmt test
 
 # Test currently only runs on the modules folder because some of the 
 # third-party libraries in lib (e.g. http) are failing.
@@ -25,7 +25,7 @@ zgrab2: $(GO_FILES)
 	rm -f zgrab2
 	ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2
 
-docker-runner: zgrab2
+docker-runner: clean
 	make -C docker-runner
 
 integration-test: docker-runner
@@ -37,12 +37,6 @@ integration-test-clean:
 	./integration_tests/cleanup.sh
 	make -C docker-runner clean
 
-# This is the target for re-building from source in the container
-container-clean:
-	rm -f zgrab2
-	cd cmd/zgrab2 && go build -v -a . && cd ../..
-	ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2
-
 clean:
 	cd cmd/zgrab2 && go clean
 	rm -f zgrab2

README.md

@@ -146,7 +146,7 @@ func init() {
 
 ### Output schema
 
-To add a schema for the new module, add a module under schemas, and update [`schemas/__init__.py`](schemas/__init__.py) to ensure that it is loaded.
+To add a schema for the new module, add a module under schemas, and update [`zgrab2_schemas/zgrab2/__init__.py`](zgrab2_schemas/zgrab2/__init__.py) to ensure that it is loaded.
 
 See [zgrab2_schemas/README.md](zgrab2_schemas/README.md) for details.
 
@@ -158,11 +158,13 @@ The only hard requirement is that the `test.sh` script drops its output in `$ZGR
 
 #### How to Run Integration Tests
 
-To run integration tests, you must have [Docker](https://www.docker.com/) installed. Then, you can follow the following steps to run integration tests:
+To run integration tests, you must have [Docker](https://www.docker.com/) and **Python 2** on host installed. Then, you can follow the following steps to run integration tests:
 
 ```shell
 go get github.com/jmespath/jp && go build github.com/jmespath/jp
-pip install --user zschema
+# or, sudo wget https://github.com/jmespath/jp/releases/download/0.2.1/jp-linux-amd64 -O /usr/local/bin/jp && sudo chmod +x /usr/local/bin/jp
+pip2 install --user zschema
+pip2 install --user -r requirements.txt
 make integration-test
 ```
 
@@ -173,5 +175,7 @@ echo $?
 0
 ```
 
+Refer to our [Github Actions workflow](.github/workflows/integration-test.yml) for an example of how to prepare environment for integration tests.
+
 ## License
 ZGrab2.0 is licensed under Apache 2.0 and ISC. For more information, see the LICENSE file.

docker-runner/Makefile

@@ -4,26 +4,23 @@ else
   EXECUTABLE_EXTENSION :=
 endif
 
-all: docker-runner.id
+PROJ_ROOT := $(shell git rev-parse --show-toplevel)
+
+all: docker-runner.id service-base-image.id
 
 .PHONY: clean clean-all base-apt-update
 
 service-base-image.id: service-base.Dockerfile
 	docker build -t zgrab2_service_base:latest -f service-base.Dockerfile -q . > service-base-image.id || (rm -f service-base-image.id && exit 1)
 
-runner-base-image.id: runner-base.Dockerfile
-	docker build -t zgrab2_runner_base:latest -f runner-base.Dockerfile -q . > runner-base-image.id || (rm -f runner-base-image.id && exit 1)
-
 base-apt-update:
 	IMAGE_ID=zgrab2_service_base:latest ./base-apt-update.sh
-	IMAGE_ID=zgrab2_runner_base:latest ./base-apt-update.sh
 
-docker-runner.id: Dockerfile ../cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) runner-base-image.id service-base-image.id
-	docker build -t zgrab2_runner:latest -f Dockerfile -q .. > docker-runner.id || (rm -f docker-runner.id && exit 1)
+docker-runner.id: $(PROJ_ROOT)/Dockerfile
+	docker build -t zgrab2_runner:latest -f $(PROJ_ROOT)/Dockerfile -q $(PROJ_ROOT) > docker-runner.id || (rm -f docker-runner.id && exit 1)
 
 clean:
 	if [ -f docker-runner.id ]; then docker rmi -f $$(cat docker-runner.id) && rm -f docker-runner.id; fi
 
 clean-all: clean
 	if [ -f service-base-image.id ]; then docker rmi -f $$(cat service-base-image.id) && rm -f service-base-image.id; fi
-	if [ -f runner-base-image.id ]; then docker rmi -f $$(cat runner-base-image.id) && rm -f runner-base-image.id; fi

docker-runner/docker-run.sh

@@ -7,4 +7,4 @@
 : "${CONTAINER_NAME:?}"
 
 set -e
-docker run --rm --link $CONTAINER_NAME:target -e ZGRAB_TARGET=target zgrab2_runner $@
+echo 'target' | docker run --rm -i --link $CONTAINER_NAME:target zgrab2_runner $@