antlir oss: custom buck test runner (#147)

Summary:
Custom buck test runner for #145

This PR provides an external test runner which is currently able to list out
individual unit tests from Python and Rust binaries coming from buck tests,
execute them in parallel and report results in JUnit's XML format, which should
be CI-compatible (although integration with the [test
reporter](https://github.com/dorny/test-reporter) is yet to be tested).

It can also retry tests and will silently ignore those manually marked for
exclusion (for instance, those with a `"disabled"` label).

The next step after this is implementing the rest of
#145, namely a way to keep
track of state between test runs on the CI so as to automatically disable some
tests.

NOTE by vmagro: I am (temporarily) duplicating the `ci.yml` workflow to use
this external runner, until it's at feature parity with the internal test
runner that is already being used

Pull Request resolved: #147

Test Plan: Was able to run this with `--config test.external_runner` on my desktop and got nice looking output from each test case

Reviewed By: zeroxoneb

Differential Revision: D30372373

Pulled By: vmagro

fbshipit-source-id: 33a3aea346375a0a60b4cbc687e642f381f33d1f

.github/workflows/ci-external-runner.yml

@@ -0,0 +1,91 @@
+# temporary copy of ci.yml to use the external test runner being developed by @baioc
+name: Antlir Tests (External Runner)
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Checkout submodules
+        run: git submodule update --init
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y attr libcap-ng-dev systemd-container libzstd-dev openssh-client
+
+      # createrepo_c is not packaged for Ubuntu 20.04. Until it is or until we
+      # can use a hosted runner that is based on CentOS (or Fedora), compile
+      # createrepo_c so that unit tests are able to use it to make ephemeral
+      # rpm repositories.
+      - name: Build createrepo_c
+        run: |
+          sudo apt-get install libssl-dev liblzma-dev libxml2-dev libmagic-dev libcurl4-openssl-dev libbz2-dev librpm-dev
+          curl -L https://github.com/rpm-software-management/createrepo_c/archive/0.16.2.tar.gz | gunzip | tar -x
+          cd createrepo_c-0.16.2
+          mkdir build && cd build
+          cmake .. -DENABLE_DRPM=0 -DWITH_ZCHUNK=0 -DWITH_LIBMODULEMD=0 -DENABLE_PYTHON=0
+          make -j$(nproc)
+          mv src/createrepo_c $GITHUB_WORKSPACE/tools/
+        working-directory: /tmp
+
+      - name: Set up $PATH
+        run: echo $(pwd)/tools > $GITHUB_PATH
+
+      # antlir is primarily developed on CentOS, which has a 'nobody:nobody'
+      # user and group. GitHub actions is running on ubuntu which uses
+      # 'nobody:nogroup', so create a 'nobody' group to use
+      - name: Set up nobody group
+        run: sudo groupadd nobody; sudo usermod -aG nobody nobody
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "~3.9"
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2021-03-25
+          override: true
+
+      - name: Fetch buck
+        run: buck --version
+
+      # Make it very clear if a failure is due to the target graph being
+      # unparseable, or an actual build failure.
+      - name: Validate target graph
+        run: buck targets //...
+
+      - name: Fetch remote artifacts
+        run: buck fetch //...
+
+      - name: Build tests
+        run: buck build --keep-going $(buck query @tools/testinfra/ci_tests_query)
+
+      # This is not quite a test, but exercises a lot of antlir and is expected
+      # to pass, unlike the unit tests below, not all of which have been fixed
+      # to support the OSS build
+      - name: Build base image(s)
+        run: buck build //images/base/...
+
+      - name: Run cxx tests
+        # TODO(vmagro): cxx_tests are excluded from 'buck test' because the
+        # test runner assumes that test output ends up in a certain spot on the
+        # host, and we have so few of these it doesn't make sense to invest in
+        # fixing our wrappers for both internal and external use for right now.
+        run: buck query 'kind(cxx_test, //...)' | xargs -n1 buck run
+        continue-on-error: true
+
+      # Run all tests, excluding any that are disabled (mainly the hidden
+      # layer tests)
+      - name: Run tests
+        run: buck test -c 'test.external_runner=//tools/testinfra/runner:runner' $(buck query @tools/testinfra/ci_tests_query) -- --report tests-junit.xml
+
+      - uses: actions/upload-artifact@v2
+        if: success() || failure()
+        with:
+          name: test-results
+          path: tests-junit.xml

antlir/website/docs/oss-test-runner.md

@@ -0,0 +1,32 @@
+---
+id: oss-test-runner
+title: Test Runner
+---
+
+# Test Runner
+
+Antlir ships with a [buck external test
+runner](https://buck.build/files-and-dirs/buckconfig.html#test.external_runner)
+that will soon become the default runner when `buck test` is invoked.
+
+This offers a number of features over the internal runner, including
+configurable retries, automatically ignoring disabled tests and more
+configurable parallel execution of individual test cases.
+
+## Implementation
+
+The test runner is implemented with a rust binary located in
+`tools/testinfra/runner`. It is currently capable of running Python and Rust
+unit tests, and outputting a mostly-JUnit-compatible XML file reporting the
+results.
+
+## CI Integration
+
+The external runner is still under development, so it is not yet the default
+test runner for the repo, but instead is launched as a secondary GitHub Actions
+workflow until it has feature parity with the internal test runner (mainly
+pretty HTML reports on workflow runs)
+
+## Future Features
+- Automatic disabling of failing tests
+- Pretty HTML reports

antlir/website/sidebars.js

@@ -7,7 +7,7 @@
  * @format
  */
 
-const {fbInternalOnly} = require('internaldocs-fb-helpers');
+const {fbContent, fbInternalOnly} = require('internaldocs-fb-helpers');
 
 module.exports = {
   docs: [
@@ -103,12 +103,19 @@ module.exports = {
         },
       ],
     },
-    // Towards the end since it's linked from Getting Started, and only
-    // relevant to OSS users.
-    {
-      type: 'doc',
-      id: 'installing',
-    },
+    ...fbContent({
+      external: [
+        // Towards the end since it's also linked from Getting Started
+        {
+          type: 'doc',
+          id: 'installing',
+        },
+        {
+          type: 'doc',
+          id: 'oss-test-runner',
+        },
+      ],
+    }),
     ...fbInternalOnly([
       {
         type: 'doc',

tools/testinfra/runner/BUCK

@@ -0,0 +1,17 @@
+load("//antlir/bzl:oss_shim.bzl", "rust_binary", "third_party")
+
+rust_binary(
+    name = "runner",
+    srcs = glob(["src/**/*.rs"]),
+    deps = third_party.libraries(
+        [
+            "anyhow",
+            "itertools",
+            "rayon",
+            "serde",
+            "serde_json",
+            "structopt",
+        ],
+        platform = "rust",
+    ),
+)