[misc] acceptance test GHA workflow (#897)

* full unit test workflow for python * temp addition of pull-request event to register workflow * add build tools install * add missing flag * add git install * debugging * more debugging * bump setup deps for package build * even more debugging * more debugging * clean up debug code * reorganize * rename * reorg files * first attempt at an R test * remove pull_request * remove testthat pkg * iterating.... * add cmake * install local package * refinement * debugging * refinement for OOM logging * fix some typos * comments * improve python logging * disable fast-fail of jobs * test smaller buffer for R * use smaller buffers * PR review f/b * add tiledbsoma package spec * debugging * remove debugging code * new LTS census has no unique cells in previous test range * add comments on usage * link back to PR * cap memory use for R acceptance tests * set job timeout to 24 hours
chanzuckerberg · Dec 20, 2023 · 2f2bfd7 · 2f2bfd7
1 parent 7cca895
commit 2f2bfd7
Show file tree

Hide file tree

Showing 5 changed files with 146 additions and 15 deletions.
diff --git a/.github/workflows/full-unittests.yml b/.github/workflows/full-unittests.yml
@@ -0,0 +1,127 @@
+name: cellxgene_census package full unit tests
+
+# Run all unit tests, including those that are too expensive to run frequently.
+# This workflow requires a very large capacity runner, e.g., 1+TiB RAM, which is
+# currently available through self-hosted runners. These runners have no swap,
+# so an OOM will cause the workflow to fail with OOMKilled (exit code 137).
+#
+# By default, will install from `main` and run the latest acceptance tests in `main`.
+#
+#   gh workflow run full-unittests.yml
+#
+# You can run it against a branch with:
+#
+#   gh workflow run full-unittests.yml --ref _branch_name_
+#
+# The python (not R) job supports installing a specific `tiledbsoma` version, allowing
+# the test to run with any tiledbsoma version, including branches from the TileDB-SOMA
+# repo. For example, to test against the head of main, do:
+#
+#   gh workflow run full-unittests.yml \
+#      -f 'tiledbsoma_python_dependency=git+https://github.com/single-cell-data/TileDB-SOMA.git#egg=tiledbsoma&subdirectory=apis/python/'
+
+on:
+  schedule:
+    - cron: "0 1 * * 6" # every Saturday night, 1AM UTC
+
+  workflow_dispatch: # used for debugging or manual validation of a branch
+    inputs:
+      tiledbsoma_python_dependency:
+        # Accepts any package spec that pip understand, e.g.,
+        #   tiledbsoma==1.0
+        #   git+https://github.com/single-cell-data/TileDB-SOMA.git#egg=tiledbsoma&subdirectory=apis/python/
+        #   git+https://github.com/single-cell-data/[email protected]#egg=tiledbsoma&subdirectory=apis/python/
+        # or whatever...
+        description: "tiledbsoma package specified as pip requirement"
+        required: false
+        default: ""
+        type: string
+
+jobs:
+  py_unit_tests:
+    runs-on: single-cell-1tb-runner
+    timeout-minutes: 1440 # 24 hour timeout
+    strategy:
+      fail-fast: false # prevent this job from killing other jobs
+    steps:
+      - name: log system state
+        run: |
+          free
+          echo ---------
+          df -kh
+          echo ---------
+          lscpu
+
+      - name: install OS dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y build-essential git-all libxml2-dev libssl-dev libcurl4-openssl-dev cmake
+
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: install python dependencies (including experimental)
+        run: |
+          python -m pip install -U pip setuptools setuptools_scm wheel
+          pip install --use-pep517 accumulation-tree # Geneformer dependency needs --use-pep517 for Cython
+          pip install -r ./api/python/cellxgene_census/scripts/requirements-dev.txt
+          pip install './api/python/cellxgene_census/[experimental]'
+
+      - name: install tiledbsoma version override
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.tiledbsoma_python_dependency != ''
+        run: |
+          # Due to a bug in the tiledbsoma setup, this must be installed editable, ie., `-e`.
+          # Filed as single-cell-data/TileDB-SOMA#1991
+          # PR https://github.com/single-cell-data/TileDB-SOMA/pull/1937 will resolve the
+          # issue and allow removal of `-e`
+          pip install -e '${{ github.event.inputs.tiledbsoma_python_dependency }}'
+
+      - name: pytest (--expensive --experimental)
+        run: |
+          echo '--------- tiledbsoma.show_package_version():'
+          python -c 'import tiledbsoma; tiledbsoma.show_package_versions()'
+          echo '--------- PIP package versions:'
+          pip freeze
+
+          PYTHONPATH=. pytest -v --durations=0 -rP --experimental --expensive ./api/python/cellxgene_census/tests/
+
+  r_unit_tests:
+    runs-on: single-cell-1tb-runner
+    timeout-minutes: 1440 # 24 hour timeout
+    strategy:
+      fail-fast: false # prevent this job from killing other jobs
+    steps:
+      - name: log system state
+        run: |
+          free
+          echo ---------
+          df -kh
+          echo ---------
+          lscpu
+
+      - name: install OS dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y build-essential git-all libxml2-dev libssl-dev libcurl4-openssl-dev cmake
+
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          extra-repositories: https://tiledb-inc.r-universe.dev, https://cloud.r-project.org, https://chanzuckerberg.r-universe.dev
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          working-directory: ./api/r/cellxgene.census
+          extra-packages: any::rcmdcheck, any::remotes
+          cache: true
+
+      - name: testthat
+        run: |
+          Rscript -e 'remotes::install_local("./api/r/cellxgene.census")'
+          Rscript -e 'library("tiledbsoma"); tiledbsoma::show_package_versions()'
+          Rscript -e 'library("testthat"); library("cellxgene.census"); test_dir("./api/r/cellxgene.census/tests/")'
+          Rscript -e 'library("cellxgene.census"); library(testthat); test_file("./api/r/cellxgene.census/tests/testthat/acceptance-tests.R")'
diff --git a/api/python/cellxgene_census/pyproject.toml b/api/python/cellxgene_census/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+requires = ["setuptools>=64", "setuptools_scm[toml]>=8"]
 build-backend = "setuptools.build_meta"
 
 [project]

diff --git a/api/python/cellxgene_census/tests/experimental/pp/test_hvg.py b/api/python/cellxgene_census/tests/experimental/pp/test_hvg.py
@@ -209,7 +209,7 @@ def test_hvg_vs_scanpy(
             "Homo sapiens",
             "is_primary_data == True",
             "dataset_id",
-            slice(500_000, 1_000_000),
+            slice(1_000_000, 4_000_000),
             marks=pytest.mark.expensive,
         ),
     ],

diff --git a/api/r/cellxgene.census/tests/testthat/acceptance-tests.R b/api/r/cellxgene.census/tests/testthat/acceptance-tests.R
@@ -102,7 +102,7 @@ test_that("test_incremental_read_X_human", {
 })
 
 test_that("test_incremental_read_X_human-large-buffer-size", {
-  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
   on.exit(census$close(), add = TRUE)
 
   organism <- "homo_sapiens"
@@ -126,7 +126,7 @@ test_that("test_incremental_read_X_mouse", {
 })
 
 test_that("test_incremental_read_X_mouse-large-buffer-size", {
-  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
   on.exit(census$close(), add = TRUE)
 
   organism <- "mus_musculus"
@@ -322,7 +322,7 @@ test_that("test_seurat_common-tissue", {
 })
 
 test_that("test_seurat_common-tissue-large-buffer-size", {
-  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
   on.exit(census$close(), add = TRUE)
 
   test_args <- list(
@@ -343,21 +343,23 @@ test_that("test_seurat_common-cell-type", {
     census = census,
     organism = "Homo sapiens",
     measurement_name = "RNA",
-    obs_value_filter = "cell_type == 'neuron'"
+    obs_value_filter = "cell_type == 'neuron'",
+    obs_coords = 1:15000000
   )
 
   test_seurat(test_args)
 })
 
 test_that("test_seurat_common-cell-type-large-buffer-size", {
-  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
   on.exit(census$close(), add = TRUE)
 
   test_args <- list(
     census = census,
     organism = "Homo sapiens",
     measurement_name = "RNA",
-    obs_value_filter = "cell_type == 'neuron'"
+    obs_value_filter = "cell_type == 'neuron'",
+    obs_coords = 1:15000000
   )
 
   test_seurat(test_args)
@@ -366,7 +368,7 @@ test_that("test_seurat_common-cell-type-large-buffer-size", {
 test_that("test_seurat_whole-enchilada-large-buffer-size", {
   # SKIP: R is not capable to load into memory
   if (FALSE) {
-    census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+    census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
     on.exit(census$close(), add = TRUE)
 
     test_args <- list(
@@ -494,7 +496,7 @@ test_that("test_sce_common-tissue", {
 })
 
 test_that("test_sce_common-tissue-large-buffer-size", {
-  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
   on.exit(census$close(), add = TRUE)
 
   test_args <- list(
@@ -515,21 +517,23 @@ test_that("test_sce_common-cell-type", {
     census = census,
     organism = "Homo sapiens",
     measurement_name = "RNA",
-    obs_value_filter = "cell_type == 'neuron'"
+    obs_value_filter = "cell_type == 'neuron'",
+    obs_coords = 1:15000000
   )
 
   test_sce(test_args)
 })
 
 test_that("test_sce_common-cell-type-large-buffer-size", {
-  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+  census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
   on.exit(census$close(), add = TRUE)
 
   test_args <- list(
     census = census,
     organism = "Homo sapiens",
     measurement_name = "RNA",
-    obs_value_filter = "cell_type == 'neuron'"
+    obs_value_filter = "cell_type == 'neuron'",
+    obs_coords = 1:15000000
   )
 
   test_sce(test_args)
@@ -538,7 +542,7 @@ test_that("test_sce_common-cell-type-large-buffer-size", {
 test_that("test_sce_whole-enchilada-large-buffer-size", {
   # SKIP: R is not capable to load into memory
   if (FALSE) {
-    census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(4 * 1024**3))
+    census <- open_soma_latest_for_test(soma.init_buffer_bytes = paste(1 * 1024**3))
     on.exit(census$close(), add = TRUE)
 
     test_args <- list(

diff --git a/api/r/cellxgene.census/tests/testthat/helper-acceptance.R b/api/r/cellxgene.census/tests/testthat/helper-acceptance.R
@@ -35,7 +35,7 @@ test_seurat <- function(get_seurat_args) {
   expect_true(ncol(this_seurat) > 0)
 }
 
-# Tests that the object is SingleCellExpiremnt and is a non-empty
+# Tests that the object is SingleCellExperiment and is a non-empty
 test_sce <- function(get_sce_args) {
   this_sce <- do.call(get_single_cell_experiment, get_sce_args)
   expect_true(is(this_sce, "SingleCellExperiment"))