test: Setup PyIceberg Tests CI

.github/workflows/test_ci.yml

@@ -0,0 +1,32 @@
+name: Test CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  python-tests:
+    name: Run Python Tests
+    runs-on: ubuntu-latest
+    steps:
+      # Setup & Install Dependencies
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+      - run: pip install -r tests/requirements.txt
+        name: Install Python Deps
+
+      # Run Python Tests
+      - run: pytest -v tests
+        name: Run Tests

tests/pyiceberg/conftest.py

@@ -1,22 +1,52 @@
 import subprocess
+import re
+import os
+from pathlib import Path
 
 import pytest
 from pyiceberg.catalog.rest import RestCatalog
-from pyiceberg.exceptions import ForbiddenError
+
+
+base_path = str(Path(os.path.realpath(__file__)).parent.parent.parent)
 
 
 @pytest.fixture(scope="session")
-def catalog():
+def build_binary():
+    subprocess.run(
+        ["go", "build", "."],
+        cwd=base_path,
+        check=True,
+        env={
+            "PATH": os.environ.get("PATH", ""),
+            "HOME": os.environ.get("HOME", ""),
+        },
+    )
+
+
+@pytest.fixture(scope="function")
+def catalog(tmp_path, build_binary):
     process = subprocess.Popen(
         ["./denali", "start"],
+        cwd=base_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
         env={
-            "DENALI_API_PORT": "5151",
-            "DENALI_WAREHOUSE_PATH": "/tmp/iceberg",
+            "DENALI_API_PORT": "0",
+            "DENALI_WAREHOUSE_PATH": str(tmp_path),
             "DENALI_DATABASE_URL": ":memory:",
-            "DENALI_DATABASE_TYPE": "sqlite3",
-        }
+            "DENALI_DATABASE_DIALECT": "sqlite3",
+        },
     )
 
-    breakpoint()
-    yield RestCatalog("rest_catalog", uri="http://localhost:5151")
+    last_line: list[str] = []
+    while len(last_line) == 0 or "Started the Denali Catalog Server at" not in last_line[-1]:
+        if process.stdout is None:
+            continue
+        process.stdout.readlines
+        last_line.append(process.stdout.readline().decode("utf-8"))
+        if len(last_line) > 15:
+            raise EnvironmentError("Failed to start Denali Catalog Server:\n\t" + "\n\t".join("`" + l + "`" for l in last_line))
+
+    url = re.search(r"Started the Denali Catalog Server at `(?P<url>[\[\]\:\.\d]+)`", last_line[-1]).group("url")
+    yield RestCatalog("rest_catalog", uri=f"http://{url}")
     process.kill()

tests/pyiceberg/test_ns.py

@@ -6,29 +6,33 @@ def test_create_drop_namespace(catalog):
     assert catalog.list_namespaces() == [("default",)]
     catalog.create_namespace("test")
     assert catalog.list_namespaces() == [("default",), ("test",)]
-    assert catalog.load_namespace_properties("test") == {}
+    assert "created_at" in catalog.load_namespace_properties("test")
     catalog.drop_namespace("test")
     assert catalog.list_namespaces() == [("default",)]
 
 
 def test_create_drop_namespace_with_properties(catalog):
     assert catalog.list_namespaces() == [("default",)]
-    props = { "creator": "denali" }
-    catalog.create_namespace("test", props)
+    catalog.create_namespace("test", { "creator": "denali" })
+
     assert catalog.list_namespaces() == [("default",), ("test",)]
-    assert catalog.load_namespace_properties("test") == props
+    props = catalog.load_namespace_properties("test")
+    assert props.get("creator") == "denali"
+    assert props.get("created_at").isnumeric()  # is numeric timestamp    
+
     catalog.drop_namespace("test")
     assert catalog.list_namespaces() == [("default",)]
 
 
 def test_create_sub_namespace(catalog):
     assert catalog.list_namespaces("default") == []
-    props = { "owner": "pyiceberg" }
-    catalog.create_namespace("default.def_inner", props)
+    catalog.create_namespace("default.def_inner", { "owner": "pyiceberg" })
 
     # Note: Bug in PyIceberg, does not follow the REST spec
     assert catalog.list_namespaces("default") == [("default", "default", "def_inner")]
-    assert catalog.load_namespace_properties("default.def_inner") == props
+    props = catalog.load_namespace_properties("default.def_inner")
+    assert props.get("owner") == "pyiceberg"
+    assert props.get("created_at").isnumeric()  # is numeric timestamp
 
     # Attempt to delete `default` should fail because of sub-namespace
     # TODO: Change error thrown from NoSuchNamespaceError to another

tests/pyiceberg/test_table.py

@@ -3,26 +3,29 @@
 
 
 def test_create_empty_table(catalog):
-    schema = pa.schema([("id", pa.int32(), False), ("name", pa.string(), True)])
+    in_schema = pa.schema([("id", pa.int32(), False), ("name", pa.string(), True)])
 
-    table = catalog.create_table(
+    created_table = catalog.create_table(
         "default.test_create_table",
-        schema=schema,
+        schema=in_schema,
         properties={"creator": "iceberg"}
     )
 
+    table = catalog.load_table("default.test_create_table")
+    assert created_table == table
+
     assert table.identifier == ("rest_catalog", "default", "test_create_table")
     schema = table.schema()
     assert schema.schema_id == 0
 
     id_col = schema.columns[0]
     assert id_col.name == "id"
-    assert isinstance(id_col.type, IntegerType)
-    assert id_col.required is False 
+    assert isinstance(id_col.field_type, IntegerType)
+    assert id_col.required is True 
 
     name_col = schema.columns[1]
     assert name_col.name == "name"
-    assert isinstance(name_col.type, StringType)
+    assert isinstance(name_col.field_type, StringType)
     assert name_col.required is False
 
     assert table.properties == {"creator": "iceberg"}
@@ -45,7 +48,6 @@ def test_append_table(catalog):
     table.append(df)
 
     read_df = table.scan().to_arrow()
-    breakpoint()
     assert read_df.equals(df)
 
     catalog.drop_table("default.test_append_table")

tests/requirements.txt

@@ -1,3 +1,4 @@
 pyiceberg==0.6.1
+pyarrow==16.1.0
 
 pytest