diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml new file mode 100644 index 0000000..a8113e0 --- /dev/null +++ b/.github/workflows/test_ci.yml @@ -0,0 +1,32 @@ +name: Test CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + python-tests: + name: Run Python Tests + runs-on: ubuntu-latest + steps: + # Setup & Install Dependencies + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + - uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: 'pip' + - run: pip install -r tests/requirements.txt + name: Install Python Deps + + # Run Python Tests + - run: pytest -v tests + name: Run Tests diff --git a/tests/pyiceberg/conftest.py b/tests/pyiceberg/conftest.py index a1d9e05..a6d7929 100644 --- a/tests/pyiceberg/conftest.py +++ b/tests/pyiceberg/conftest.py @@ -1,22 +1,52 @@ import subprocess +import re +import os +from pathlib import Path import pytest from pyiceberg.catalog.rest import RestCatalog -from pyiceberg.exceptions import ForbiddenError + + +base_path = str(Path(os.path.realpath(__file__)).parent.parent.parent) @pytest.fixture(scope="session") -def catalog(): +def build_binary(): + subprocess.run( + ["go", "build", "."], + cwd=base_path, + check=True, + env={ + "PATH": os.environ.get("PATH", ""), + "HOME": os.environ.get("HOME", ""), + }, + ) + + +@pytest.fixture(scope="function") +def catalog(tmp_path, build_binary): process = subprocess.Popen( ["./denali", "start"], + cwd=base_path, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env={ - "DENALI_API_PORT": "5151", - "DENALI_WAREHOUSE_PATH": "/tmp/iceberg", + "DENALI_API_PORT": "0", + "DENALI_WAREHOUSE_PATH": str(tmp_path), "DENALI_DATABASE_URL": ":memory:", - "DENALI_DATABASE_TYPE": "sqlite3", - } + "DENALI_DATABASE_DIALECT": "sqlite3", + }, ) - breakpoint() - yield RestCatalog("rest_catalog", uri="http://localhost:5151") + last_line: list[str] = [] + while len(last_line) == 0 or "Started the Denali Catalog Server at" not in last_line[-1]: + if process.stdout is None: + continue + process.stdout.readlines + last_line.append(process.stdout.readline().decode("utf-8")) + if len(last_line) > 15: + raise EnvironmentError("Failed to start Denali Catalog Server:\n\t" + "\n\t".join("`" + l + "`" for l in last_line)) + + url = re.search(r"Started the Denali Catalog Server at `(?P[\[\]\:\.\d]+)`", last_line[-1]).group("url") + yield RestCatalog("rest_catalog", uri=f"http://{url}") process.kill() diff --git a/tests/pyiceberg/test_ns.py b/tests/pyiceberg/test_ns.py index 5d44047..a9d8363 100644 --- a/tests/pyiceberg/test_ns.py +++ b/tests/pyiceberg/test_ns.py @@ -6,29 +6,33 @@ def test_create_drop_namespace(catalog): assert catalog.list_namespaces() == [("default",)] catalog.create_namespace("test") assert catalog.list_namespaces() == [("default",), ("test",)] - assert catalog.load_namespace_properties("test") == {} + assert "created_at" in catalog.load_namespace_properties("test") catalog.drop_namespace("test") assert catalog.list_namespaces() == [("default",)] def test_create_drop_namespace_with_properties(catalog): assert catalog.list_namespaces() == [("default",)] - props = { "creator": "denali" } - catalog.create_namespace("test", props) + catalog.create_namespace("test", { "creator": "denali" }) + assert catalog.list_namespaces() == [("default",), ("test",)] - assert catalog.load_namespace_properties("test") == props + props = catalog.load_namespace_properties("test") + assert props.get("creator") == "denali" + assert props.get("created_at").isnumeric() # is numeric timestamp + catalog.drop_namespace("test") assert catalog.list_namespaces() == [("default",)] def test_create_sub_namespace(catalog): assert catalog.list_namespaces("default") == [] - props = { "owner": "pyiceberg" } - catalog.create_namespace("default.def_inner", props) + catalog.create_namespace("default.def_inner", { "owner": "pyiceberg" }) # Note: Bug in PyIceberg, does not follow the REST spec assert catalog.list_namespaces("default") == [("default", "default", "def_inner")] - assert catalog.load_namespace_properties("default.def_inner") == props + props = catalog.load_namespace_properties("default.def_inner") + assert props.get("owner") == "pyiceberg" + assert props.get("created_at").isnumeric() # is numeric timestamp # Attempt to delete `default` should fail because of sub-namespace # TODO: Change error thrown from NoSuchNamespaceError to another diff --git a/tests/pyiceberg/test_table.py b/tests/pyiceberg/test_table.py index 37d37e2..f90dfa8 100644 --- a/tests/pyiceberg/test_table.py +++ b/tests/pyiceberg/test_table.py @@ -3,26 +3,29 @@ def test_create_empty_table(catalog): - schema = pa.schema([("id", pa.int32(), False), ("name", pa.string(), True)]) + in_schema = pa.schema([("id", pa.int32(), False), ("name", pa.string(), True)]) - table = catalog.create_table( + created_table = catalog.create_table( "default.test_create_table", - schema=schema, + schema=in_schema, properties={"creator": "iceberg"} ) + table = catalog.load_table("default.test_create_table") + assert created_table == table + assert table.identifier == ("rest_catalog", "default", "test_create_table") schema = table.schema() assert schema.schema_id == 0 id_col = schema.columns[0] assert id_col.name == "id" - assert isinstance(id_col.type, IntegerType) - assert id_col.required is False + assert isinstance(id_col.field_type, IntegerType) + assert id_col.required is True name_col = schema.columns[1] assert name_col.name == "name" - assert isinstance(name_col.type, StringType) + assert isinstance(name_col.field_type, StringType) assert name_col.required is False assert table.properties == {"creator": "iceberg"} @@ -45,7 +48,6 @@ def test_append_table(catalog): table.append(df) read_df = table.scan().to_arrow() - breakpoint() assert read_df.equals(df) catalog.drop_table("default.test_append_table") diff --git a/tests/requirements.txt b/tests/requirements.txt index 60d7d61..d65daf2 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ pyiceberg==0.6.1 +pyarrow==16.1.0 pytest