Replace IOExceptions

.github/workflows/CloudTesting.yml

@@ -57,4 +57,4 @@ jobs:
           AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
           ICEBERG_AWS_REMOTE_AVAILABLE: 1
         run: |
-          make test_release
+          make test_release

.github/workflows/CodeQuality.yml

@@ -44,6 +44,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          submodules: 'true'
 
       - name: Install
         shell: bash
@@ -61,8 +62,3 @@ jobs:
           black --version
           make format-check-silent
 
-      - name: Generated Check
-        shell: bash
-        run: |
-          make generate-files
-          git diff --exit-code

.github/workflows/LocalTesting.yml

@@ -27,16 +27,48 @@ jobs:
           sudo apt-get install -y -qq software-properties-common
           sudo add-apt-repository ppa:git-core/ppa
           sudo apt-get update -y -qq
-          sudo apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
+          sudo apt-get install -y -qq \
+            ninja-build \
+            make gcc-multilib \
+            g++-multilib \
+            libssl-dev \
+            wget \
+            openjdk-8-jdk \
+            zip \
+            maven \
+            unixodbc-dev \
+            libc6-dev-i386 \
+            lib32readline6-dev \
+            libssl-dev \
+            libcurl4-gnutls-dev \
+            libexpat1-dev \
+            gettext \
+            unzip \
+            build-essential \
+            checkinstall \
+            libffi-dev \
+            curl \
+            libz-dev \
+            openssh-client
           sudo apt-get install -y -qq tar pkg-config
           sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
 
+      - name: Install CMake 3.x
+        run: |
+          sudo apt-get remove -y cmake cmake-data
+          sudo apt-get install --allow-downgrades -y -qq 'cmake=3.*' 'cmake-data=3.*'
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: 'true'
 
+      - name: Check installed versions
+        run: |
+          ninja --version
+          cmake --version
+
       - name: Setup vcpkg
         uses: lukka/[email protected]
         with:

.github/workflows/PolarisTesting.yml

@@ -0,0 +1,135 @@
+name: Local Polaris Testing
+on: [push, pull_request,repository_dispatch]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+defaults:
+  run:
+    shell: bash
+
+env:
+  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
+  CMAKE_POLICY_VERSION_MINIMUM: 3.5
+
+jobs:
+  rest:
+    name: Test against Polaris Catalog
+    runs-on: ubuntu-latest
+    env:
+      VCPKG_TARGET_TRIPLET: 'x64-linux'
+      GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+      PIP_BREAK_SYSTEM_PACKAGES: 1
+
+    steps:
+      - name: Install required ubuntu packages
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get install -y -qq software-properties-common
+          sudo add-apt-repository ppa:git-core/ppa
+          sudo apt-get update -y -qq
+          sudo apt-get install -y -qq \
+            ninja-build \
+            make gcc-multilib \
+            g++-multilib \
+            libssl-dev \
+            wget \
+            openjdk-8-jdk \
+            zip \
+            maven \
+            unixodbc-dev \
+            libc6-dev-i386 \
+            lib32readline6-dev \
+            libssl-dev \
+            libcurl4-gnutls-dev \
+            libexpat1-dev \
+            gettext \
+            unzip \
+            build-essential \
+            checkinstall \
+            libffi-dev \
+            curl \
+            libz-dev \
+            openssh-client
+          sudo apt-get install -y -qq tar pkg-config
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Install CMake 3.x
+        run: |
+          sudo apt-get remove -y cmake cmake-data
+          sudo apt-get install --allow-downgrades -y -qq 'cmake=3.*' 'cmake-data=3.*'
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        continue-on-error: true
+
+      - name: Build extension
+        env:
+          GEN: ninja
+          STATIC_LIBCPP: 1
+        run: |
+          make release
+
+      - name: Set up for Polaris
+        run: |
+          # install java
+          sudo apt install -y -qq openjdk-21-jre-headless
+          sudo apt install -y -qq openjdk-21-jdk-headless
+          # install python virtual environment (is this needed?)
+          sudo apt-get install -y -qq python3-venv
+
+      - name: Wait for polaris initialization
+        env:
+          JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64
+        run: |
+          make setup_polaris_ci
+          # let polaris initialize 
+          max_attempts=50
+          attempt=1
+          while ! (curl -sf http://localhost:8182/healthcheck || curl -sf http://localhost:8182/q/health); do
+            if [ $attempt -gt $max_attempts ]; then
+              echo "Polaris failed to initialize after $max_attempts attempts"
+              exit 1
+            fi
+            echo "Waiting for Polaris to initialize (attempt $attempt/$max_attempts)..."
+            sleep 5
+            attempt=$((attempt + 1))
+          done
+          echo "Polaris is healthy"
+
+      - name: Generate Polaris Data
+        run: |
+          python3 -m venv .
+          source ./bin/activate
+          python3 -m pip install poetry
+          python3 -m pip install pyspark==3.5.0
+          python3 -m pip install duckdb
+          python3 scripts/polaris/get_polaris_root_creds.py
+          # needed for setup_polaris_catalog.sh
+          export POLARIS_ROOT_ID=$(cat polaris_root_id.txt)
+          export POLARIS_ROOT_SECRET=$(cat polaris_root_password.txt)
+          cd polaris_catalog && ../scripts/polaris/setup_polaris_catalog.sh > user_credentials.json
+          cd ..
+          python3 scripts/polaris/get_polaris_client_creds.py
+          export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt)
+          export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt)
+          python3 scripts/data_generators/generate_data.py polaris
+
+      - name: Test with rest catalog
+        env:
+          POLARIS_SERVER_AVAILABLE: 1
+        run: |
+          export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt)
+          export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt)
+          make test_release

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.5...3.29)
 
 # Set extension name here
 set(TARGET_NAME iceberg)
@@ -16,7 +16,7 @@ set(EXTENSION_SOURCES
     src/iceberg_manifest.cpp
     src/manifest_reader.cpp
     src/catalog_api.cpp
-    src/catalog_utils.cpp 
+    src/catalog_utils.cpp
     src/common/utils.cpp
     src/common/url_utils.cpp
     src/common/schema.cpp
@@ -34,8 +34,7 @@ set(EXTENSION_SOURCES
     src/storage/irc_table_entry.cpp
     src/storage/irc_table_set.cpp
     src/storage/irc_transaction.cpp
-    src/storage/irc_transaction_manager.cpp
-)
+    src/storage/irc_transaction_manager.cpp)
 
 add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES})
 
@@ -46,14 +45,20 @@ find_package(CURL REQUIRED)
 find_package(AWSSDK REQUIRED COMPONENTS core sso sts)
 include_directories(${CURL_INCLUDE_DIRS})
 
+# Reset the TARGET_NAME, the AWS find_package build could bleed into our build - overriding `TARGET_NAME`
+set(TARGET_NAME iceberg)
+
 # AWS SDK FROM vcpkg
-target_include_directories(${EXTENSION_NAME} PUBLIC $<BUILD_INTERFACE:${AWSSDK_INCLUDE_DIRS}>)
+target_include_directories(${EXTENSION_NAME}
+                           PUBLIC $<BUILD_INTERFACE:${AWSSDK_INCLUDE_DIRS}>)
 target_link_libraries(${EXTENSION_NAME} PUBLIC ${AWSSDK_LINK_LIBRARIES})
-target_include_directories(${TARGET_NAME}_loadable_extension PRIVATE $<BUILD_INTERFACE:${AWSSDK_INCLUDE_DIRS}>)
-target_link_libraries(${TARGET_NAME}_loadable_extension ${AWSSDK_LINK_LIBRARIES})
+target_include_directories(${TARGET_NAME}_loadable_extension
+                           PRIVATE $<BUILD_INTERFACE:${AWSSDK_INCLUDE_DIRS}>)
+target_link_libraries(${TARGET_NAME}_loadable_extension
+                      ${AWSSDK_LINK_LIBRARIES})
 
 # Link dependencies into extension
-target_link_libraries(${EXTENSION_NAME} PUBLIC  ${CURL_LIBRARIES})
+target_link_libraries(${EXTENSION_NAME} PUBLIC ${CURL_LIBRARIES})
 target_link_libraries(${TARGET_NAME}_loadable_extension ${CURL_LIBRARIES})
 
 install(

Makefile

@@ -18,10 +18,18 @@ install_requirements:
 
 # Custom makefile targets
 data: data_clean start-rest-catalog
-	python3 scripts/data_generators/generate_data.py
+	python3 scripts/data_generators/generate_data.py spark-rest local
 
 data_large: data data_clean
-	python3 scripts/data_generators/generate_data.py
+	python3 scripts/data_generators/generate_data.py spark-rest local
+
+# setup polaris server. See PolarisTesting.yml to see instructions for a specific machine.
+setup_polaris_ci:
+	mkdir polaris_catalog
+	git clone https://github.com/apache/polaris.git polaris_catalog
+	cd polaris_catalog && ./gradlew clean :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true --no-build-cache
+	cd polaris_catalog && ./gradlew --stop
+	cd polaris_catalog && nohup ./gradlew run > polaris-server.log 2> polaris-error.log &
 
 data_clean:
 	rm -rf data/generated

scripts/data_generators/generate_data.py

@@ -1,17 +1,43 @@
 from generate_spark_local.generate_iceberg_spark_local import IcebergSparkLocal
 from generate_spark_rest.generate_iceberg_spark_rest import IcebergSparkRest
+from generate_polaris_rest.generate_iceberg_polaris_rest import IcebergPolarisRest
+import sys
 
-# Example usage:
-if __name__ == "__main__":
+def GenerateSparkRest():
     db2 = IcebergSparkRest()
     conn2 = db2.GetConnection()
     db2.GenerateTables(conn2)
     db2.CloseConnection(conn2)
     del db2
     del conn2
+
+def GenerateSparkLocal():
     db = IcebergSparkLocal()
     conn = db.GetConnection()
     db.GenerateTables(conn)
     db.CloseConnection(conn)
     del db
     del conn
+
+def GeneratePolarisData():
+    db = IcebergPolarisRest()
+    conn = db.GetConnection()
+    db.GenerateTables(conn)
+    db.CloseConnection(conn)
+    del db
+    del conn
+
+if __name__ == "__main__":
+    argv = sys.argv
+    for i in range(1, len(argv)):
+        if argv[i] == "polaris":
+            print("generating polaris data")
+            GeneratePolarisData()
+        elif argv[i] == "local":
+            print("generating local iceberg data")
+            GenerateSparkLocal()
+        elif argv[i] == "spark-rest":
+            print("generating local iceberg REST data")
+            GenerateSparkRest()
+        else:
+            print(f"{argv[i]} not recognized, skipping")

scripts/data_generators/generate_polaris_rest/__init__.py

@@ -0,0 +1 @@
+