feat: initial commit

.env.example

@@ -0,0 +1,2 @@
+DELTA_BEARER_TOKEN="<openssl rand -base64 32>"
+STORAGE_ACCOUNT_KEY="get from Azure portal"

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+ignore = E203, E266, E501, W503
+max-line-length = 88
+max-complexity = 18

.gitignore

@@ -0,0 +1,163 @@
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+conf/

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+repos:
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v2.4.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [ commit-msg ]
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+        language_version: python3
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args: [ '--profile', 'black' ]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        args: [ '--config', 'api/.flake8' ]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-added-large-files
+        args: [ '--maxkb=500' ]
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+        args: [ '--fix=lf' ]
+      - id: check-merge-conflict
+      - id: check-yaml
+  - repo: https://github.com/zricethezav/gitleaks
+    rev: v8.18.0
+    hooks:
+      - id: gitleaks
+        args: [ '--verbose' ]

.python-version

@@ -0,0 +1 @@
+3.11

README.md

@@ -0,0 +1,17 @@
+# Delta Sharing POC
+
+## Prerequisites
+
+- [ ] Docker
+- [ ] Task
+
+## Development
+
+1. Copy contents of `.env.example` into a new file named `.env`.
+2. Fill in the required values.
+3. Run `task` to launch the containers.
+4. Use the `deltalake` Python package to work with Delta Lake tables in your
+   scripts/pipelines.
+5. Use an API testing tool like Postman
+   to [interact](https://github.com/delta-io/delta-sharing/blob/main/PROTOCOL.md) with
+   the Delta Sharing server.

Taskfile.yml

@@ -0,0 +1,53 @@
+version: '3'
+
+dotenv:
+  - .env
+
+vars:
+  NAME: giga-dataops-platform_delta-lake
+
+tasks:
+  default:
+    desc: Build and start Docker containers
+    cmds:
+      - task: config
+      - docker compose --project-name {{.NAME}} up --detach --build --remove-orphans {{.CLI_ARGS}}
+
+  setup:
+    desc: Run initial setup
+    cmds:
+      - pip install -U pre-commit
+      - pre-commit install
+      - poetry env use 3.11
+      - poetry install
+
+  config:
+    desc: Generate config files
+    cmds:
+      - sed 's!STORAGE_ACCOUNT_KEY!{{.STORAGE_ACCOUNT_KEY}}!g' conf-template/core-site.xml > conf/core-site.xml
+      - sed 's!DELTA_BEARER_TOKEN!{{.DELTA_BEARER_TOKEN}}!g' conf-template/delta-sharing-server-config.yml > conf/delta-sharing-server-config.yml
+
+  logs:
+    desc: Attach to Docker logs
+    cmds:
+      - docker compose --project-name {{.NAME}} logs --follow {{.CLI_ARGS}}
+
+  stop:
+    desc: Stop containers
+    cmds:
+      - docker compose --project-name {{.NAME}} stop {{.CLI_ARGS}}
+
+  restart:
+    desc: Restart containers
+    cmds:
+      - docker compose --project-name {{.NAME}} restart {{.CLI_ARGS}}
+
+  exec:
+    desc: Run a command in the specified container
+    cmds:
+      - docker compose --project-name {{.NAME}} exec {{.CLI_ARGS}}
+
+  clean:
+    desc: Remove containers
+    cmds:
+      - docker compose --project-name {{.NAME}} down --volumes --remove-orphans {{.CLI_ARGS}}

conf-template/core-site.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+    <property>
+        <name>fs.azure.account.auth.type.tmgigasandboxadlsg2.dfs.core.windows.net</name>
+        <value>SharedKey</value>
+    </property>
+    <property>
+        <name>fs.azure.account.key.tmgigasandboxadlsg2.dfs.core.windows.net</name>
+        <value>SAS_TOKEN</value>
+    </property>
+</configuration>

conf-template/delta-sharing-server-config.yml

@@ -0,0 +1,42 @@
+# The format version of this config file
+version: 1
+# Config shares/schemas/tables to share
+shares:
+  - name: "qos"
+    schemas:
+      - name: "proco-api"
+        tables:
+          - name: "uz"
+            # Azure Blob Storage. See https://github.com/delta-io/delta-sharing#azure-blob-storage for how to config the credentials
+            location: "wasbs://[email protected]/qos/proco-api/2023-10-23_Proco-API_QoS"
+            id: "458dc4db-666e-41e6-8c12-3f025b060a9b"
+
+# Set the host name that the server will use
+host: "localhost"
+# Set the port that the server will listen on. Note: using ports below 1024
+# may require a privileged user in some operating systems.
+port: 8890
+# Set the url prefix for the REST APIs
+endpoint: "/delta-sharing"
+# Set the timeout of S3 presigned url in seconds
+preSignedUrlTimeoutSeconds: 3600
+# How many tables to cache in the server
+deltaTableCacheSize: 10
+# Whether we can accept working with a stale version of the table. This is useful when sharing
+# static tables that will never be changed.
+stalenessAcceptable: false
+# Whether to evaluate user provided `predicateHints`
+evaluatePredicateHints: false
+# Whether to evaluate user provided `jsonPredicateHints`
+evaluateJsonPredicateHints: false
+# Whether to evaluate user provided `jsonPredicateHints` for V2 predicates.
+evaluateJsonPredicateHintsV2: false
+# The maximum page size permitted by queryTable/queryTableChanges API.
+queryTablePageSizeLimit: 10000
+# The TTL of the page token generated in queryTable/queryTableChanges API (in milliseconds).
+queryTablePageTokenTtlMs: 259200000
+# The TTL of the refresh token generated in queryTable API (in milliseconds).
+refreshTokenTtlMs: 3600000
+
+authorization:
+  bearerToken: "DELTA_BEARER_TOKEN"

docker-compose.yml

@@ -0,0 +1,24 @@
+x-common-config: &common-config
+  restart: unless-stopped
+  init: true
+
+services:
+  server:
+    <<: *common-config
+    image: deltaio/delta-docker:0.8.1_2.3.0
+    platform: linux/amd64
+    command: [ delta_quickstart ]
+    ports:
+      - "8888-8889:8888-8889"
+
+  sharing:
+    <<: *common-config
+    image: deltaio/delta-sharing-server:0.6.7
+    platform: linux/amd64
+    command:
+      - --config
+      - /opt/docker/conf/delta-sharing-server-config.yml
+    ports:
+      - "8890:8890"
+    volumes:
+      - ./conf:/opt/docker/conf