01_standalone_examples/aws-glue-athena/glue-exporter-athena-demo-notebook.ipynb

{
	"metadata": {
		"kernelspec": {
			"name": "glue_pyspark",
			"display_name": "Glue PySpark",
			"language": "python"
		},
		"language_info": {
			"name": "Python_Glue_Session",
			"mimetype": "text/x-python",
			"codemirror_mode": {
				"name": "python",
				"version": 3
			},
			"pygments_lexer": "python3",
			"file_extension": ".py"
		},
		"toc-showtags": false
	},
	"nbformat_minor": 5,
	"nbformat": 4,
	"cells": [
		{
			"cell_type": "markdown",
			"source": "# Integration of lakeFS with Glue Catalog and Athena\n\n[📚 Docs](https://docs.lakefs.io/integrations/glue_metastore.html)",
			"metadata": {},
			"id": "c0941866-5ab1-46c6-ae4e-3ae228311623"
		},
		{
			"cell_type": "markdown",
			"source": "## Use Case: Isolated Dev/Test Environments",
			"metadata": {
				"tags": []
			},
			"id": "29105bda-b8b6-4bff-aaa8-7eadbacc7923"
		},
		{
			"cell_type": "markdown",
			"source": "## Config",
			"metadata": {
				"tags": []
			},
			"id": "531dde95-2a5b-4625-becf-55aaf9c496ec"
		},
		{
			"cell_type": "markdown",
			"source": "### Glue session configuration",
			"metadata": {},
			"id": "e64203a5-f381-4ded-b77c-12d1255d9625"
		},
		{
			"cell_type": "code",
			"source": "%stop_session\n%session_id_prefix 'lakefs-glue-exporter-demo'\n%idle_timeout 120\n%glue_version 4.0\n%worker_type G.1X\n%number_of_workers 2\n\n%additional_python_modules 'lakefs'",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "6721d3eb-5fa2-4310-9e11-fdc9056912d2"
		},
		{
			"cell_type": "markdown",
			"source": "### lakeFS endpoint and credentials",
			"metadata": {},
			"id": "61c5aea5-5195-4caa-a535-95b9bab8a1be"
		},
		{
			"cell_type": "code",
			"source": "lakefsEndPoint = '<lakeFS Endpoint URL>' # e.g. 'https://username.aws_region_name.lakefscloud.io' \nlakefsAccessKey = '<lakeFS Access Key>'\nlakefsSecretKey = '<lakeFS Secret Key>'",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "79a78d9a-10b0-4ffa-b501-213876abe1ad"
		},
		{
			"cell_type": "markdown",
			"source": "### AWS credentials",
			"metadata": {},
			"id": "a4136247-debe-42c2-a125-98a8ab660e8a"
		},
		{
			"cell_type": "code",
			"source": "awsAccessKeyId = '<YOUR_AwsAccessKey_HERE>'\nawsSecretAccessKey = '<YOUR_AwsSecretKey_HERE>'\nawsRegion = '<YOUR_AwsRegion_HERE>'",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "1a9efe49-d997-4492-9a86-6ed8ac75bf4b"
		},
		{
			"cell_type": "markdown",
			"source": "### Object Storage",
			"metadata": {
				"tags": []
			},
			"id": "31ca7f50-bd3d-4a86-a4ed-30066b42c971"
		},
		{
			"cell_type": "code",
			"source": "storageNamespace = 's3://<Bucket Name>' # e.g. 's3://bucket'",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "f6a5b298-1879-4441-a607-0e64289bf90e"
		},
		{
			"cell_type": "markdown",
			"source": "### Glue Catalog Information",
			"metadata": {},
			"id": "5943c930-837a-452b-9a50-7a60f7d2ec9c"
		},
		{
			"cell_type": "code",
			"source": "glueDatabaseName = \"glue_exporter_demo\" # This notebook will create this database\nglueCatalogId = \"<Glue Catalog ID or AWS Account ID>\"\nBucketNameForGlueCatalog = '<Bucket Name to store Glue Catalog>'",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "bcca1120-7b55-48dc-809a-0d241b2b4048"
		},
		{
			"cell_type": "markdown",
			"source": "### Athena Information",
			"metadata": {},
			"id": "9157c75f-97fa-406c-b7a3-435f9dad4f40"
		},
		{
			"cell_type": "code",
			"source": "BucketNameForAthenaOutput = '<Bucket Name to store Athena Output>'\nFolderNameForAthenaOutput = 'lakefs-glue-exporter-demo'",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "db9a490a-ee18-4241-9cb9-58c9c684eeb1"
		},
		{
			"cell_type": "markdown",
			"source": "### Install and configure lakectl (lakeFS command-line tool): https://docs.lakefs.io/reference/cli.html",
			"metadata": {},
			"id": "71929b1b-04c4-4bca-9eeb-49fffe382331"
		},
		{
			"cell_type": "markdown",
			"source": "---",
			"metadata": {},
			"id": "853dd30d-ec0e-4302-9dd2-97c4fa21257e"
		},
		{
			"cell_type": "markdown",
			"source": "## Setup\n\n**(you shouldn't need to change anything in this section, just run it)**",
			"metadata": {
				"tags": []
			},
			"id": "7e8191ce-2e9a-4eda-a107-ae99659a5397"
		},
		{
			"cell_type": "code",
			"source": "repo_name = \"glue-exporter-demo\"",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "a45fbab3-5f98-46ad-8e12-9021ed2bf81a"
		},
		{
			"cell_type": "markdown",
			"source": "### Versioning Information",
			"metadata": {},
			"id": "44cd3a7d-8441-432c-b18e-85d98a4364b5"
		},
		{
			"cell_type": "code",
			"source": "mainBranch = \"main\"\nglueTestBranch = \"glue_test_branch\"\ncustomersTable = \"customers\"",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "35909158-397d-4c7a-9436-eb027f75d96b"
		},
		{
			"cell_type": "markdown",
			"source": "### Import libraries",
			"metadata": {},
			"id": "26deb70e-bbb9-4238-a301-cff4f2be1d09"
		},
		{
			"cell_type": "code",
			"source": "import os\nimport lakefs\nimport yaml\nfrom pyspark.sql.types import ByteType, IntegerType, LongType, StringType, StructType, StructField\nfrom pyspark.sql.functions import *",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "401b6db8-92b5-4727-b579-d1fd49379560"
		},
		{
			"cell_type": "markdown",
			"source": "### Create lakeFSClient",
			"metadata": {
				"tags": []
			},
			"id": "eaa5bb3f-7432-4637-bc8e-68159bdcb969"
		},
		{
			"cell_type": "code",
			"source": "os.environ[\"LAKECTL_SERVER_ENDPOINT_URL\"] = lakefsEndPoint\nos.environ[\"LAKECTL_CREDENTIALS_ACCESS_KEY_ID\"] = lakefsAccessKey\nos.environ[\"LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY\"] = lakefsSecretKey",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "6bd83c8a-4655-4c27-97fe-434eb7c2801c"
		},
		{
			"cell_type": "markdown",
			"source": "#### Verify lakeFS credentials by getting lakeFS version",
			"metadata": {},
			"id": "d6de94a4-493f-41cc-bdc0-dec9242afc54"
		},
		{
			"cell_type": "code",
			"source": "print(\"Verifying lakeFS credentials…\")\ntry:\n    v=lakefs.client.Client().version\nexcept:\n    print(\"🛑 failed to get lakeFS version\")\nelse:\n    print(f\"…✅lakeFS credentials verified\\n\\nℹ️lakeFS version {v}\")",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "73519e57-00df-46b9-bb36-8eacfe9bf396"
		},
		{
			"cell_type": "markdown",
			"source": "### Define lakeFS Repository",
			"metadata": {},
			"id": "e1c3ca40-a690-4885-90f9-fc7c069c66c5"
		},
		{
			"cell_type": "code",
			"source": "repo = lakefs.Repository(repo_name).create(storage_namespace=f\"{storageNamespace}/{repo_name}\", default_branch=mainBranch, exist_ok=True)\nbranchMain = repo.branch(mainBranch)\nprint(repo)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "cf82a720-f5c3-4e4c-8c42-fbd017639f7f"
		},
		{
			"cell_type": "markdown",
			"source": "### Set up Spark",
			"metadata": {},
			"id": "f0a012b1-a70b-4d76-b433-94d9f4610a61"
		},
		{
			"cell_type": "code",
			"source": "spark.sparkContext._jsc.hadoopConfiguration().set(\"fs.s3.impl\",\"org.apache.hadoop.fs.s3a.S3AFileSystem\")\nspark.sparkContext._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\",lakefsSecretKey)\nspark.sparkContext._jsc.hadoopConfiguration().set(\"fs.s3a.endpoint\",lakefsEndPoint)\nspark.sparkContext._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\",lakefsAccessKey)\nspark.sparkContext._jsc.hadoopConfiguration().set(\"fs.s3a.path.style.access\",\"true\")",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "e2066714-11ca-422f-b809-053ca6d6add5"
		},
		{
			"cell_type": "markdown",
			"source": "### Define some helper functions",
			"metadata": {},
			"id": "1c05db5e-a09c-44b9-8cac-1c34aaa5e552"
		},
		{
			"cell_type": "code",
			"source": "def print_diff_refs(diff_refs):\n    results = map(\n        lambda n:[n.path,n.path_type,n.size_bytes,n.type],\n        diff_refs.results)\n    return results\n\ndef print_commit(log):\n    from datetime import datetime\n    from pprint import pprint\n\n    print('Message:', log.message)\n    print('ID:', log.id)\n    print('Committer:', log.committer)\n    print('Creation Date:', datetime.utcfromtimestamp(log.creation_date).strftime('%Y-%m-%d %H:%M:%S'))\n    print('Parents:', log.parents)\n    print('Metadata:')\n    pprint(log.metadata)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "f6375c6e-de1e-4abb-86d2-e53397606aed"
		},
		{
			"cell_type": "code",
			"source": "import boto3\nimport pandas as pd\n\ns3 = boto3.resource('s3')\ns3_client = boto3.client('s3')\nathena_client = boto3.client(service_name='athena', region_name=awsRegion)\n\ndef run_query(client, query):\n    response = client.start_query_execution(\n        QueryString=query,\n        QueryExecutionContext={ 'Database': glueDatabaseName },\n        ResultConfiguration={ 'OutputLocation': 's3://{}/{}/'.format(BucketNameForAthenaOutput, FolderNameForAthenaOutput) },\n    )\n    return response\n\ndef validate_query(client, query_id):\n    resp = [\"FAILED\", \"SUCCEEDED\", \"CANCELLED\"]\n    response = client.get_query_execution(QueryExecutionId=query_id)\n    # wait until query finishes\n    while response[\"QueryExecution\"][\"Status\"][\"State\"] not in resp:\n        response = client.get_query_execution(QueryExecutionId=query_id)\n\n    return response[\"QueryExecution\"][\"Status\"][\"State\"]\n\ndef execute_athena_query(query):\n    print('start query: {}\\n'.format(query))\n    qe = run_query(athena_client, query)\n    qstate = validate_query(athena_client, qe[\"QueryExecutionId\"])\n    print('query state: {}\\n'.format(qstate))\n\n    file_name = \"{}/{}.csv\".format(FolderNameForAthenaOutput, qe[\"QueryExecutionId\"])\n    obj = s3_client.get_object(Bucket=BucketNameForAthenaOutput, Key=file_name)\n    return pd.read_csv(obj['Body'])",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "a9d5cdf4-4ac4-4e8d-8698-96dd4d417dd1"
		},
		{
			"cell_type": "markdown",
			"source": "### Define CUSTOMER.csv data file schema",
			"metadata": {},
			"id": "b51259d8-85da-45d3-986b-f898fc228bd4"
		},
		{
			"cell_type": "code",
			"source": "customersSchema = StructType([\n  StructField(\"Customer_ID\", IntegerType(), False),\n  StructField(\"Country\", StringType(), False),\n  StructField(\"Gender\", StringType(), False),\n  StructField(\"Personal_ID\", IntegerType(), True),\n  StructField(\"Customer_Name\", StringType(), False),\n  StructField(\"Customer_FirstName\", StringType(), False),\n  StructField(\"Customer_LastName\", StringType(), False),\n  StructField(\"Birth_Date\", StringType(), False),\n  StructField(\"Customer_Address\", StringType(), False),\n  StructField(\"Street_ID\", LongType(), False),\n  StructField(\"Street_Number\", IntegerType(), False),\n  StructField(\"Customer_Type_ID\", IntegerType(), False)\n])",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "fafcc0b8-c9e6-4574-b56d-2fc9075d9299"
		},
		{
			"cell_type": "markdown",
			"source": "### Let’s define the table descriptor and upload it to lakeFS",
			"metadata": {},
			"id": "a65e8efe-ca2b-4e80-9be2-6bd58f87ad8c"
		},
		{
			"cell_type": "code",
			"source": "table_descriptor = {\n    'name': customersTable,\n    'type': 'hive',\n    'path': f'tables/{customersTable}',\n    'schema': {\n        'type': 'struct',\n        # all the columns spec\n        'fields': [\n            {\n                'name': 'Customer_ID',\n                'type': 'integer',\n                'nullable': False,\n                'metadata': {\n                    'comment': 'Customer ID'\n                }\n            },\n            {\n                'name': 'Country',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Gender',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Personal_ID',\n                'type': 'integer',\n                'nullable': True,\n                'metadata': {}\n            },\n            {\n                'name': 'Customer_Name',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Customer_FirstName',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Customer_LastName',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Birth_Date',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Customer_Address',\n                'type': 'string',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Street_ID',\n                'type': 'bigint',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Street_Number',\n                'type': 'integer',\n                'nullable': False,\n                'metadata': {}\n            },\n            {\n                'name': 'Customer_Type_ID',\n                'type': 'integer',\n                'nullable': False,\n                'metadata': {}\n            }\n        ]\n    }\n}\n\n# Write table descriptor to lakeFS\nwith branchMain.object(path=f'_lakefs_tables/{customersTable}.yaml').writer() as out:\n    yaml.safe_dump(table_descriptor, out)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "a0067248-f2de-44d2-8196-51702e7fa62b"
		},
		{
			"cell_type": "markdown",
			"source": "### Upload the Glue Catalog exporter script to lakeFS",
			"metadata": {},
			"id": "89397a46-2054-4e25-b278-350ca7c3f4b2"
		},
		{
			"cell_type": "code",
			"source": "luaScriptName = \"scripts/glue_export.lua\"\n\nlua_script = \"\"\"\n\nlocal aws = require(\"aws\")\nlocal symlink_exporter = require(\"lakefs/catalogexport/symlink_exporter\")\nlocal glue_exporter = require(\"lakefs/catalogexport/glue_exporter\")\nlocal utils = require(\"lakefs/catalogexport/internal\")\n\n-- settings \nlocal access_key = args.aws.aws_access_key_id\nlocal secret_key = args.aws.aws_secret_access_key\nlocal region = args.aws.aws_region\nlocal table_path = args.table_source -- table descriptor \nlocal branch_or_tag = utils.ref_from_branch_or_tag(action)\nlocal db = string.format(\"%s_%s\", args.catalog.db_name, branch_or_tag)\nlocal table_input = args.catalog.table_input -- table input (AWS input spec) for Glue\n-- export symlinks \nlocal s3 = aws.s3_client(access_key, secret_key, region)\nlocal result = symlink_exporter.export_s3(s3, table_path, action, {debug=true})\n-- register glue table\nlocal glue = aws.glue_client(access_key, secret_key, region)\nlocal create_db_input = {DatabaseInput = {Description = \"Created via LakeFS Action\"}, Tags = {Owner = \"Amit\"}}\nlocal res = glue_exporter.export_glue(glue, db, table_path, table_input, action, {debug=true, create_db_input=create_db_input})\n\n\"\"\"\n\nbranchMain.object(path=luaScriptName).upload(data=lua_script, mode='wb')",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "d3dca116-dbb8-4dcd-a9eb-6b4c9e7cfa9d"
		},
		{
			"cell_type": "markdown",
			"source": "### Define an action configuration that will run the above script after a commit/merge is completed over the main or glueTestBranch branches or after glueTestBranch branch is created.\n### Upload it to lakeFS.",
			"metadata": {},
			"id": "a97d549c-4798-4e32-9d24-a77238077638"
		},
		{
			"cell_type": "code",
			"source": "hook_definition = {\n    'name': 'Glue Exporter',\n    'on': {\n        'post-commit': {\n            'branches': [mainBranch, glueTestBranch+'*']\n        },\n        'post-create-branch': {\n            'branches': [glueTestBranch+'*']\n        }\n    },\n    'hooks': [\n        {\n            'id': 'Glue-Registration',\n            'type': 'lua',\n            'properties': {\n                'script_path': luaScriptName,\n                'args': {\n                    'aws': {\n                        'aws_access_key_id': awsAccessKeyId,\n                        'aws_secret_access_key': awsSecretAccessKey,\n                        'aws_region': awsRegion\n                    },\n                    'table_source': f'_lakefs_tables/{customersTable}.yaml',\n                    'catalog': {\n                        'db_name': glueDatabaseName,\n                        'table_input': {\n                            'StorageDescriptor': {\n                                'InputFormat': 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat',\n                                'OutputFormat': 'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat',\n                                'SerdeInfo': {\n                                   'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'\n                                }\n                            },\n                            'Parameters': {\n                                'classification': 'parquet',\n                                'EXTERNAL': 'TRUE',\n                                'parquet.compression': 'SNAPPY'\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    ]\n}\n\nwith branchMain.object(path='_lakefs_actions/glue_exporter_action.yaml').writer() as out:\n    yaml.safe_dump(hook_definition, out)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "b9c346f4-40eb-4bae-8ec0-ab4eebc3f7ee"
		},
		{
			"cell_type": "markdown",
			"source": "---",
			"metadata": {},
			"id": "25ea2edb-6c7e-46b9-9aea-fa15952cf3cb"
		},
		{
			"cell_type": "markdown",
			"source": "# Main demo starts here 🚦 👇🏻",
			"metadata": {},
			"id": "406d9862-c34a-4281-80dc-32d15dc446b3"
		},
		{
			"cell_type": "markdown",
			"source": "For this demo - we'll be utilizing a dataset - [Orion Star - Sports and outdoors RDBMS dataset](https://www.kaggle.com/datasets/chethanp11/orion-star-sports-and-outdoors-rdbms-dataset) from [Kaggle](https://www.kaggle.com/).",
			"metadata": {},
			"id": "cbc87fa4-3d84-4c13-a612-b2f14d15d4e6"
		},
		{
			"cell_type": "markdown",
			"source": "## Run following command on your computer to clone lakeFS samples repo along with sample data used by this notebook:",
			"metadata": {},
			"id": "d70fb771-6b77-4905-924e-439329b03716"
		},
		{
			"cell_type": "markdown",
			"source": "### git clone https://github.com/treeverse/lakeFS-samples.git",
			"metadata": {},
			"id": "93deb103-a2df-4a0e-85c7-9c6570d84b26"
		},
		{
			"cell_type": "markdown",
			"source": "## Print the command and run it on your computer to upload sample data to lakeFS repository",
			"metadata": {},
			"id": "d3979d6e-de31-43d0-b12d-b9b33580cb64"
		},
		{
			"cell_type": "code",
			"source": "print(f\"cd lakeFS-samples && lakectl fs upload -s ./data/OrionStar lakefs://{repo.id}/main/data/OrionStar/ --recursive && lakectl commit lakefs://{repo.id}/main -m 'Uploaded sample data'\")",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "fffbd19b-38c6-44e3-ae7e-e28fcd86bdeb"
		},
		{
			"cell_type": "markdown",
			"source": "## Create Customers table in the main branch (using [CUSTOMER.csv](./data/samples/OrionStar/CUSTOMER.csv) file)",
			"metadata": {},
			"id": "b1577257-81b2-4971-bafb-5aabfd1e603e"
		},
		{
			"cell_type": "code",
			"source": "customersTablePath = f\"s3a://{repo.id}/{mainBranch}/tables/{customersTable}\"\nprint(customersTablePath)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "7229f90d-accb-4f8f-9713-50cd7c8044b5"
		},
		{
			"cell_type": "markdown",
			"source": "#### Read CSV file and create Hive table",
			"metadata": {},
			"id": "be593469-8b0a-4a66-ac45-0d77c74808e4"
		},
		{
			"cell_type": "code",
			"source": "df = spark.read.csv(f\"s3a://{repo.id}/{mainBranch}/data/OrionStar/CUSTOMER.csv\",header=True,schema=customersSchema)\ndf.write.parquet(customersTablePath)\ndf.show(10)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "e2af9b98-118d-4952-84c7-2d0a9057bb69"
		},
		{
			"cell_type": "markdown",
			"source": "## Commit changes and attach some metadata",
			"metadata": {},
			"id": "b763c2a9-561c-447c-ae2b-0b8fa13fedb0"
		},
		{
			"cell_type": "code",
			"source": "ref = branchMain.commit(\n    message=\"Added configuration files and customers tables!\",\n    metadata={'using': 'python_api'})\nprint_commit(ref.get_commit())",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "6aa5bcb2-2505-46d3-9ba8-7faa4abdfd69"
		},
		{
			"cell_type": "markdown",
			"source": "## Execute Athena query to read the data",
			"metadata": {},
			"id": "f1ad229b-51da-4c1b-82d7-d30916a287ce"
		},
		{
			"cell_type": "code",
			"source": "print(execute_athena_query(f'SELECT * FROM \"{glueDatabaseName}_{mainBranch}\".\"{customersTable}_{repo_name}_{mainBranch}_{ref.get_commit().id[:6]}\"'))",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "34f6d289-d680-45c6-b526-becb92c941ff"
		},
		{
			"cell_type": "markdown",
			"source": "# 🟢 ETL Job Starts",
			"metadata": {},
			"id": "63ec287f-b340-43be-983f-fad43b024ac5"
		},
		{
			"cell_type": "markdown",
			"source": "## Create a new branch to run your ETL job",
			"metadata": {},
			"id": "5fa6a1a9-70c2-4416-b860-97aa1c2a6dfa"
		},
		{
			"cell_type": "code",
			"source": "branchGlueTest = repo.branch(glueTestBranch).create(source_reference=mainBranch)\nprint(f\"{glueTestBranch} ref:\", branchGlueTest.get_commit().id)",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "8a3f0d8e-9001-46b5-b52c-83e24b8d8338"
		},
		{
			"cell_type": "markdown",
			"source": "## Execute Athena query to read the data from the new branch",
			"metadata": {},
			"id": "f7985acd-b71f-478d-899d-365452296727"
		},
		{
			"cell_type": "code",
			"source": "print(execute_athena_query(f'SELECT * FROM \"{glueDatabaseName}_{glueTestBranch}\".\"{customersTable}_{repo_name}_{glueTestBranch}_{branchGlueTest.get_commit().id[:6]}\"'))",
			"metadata": {
				"trusted": true,
				"tags": []
			},
			"execution_count": null,
			"outputs": [],
			"id": "452a9a5b-3a1d-45ca-87d7-3b0b0373bdc1"
		},
		{
			"cell_type": "markdown",
			"source": "## More Questions?\n\n###### Join the lakeFS Slack group - https://lakefs.io/slack",
			"metadata": {},
			"id": "557472bd-39ed-43d3-a00a-ed69ed8ab280"
		}
	]
}