Add composer-dev for local Airflow (#3874)

* Remove old local airflow setup

* Add composer-local-dev setup

* Delete unused gitkeep file

* Update README.md

* Build and test make file

Signed-off-by: Doc Ritezel <[email protected]>

* Reorganize README.md to reflect the Makefile updates

---------

Signed-off-by: Doc Ritezel <[email protected]>
Co-authored-by: Jeff Parr <[email protected]>

Author: ohrite

airflow/.development.env

@@ -0,0 +1,51 @@
+AIRFLOW_VAR_EXTRACT_BUCKET=gs://gtfs-data
+AIRFLOW_ENV=cal-itp-data-infra-staging
+AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT='google-cloud-platform://'
+AIRFLOW_CONN_HTTP_NTD=https://data.transportation.gov
+AIRFLOW_CONN_HTTP_BLACKCAT=https://services.blackcattransit.com
+AIRFLOW_CONN_HTTP_MOBILITY_DATABASE=https://bit.ly/catalogs-csv
+GOOGLE_CLOUD_PROJECT=cal-itp-data-infra-staging
+CALITP_USER=pipeline
+CALITP_AUTH=cloud
+CALITP_BQ_MAX_BYTES=50000000000
+CALITP_BQ_LOCATION=us-west2
+POD_LOCATION=us-west2
+CALITP__ELAVON_SFTP_HOSTNAME=34.145.56.125
+CALITP__ELAVON_SFTP_PORT=2200
+CALITP__ELAVON_SFTP_USERNAME=elavon
+DBT_TARGET=staging
+SENTRY_DSN=https://[email protected]/2
+SENTRY_ENVIRONMENT=cal-itp-data-infra-staging
+CALITP_BUCKET__AGGREGATOR_SCRAPER=gs://calitp-staging-aggregator-scraper
+CALITP_BUCKET__AIRTABLE=gs://calitp-staging-airtable
+CALITP_BUCKET__AMPLITUDE_BENEFITS_EVENTS=gs://calitp-staging-amplitude-benefits-events
+CALITP_BUCKET__ANALYSIS_OUTPUT_MODELS=gs://calitp-staging-analysis-output-models
+CALITP_BUCKET__DBT_ARTIFACTS=gs://calitp-staging-dbt-artifacts
+CALITP_BUCKET__DBT_DOCS=gs://calitp-staging-dbt-docs
+CALITP_BUCKET__ELAVON_PARSED=gs://calitp-staging-elavon-parsed
+CALITP_BUCKET__ELAVON_RAW=gs://calitp-staging-elavon-raw
+CALITP_BUCKET__GTFS_DOWNLOAD_CONFIG=gs://calitp-staging-gtfs-download-config
+CALITP_BUCKET__GTFS_DOWNLOAD_CONFIG_PROD_SOURCE=gs://calitp-staging-gtfs-download-config
+CALITP_BUCKET__GTFS_DOWNLOAD_CONFIG_TEST_DESTINATION=gs://calitp-staging-gtfs-download-config-test
+CALITP_BUCKET__GTFS_RT_PARSED=gs://calitp-staging-gtfs-rt-parsed
+CALITP_BUCKET__GTFS_RT_RAW=gs://calitp-staging-gtfs-rt-raw-v2
+CALITP_BUCKET__GTFS_RT_VALIDATION=gs://calitp-staging-gtfs-rt-validation
+CALITP_BUCKET__GTFS_SCHEDULE_PARSED=gs://calitp-staging-gtfs-schedule-parsed
+CALITP_BUCKET__GTFS_SCHEDULE_PARSED_HOURLY=gs://calitp-staging-gtfs-schedule-parsed-hourly
+CALITP_BUCKET__GTFS_SCHEDULE_RAW=gs://calitp-staging-gtfs-schedule-raw-v2
+CALITP_BUCKET__GTFS_SCHEDULE_UNZIPPED=gs://calitp-staging-gtfs-schedule-unzipped
+CALITP_BUCKET__GTFS_SCHEDULE_UNZIPPED_HOURLY=gs://calitp-staging-gtfs-schedule-unzipped-hourly
+CALITP_BUCKET__GTFS_SCHEDULE_VALIDATION=gs://calitp-staging-gtfs-schedule-validation
+CALITP_BUCKET__GTFS_SCHEDULE_VALIDATION_HOURLY=gs://calitp-staging-gtfs-schedule-validation-hourly
+CALITP_BUCKET__KUBA=gs://calitp-staging-kuba
+CALITP_BUCKET__LITTLEPAY_PARSED=gs://calitp-staging-payments-littlepay-parsed
+CALITP_BUCKET__LITTLEPAY_PARSED_V3=gs://calitp-staging-payments-littlepay-parsed-v3
+CALITP_BUCKET__LITTLEPAY_RAW=gs://calitp-staging-payments-littlepay-raw
+CALITP_BUCKET__LITTLEPAY_RAW_V3=gs://calitp-staging-payments-littlepay-raw-v3
+CALITP_BUCKET__NTD_API_DATA_PRODUCTS=gs://calitp-staging-ntd-api-products
+CALITP_BUCKET__NTD_REPORT_VALIDATION=gs://calitp-staging-ntd-report-validation
+CALITP_BUCKET__NTD_XLSX_DATA_PRODUCTS__CLEAN=gs://calitp-staging-ntd-xlsx-products-clean
+CALITP_BUCKET__NTD_XLSX_DATA_PRODUCTS__RAW=gs://calitp-staging-ntd-xlsx-products-raw
+CALITP_BUCKET__PUBLISH=gs://calitp-staging-publish
+CALITP_BUCKET__SENTRY_EVENTS=gs://calitp-staging-sentry
+CALITP_BUCKET__STATE_GEOPORTAL_DATA_PRODUCTS=gs://calitp-staging-state-geoportal-scrape

airflow/.gitignore

@@ -2,3 +2,4 @@ logs/*
 airflow.sh
 *.db
 .env
+/composer

airflow/Dockerfile

@@ -0,0 +1,63 @@
+.PHONY: clean sync setup start restart stop
+
+COMPOSER_ENVIRONMENT_SOURCE := calitp-staging-composer
+COMPOSER_ENVIRONMENT_NAME := calitp-development-composer
+COMPOSER_ENVIRONMENT_PATH := composer/$(COMPOSER_ENVIRONMENT_NAME)
+
+COMPOSER_VERSION := composer-2.8.6-airflow-2.6.3
+COMPOSER_PROJECT := cal-itp-data-infra-staging
+COMPOSER_LOCATION := us-west2
+
+WAREHOUSE_PATH := ../warehouse
+WAREHOUSE_TARGET := staging
+
+restart: sync
+	poetry run composer-dev restart
+
+start: sync
+	poetry run composer-dev start
+
+stop:
+	poetry run composer-dev stop
+
+sync: $(WAREHOUSE_PATH)/target/manifest.json $(COMPOSER_ENVIRONMENT_PATH)
+	rsync -ar plugins/ $(COMPOSER_ENVIRONMENT_PATH)/plugins/
+	rsync -ar $(WAREHOUSE_PATH)/ $(COMPOSER_ENVIRONMENT_PATH)/data/warehouse/
+	cp -f requirements.txt $(COMPOSER_ENVIRONMENT_PATH)/requirements.txt
+	cp -f .development.env $(COMPOSER_ENVIRONMENT_PATH)/variables.env
+	cat .env >> $(COMPOSER_ENVIRONMENT_PATH)/variables.env
+
+$(WAREHOUSE_PATH)/target/manifest.json:
+	cd $(WAREHOUSE_PATH) && \
+	poetry install && \
+	poetry run dbt deps && \
+	poetry run dbt compile --target $(WAREHOUSE_TARGET)
+
+$(COMPOSER_ENVIRONMENT_PATH):
+	poetry run composer-dev create \
+		--project $(COMPOSER_PROJECT) \
+		--from-source-environment $(COMPOSER_ENVIRONMENT_SOURCE) \
+		--location $(COMPOSER_LOCATION) \
+		--port 8080 \
+		--dags-path dags/ \
+		--database postgresql \
+		$(COMPOSER_ENVIRONMENT_NAME)
+
+setup: $(COMPOSER_ENVIRONMENT_PATH)
+
+clean:
+ifneq (,$(wildcard $(COMPOSER_ENVIRONMENT_PATH)/data))
+	rm -r $(COMPOSER_ENVIRONMENT_PATH)/data
+endif
+ifneq (,$(wildcard $(COMPOSER_ENVIRONMENT_PATH)/plugins))
+	rm -r $(COMPOSER_ENVIRONMENT_PATH)/plugins
+endif
+ifneq (,$(wildcard $(COMPOSER_ENVIRONMENT_PATH)/requirements.txt))
+	rm -r $(COMPOSER_ENVIRONMENT_PATH)/requirements.txt
+endif
+ifneq (,$(wildcard $(COMPOSER_ENVIRONMENT_PATH)/variables.env))
+	rm -r $(COMPOSER_ENVIRONMENT_PATH)/variables.env
+endif
+
+teardown: stop
+	rm -r $(COMPOSER_ENVIRONMENT_PATH)