From 3067564d02f3d934f79cde5852b631185336bfeb Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Tue, 10 Sep 2024 13:43:28 -0400 Subject: [PATCH 01/50] Adjust brush size selector style (#85) --- .../components/sidebar/BrushSizeSelector.tsx | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/app/src/app/components/sidebar/BrushSizeSelector.tsx b/app/src/app/components/sidebar/BrushSizeSelector.tsx index 05a5f8122..948b5fab9 100644 --- a/app/src/app/components/sidebar/BrushSizeSelector.tsx +++ b/app/src/app/components/sidebar/BrushSizeSelector.tsx @@ -1,6 +1,15 @@ -import { Slider, Flex } from "@radix-ui/themes"; +import { Slider, Flex, Heading, Text } from "@radix-ui/themes"; import { useMapStore } from "../../store/mapStore"; +/** + * BrushSizeSelector + * Note: right now the brush size is an arbitrary value between + * 1 and 100. This is slightly arbitrary. Should we communicate brush size + * differently or not display the brush size? + * + * @description A slider to select the brush size + * @returns {JSX.Element} The component + */ export function BrushSizeSelector() { const { brushSize, setBrushSize } = useMapStore((state) => ({ brushSize: state.brushSize, @@ -13,8 +22,15 @@ export function BrushSizeSelector() { }; return ( - -

Brush Size

+ + + Brush Size + - {brushSize} + + {brushSize} + ); } From 6148443f8b2599e3bba809e3bc51bd7b6b2cebf9 Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Tue, 10 Sep 2024 23:13:26 -0500 Subject: [PATCH 02/50] Eraser feature! (#83) Co-authored-by: Raphael Paul Laude --- app/src/app/components/Map.tsx | 8 +++- .../components/sidebar/MapModeSelector.jsx | 2 +- app/src/app/components/sidebar/Sidebar.tsx | 8 +++- app/src/app/constants/types.ts | 2 +- app/src/app/store/mapStore.ts | 2 +- app/src/app/utils/events/handlers.ts | 24 +++++++++--- app/src/app/utils/events/mapEvents.ts | 30 +++++++-------- .../09d011c1b387_zones_can_be_null.py | 27 +++++++++++++ backend/app/main.py | 2 +- backend/app/models.py | 2 +- backend/tests/test_main.py | 38 +++++++++++++++++++ 11 files changed, 115 insertions(+), 30 deletions(-) create mode 100644 backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index 67fc00572..700a75a43 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -52,6 +52,7 @@ export const MapComponent: React.FC = () => { }); const { + activeTool, freshMap, zoneAssignments, mapDocument, @@ -60,6 +61,7 @@ export const MapComponent: React.FC = () => { setMapRef, setMapMetrics, } = useMapStore((state) => ({ + activeTool: state.activeTool, freshMap: state.freshMap, zoneAssignments: state.zoneAssignments, mapDocument: state.mapDocument, @@ -172,8 +174,10 @@ export const MapComponent: React.FC = () => { */ useEffect(() => { if (mapLoaded && map.current && zoneAssignments.size) { - const assignments = FormatAssignments(); - patchUpdates.mutate(assignments); + if (activeTool === "brush" || activeTool === "eraser") { + const assignments = FormatAssignments(); + patchUpdates.mutate(assignments); + } } }, [mapLoaded, zoneAssignments]); diff --git a/app/src/app/components/sidebar/MapModeSelector.jsx b/app/src/app/components/sidebar/MapModeSelector.jsx index 1a04a62ca..e2f9ddc88 100644 --- a/app/src/app/components/sidebar/MapModeSelector.jsx +++ b/app/src/app/components/sidebar/MapModeSelector.jsx @@ -17,7 +17,7 @@ export function MapModeSelector() { const activeTools = [ { mode: "pan", disabled: false, label: "Pan", icon: }, { mode: "brush", disabled: false, label: "Brush", icon: }, - { mode: "erase", disabled: true, label: "Erase", icon: }, + { mode: "eraser", disabled: false, label: "Erase", icon: }, ]; const handleRadioChange = (value) => { diff --git a/app/src/app/components/sidebar/Sidebar.tsx b/app/src/app/components/sidebar/Sidebar.tsx index 789af0126..17d00cd41 100644 --- a/app/src/app/components/sidebar/Sidebar.tsx +++ b/app/src/app/components/sidebar/Sidebar.tsx @@ -26,13 +26,17 @@ export default function SidebarComponent() { - {activeTool === "brush" ? ( + {activeTool === "brush" || activeTool === "eraser" ? (
- {" "}
) : null} + {activeTool === "brush" ? ( +
+ +
+ ) : null} diff --git a/app/src/app/constants/types.ts b/app/src/app/constants/types.ts index 4f9bebaea..b5c74534c 100644 --- a/app/src/app/constants/types.ts +++ b/app/src/app/constants/types.ts @@ -1,6 +1,6 @@ import type { MapOptions, MapLibreEvent } from "maplibre-gl"; -export type Zone = number; +export type Zone = number | null; export type GEOID = string; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 08404d000..da4c5f0f2 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -39,7 +39,7 @@ export interface MapStore { setSelectedZone: (zone: Zone) => void; accumulatedBlockPopulations: Map; resetAccumulatedBlockPopulations: () => void; - zoneAssignments: Map; // geoid -> zone + zoneAssignments: Map; // geoid -> zone setZoneAssignments: (zone: Zone, gdbPaths: Set) => void; resetZoneAssignments: () => void; zonePopulations: Map; diff --git a/app/src/app/utils/events/handlers.ts b/app/src/app/utils/events/handlers.ts index fe03ab266..92b8f1878 100644 --- a/app/src/app/utils/events/handlers.ts +++ b/app/src/app/utils/events/handlers.ts @@ -2,6 +2,7 @@ import { BLOCK_SOURCE_ID } from "@/app/constants/layers"; import { MutableRefObject } from "react"; import { Map, MapGeoJSONFeature } from "maplibre-gl"; import { debounce } from "lodash"; +import { Zone } from "@/app/constants/types"; import { MapStore } from "@/app/store/mapStore"; /** @@ -11,7 +12,7 @@ import { MapStore } from "@/app/store/mapStore"; * @returns void - but updates the zoneAssignments and zonePopulations in the store */ const debouncedSetZoneAssignments = debounce( - (mapStoreRef: MapStore, selectedZone: number, geoids: Set) => { + (mapStoreRef: MapStore, selectedZone: Zone, geoids: Set) => { mapStoreRef.setZoneAssignments(selectedZone, geoids); const accumulatedBlockPopulations = mapStoreRef.accumulatedBlockPopulations; @@ -41,21 +42,32 @@ export const SelectMapFeatures = ( map: MutableRefObject, mapStoreRef: MapStore, ) => { + let { + accumulatedGeoids, + accumulatedBlockPopulations, + activeTool, + selectedLayer, + selectedZone, + } = mapStoreRef; + if (activeTool === "eraser") { + selectedZone = null; + } + features?.forEach((feature) => { map.current?.setFeatureState( { source: BLOCK_SOURCE_ID, id: feature?.id ?? undefined, - sourceLayer: mapStoreRef.selectedLayer?.name, + sourceLayer: selectedLayer?.name, }, - { selected: true, zone: mapStoreRef.selectedZone }, + { selected: true, zone: selectedZone }, ); }); if (features?.length) { features.forEach((feature) => { - mapStoreRef.accumulatedGeoids.add(feature.properties?.path); + accumulatedGeoids.add(feature.properties?.path); - mapStoreRef.accumulatedBlockPopulations.set( + accumulatedBlockPopulations.set( feature.properties?.path, feature.properties?.total_pop, ); @@ -80,7 +92,7 @@ export const SelectZoneAssignmentFeatures = (mapStoreRef: MapStore) => { if (accumulatedGeoids?.size) { debouncedSetZoneAssignments( mapStoreRef, - mapStoreRef.selectedZone, + mapStoreRef.activeTool === "brush" ? mapStoreRef.selectedZone : null, mapStoreRef.accumulatedGeoids, ); } diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index dd8470248..59cfa12dd 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -36,14 +36,11 @@ export const handleMapClick = ( if (activeTool === "brush" || activeTool === "eraser") { const selectedFeatures = mapStore.paintFunction(map, e, mapStore.brushSize); - if (activeTool === "brush" && sourceLayer) { + if (sourceLayer) { // select on both the map object and the store SelectMapFeatures(selectedFeatures, map, mapStore).then(() => { SelectZoneAssignmentFeatures(mapStore); }); - } else if (activeTool === "eraser") { - // erase features - // TODO: implement eraser } } else { // tbd, for pan mode - is there an info mode on click? @@ -59,7 +56,7 @@ export const handleMapMouseUp = ( const activeTool = mapStore.activeTool; const isPainting = mapStore.isPainting; - if (activeTool === "brush" && isPainting) { + if ((activeTool === "brush" || activeTool === "eraser") && isPainting) { // set isPainting to false mapStore.setIsPainting(false); SelectZoneAssignmentFeatures(mapStore); @@ -80,12 +77,7 @@ export const handleMapMouseDown = ( } else if (activeTool === "brush" || activeTool === "eraser") { // disable drag pan map.current?.dragPan.disable(); - if (activeTool === "brush") { - mapStore.setIsPainting(true); - return; - } else if (activeTool === "eraser") { - // erase features tbd - } + mapStore.setIsPainting(true); } }; @@ -109,7 +101,11 @@ export const handleMapMouseLeave = ( const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; const sourceLayer = mapStore.selectedLayer?.name; - if (sourceLayer && hoverFeatureIds.current.size && activeTool === "brush") { + if ( + sourceLayer && + hoverFeatureIds.current.size && + (activeTool === "brush" || activeTool === "eraser") + ) { UnhighlightFeature(map, hoverFeatureIds, sourceLayer); } }; @@ -132,10 +128,14 @@ export const handleMapMouseMove = ( const isPainting = mapStore.isPainting; const sourceLayer = mapStore.selectedLayer?.name; const selectedFeatures = mapStore.paintFunction(map, e, mapStore.brushSize); - if (sourceLayer && activeTool === "brush") { + if (sourceLayer && (activeTool === "brush" || activeTool === "eraser")) { HighlightFeature(selectedFeatures, map, hoverFeatureIds, sourceLayer); - } - if (activeTool === "brush" && isPainting && sourceLayer) { + } + if ( + (activeTool === "brush" || activeTool === "eraser") && + isPainting && + sourceLayer + ) { // selects in the map object; the store object // is updated in the mouseup event SelectMapFeatures(selectedFeatures, map, mapStore); diff --git a/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py b/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py new file mode 100644 index 000000000..d795225bf --- /dev/null +++ b/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py @@ -0,0 +1,27 @@ +"""zones_can_be_null + +Revision ID: 09d011c1b387 +Revises: 8437ce954087 +Create Date: 2024-09-09 13:34:59.347083 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '09d011c1b387' +down_revision: Union[str, None] = '8437ce954087' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(sa.text("ALTER TABLE document.assignments ALTER zone DROP NOT NULL")) + + +def downgrade() -> None: + op.execute(sa.text("DELETE FROM document.assignments WHERE zone IS NULL")) + op.execute(sa.text("ALTER TABLE document.assignments ALTER zone SET NOT NULL")) diff --git a/backend/app/main.py b/backend/app/main.py index 17f45e28e..011d381b2 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -174,7 +174,7 @@ async def get_document(document_id: str, session: Session = Depends(get_session) async def get_total_population( document_id: str, session: Session = Depends(get_session) ): - stmt = text("SELECT * from get_total_population(:document_id)") + stmt = text("SELECT * from get_total_population(:document_id) WHERE zone IS NOT NULL") try: result = session.execute(stmt, {"document_id": document_id}) return [ diff --git a/backend/app/models.py b/backend/app/models.py index 22b390caf..df0829df7 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -75,7 +75,7 @@ class AssignmentsBase(SQLModel): metadata = MetaData(schema=DOCUMENT_SCHEMA) document_id: str = Field(sa_column=Column(UUIDType, primary_key=True)) geo_id: str = Field(primary_key=True) - zone: int + zone: int | None class Assignments(AssignmentsBase, table=True): diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 08ecf4c06..8144f5293 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -383,6 +383,21 @@ def test_patch_assignments(client, document_id): assert response.json() == {"assignments_upserted": 3} +def test_patch_assignments_nulls(client, document_id): + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": None}, + ] + }, + ) + assert response.status_code == 200 + assert response.json() == {"assignments_upserted": 3} + + def test_patch_assignments_twice(client, document_id): response = client.patch( "/api/update_assignments", @@ -430,6 +445,29 @@ def test_get_document_population_totals( assert data == [{"zone": 1, "total_pop": 67}, {"zone": 2, "total_pop": 130}] +def test_get_document_population_totals_null_assignments( + client, document_id, ks_demo_view_census_blocks +): + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": None}, + ] + }, + ) + assert response.status_code == 200 + assert response.json() == {"assignments_upserted": 3} + + doc_uuid = str(uuid.UUID(document_id)) + result = client.get(f"/api/document/{doc_uuid}/total_pop") + assert result.status_code == 200 + data = result.json() + assert data == [{"zone": 1, "total_pop": 67}] + + def test_get_document_vap_totals( client, assignments_document_id_total_vap, ks_demo_view_census_blocks_total_vap ): From 4f4e2dd3982fbef02f068790e948791fb1fbb3c4 Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 23 Sep 2024 10:18:25 -0500 Subject: [PATCH 03/50] Docker compose config and data load script (#88) --- .gitignore | 1 + README.md | 10 ++++++ app/.env.docker | 2 ++ app/Dockerfile.dev | 20 +++++++++++ backend/.env.docker | 19 ++++++++++ backend/Dockerfile.dev | 20 +++++++++++ backend/scripts/load_data.py | 66 ++++++++++++++++++++++++++++++++++ docker-compose.yml | 70 ++++++++++++++++++++++++++++++++++++ 8 files changed, 208 insertions(+) create mode 100644 app/.env.docker create mode 100644 app/Dockerfile.dev create mode 100644 backend/.env.docker create mode 100644 backend/Dockerfile.dev create mode 100755 backend/scripts/load_data.py create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 4708bacea..e7b9d8366 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ scratch/ PG:* *.dev +sample_data/* \ No newline at end of file diff --git a/README.md b/README.md index 9454bcc46..a81108fb6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,14 @@ The Districtr reboot monorepo. - [`pipelines`](pipelines/): Data pipelines, ETL. Not a main focus of the reboot. For now, will mostly contain scratch data transformation scripts before being integrated into the backend CLI. - [`prototypes`](prototypes/): Prototypes conducted as part of the reboot. +## Quickstart + +The backend (Python), frontend (NextJS), and database (postgres) can be run locally using Docker. + +- Install and configure [Docker](https://www.docker.com/) for your machine +- From the repo root, run `docker-compose up --build` +- To load in data, add data to a folder `sample_data` in the repo root, and in `docker-compose.yml` set `services > backend > environment > LOAD_GERRY_DB_DATA` to `true`. You can change where the script looks for available data with the `GPKG_DATA_DIR` variable. + ## Districtr reboot architecture After experimenting with various technologies (see [`prototypes`](prototypes/)) we landed on the following architecture for the Districtr reboot: @@ -16,11 +24,13 @@ After experimenting with various technologies (see [`prototypes`](prototypes/)) ![Districtr architecture](docs/images/districtr-architecture.png "Districtr architecture") The redesign aims to principally to address three key pain points in the Districtr application’s performance and maintainability: + 1. Slow tile rendering 1. Cumbersome use of tiles as global state for tile rendering and most metric calculation 1. Complexity and poor interoperability in architecture without slow copies And two key feature additions + 1. Block “shattering” 1. A headless CMS (this will be added in a later phase of work / is not currently a focus of the reboot) diff --git a/app/.env.docker b/app/.env.docker new file mode 100644 index 000000000..4e3f04c60 --- /dev/null +++ b/app/.env.docker @@ -0,0 +1,2 @@ +NEXT_PUBLIC_API_URL=http://localhost:8000 +NEXT_PUBLIC_S3_BUCKET_URL=https://pub-fa71193941a74e14a38eee99f30f53d9.r2.dev diff --git a/app/Dockerfile.dev b/app/Dockerfile.dev new file mode 100644 index 000000000..2e4fdf665 --- /dev/null +++ b/app/Dockerfile.dev @@ -0,0 +1,20 @@ +# Use official Node.js image as the base image +FROM node:18-alpine + +# Set working directory inside the container +WORKDIR /app + +# Copy package.json and package-lock.json to install dependencies first +COPY package.json package-lock.json ./ + +# Install dependencies +RUN npm install + +# Copy the rest of the application code +COPY . . + +# Expose the Next.js dev server port +EXPOSE 3000 + +# Start the Next.js application in development mode +CMD ["npm", "run", "dev"] \ No newline at end of file diff --git a/backend/.env.docker b/backend/.env.docker new file mode 100644 index 000000000..e1ab04d0f --- /dev/null +++ b/backend/.env.docker @@ -0,0 +1,19 @@ +# Backend +DOMAIN=localhost +ENVIRONMENT=local +PROJECT_NAME="Districtr v2 backend" +BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,http://localhost:3000,http://127.0.0.1:3000" +SECRET_KEY="super-secret" + +# Postgres +DATABASE_URL=postgresql+psycopg://postgres:postgres@db:5432/districtr +POSTGRES_SCHEME=postgresql+psycopg +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=districtr +POSTGRES_SERVER=db # Use the service name `db` from docker-compose +POSTGRES_PORT=5432 + +# Volumes +VOLUME_PATH=/data + diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev new file mode 100644 index 000000000..478c414bc --- /dev/null +++ b/backend/Dockerfile.dev @@ -0,0 +1,20 @@ +# Use official Python image with version >= 3.11 +FROM python:3.12-slim + +# Set working directory inside the container +WORKDIR /districtr-backend + +# Copy requirements file first and install dependencies +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the backend code into the container +COPY . . + +# Install PostGIS extension and PostgreSQL client for database operations +RUN apt-get update && apt-get install -y postgresql-client libpq-dev gdal-bin + +# Command to run the server +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--reload"] diff --git a/backend/scripts/load_data.py b/backend/scripts/load_data.py new file mode 100755 index 000000000..cc27a2b56 --- /dev/null +++ b/backend/scripts/load_data.py @@ -0,0 +1,66 @@ +import subprocess +import sqlalchemy as sa +from os import environ +from glob import glob + +# Optionally, set a data directory to load in +DATA_DIR = environ.get("GPKG_DATA_DIR", "sample_data") +# flag to load data, by default, will load data +LOAD_DATA = environ.get("LOAD_GERRY_DB_DATA", "true") + + +def update_tile_column(engine): + """ + Update the 'tiles_s3_path' column in the 'gerrydbtable' of the public schema. + + This function connects to the database using the provided SQLAlchemy engine + and executes an UPDATE query. It sets the 'tiles_s3_path' column to a + concatenated string based on the 'name' column. + + Args: + engine (sqlalchemy.engine.Engine): SQLAlchemy engine instance for database connection. + + Prints: + Success message with the number of updated rows or an error message if the update fails. + + Raises: + SQLAlchemyError: If there's an error during the database operation. + """ + print("UPDATING GERRYDB COLUMN") + with engine.connect() as connection: + try: + result = connection.execute( + sa.text( + "UPDATE public.gerrydbtable SET tiles_s3_path = CONCAT('tilesets/', name, '.pmtiles')" + ) + ) + updated_rows = result.rowcount + print(f"Successfully updated {updated_rows} rows in gerrydbtable.") + connection.commit() + except sa.exc.SQLAlchemyError as e: + print(f"Error updating gerrydbtable: {str(e)}") + connection.rollback() + + +def load_sample_data(): + """ + Load sample data from the specified data directory. + + This function iterates through all files with a '.gpkg' extension in the + specified data directory, and for each file, it runs a script to load the + GerryDB view. + + Args: + None + Returns: + None + """ + for gpkg in glob(f"{DATA_DIR}/*.gpkg"): + subprocess.run(["bash", "./scripts/load_gerrydb_view.sh", gpkg]) + + +if __name__ == "__main__": + if LOAD_DATA == "true": + load_sample_data() + engine = sa.create_engine(environ.get("DATABASE_URL")) + update_tile_column(engine) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..fe52aa450 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +services: + backend: + build: + context: ./backend + dockerfile: Dockerfile.dev + container_name: backend + volumes: + - ./backend/app:/districtr-backend/app # Adjust volumes to point to the backend folder + - ./backend/scripts:/districtr-backend/scripts # Ensure scripts are mounted from backend directory + - ./sample_data:/districtr-backend/sample_data + + env_file: + - ./backend/.env.docker # Ensure the env file is also from the backend folder + environment: + LOAD_GERRY_DB_DATA: false + GPKG_DATA_DIR: sample_data + depends_on: + db: + condition: service_healthy + command: bash -c " + until alembic upgrade head; do + echo 'Alembic failed, retrying in 5 seconds...' + sleep 5; + done && + python3 scripts/load_data.py && + uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload --reload-exclude '.venv/**/*.py' + " + ports: + - "8000:8000" + + db: + image: postgis/postgis:15-3.3-alpine + container_name: postgres_db + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: districtr + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + retries: 5 + + frontend: + build: + context: ./app + dockerfile: Dockerfile.dev + container_name: frontend + volumes: + - ./app:/app # Mount the app folder for hot reloading + - ./app/node_modules:/app/node_modules # Bind mount for node_modules + + env_file: + - ./app/.env.docker + ports: + - "3000:3000" # Expose Next.js on port 3000 + command: sh -c "npm install && npm run dev" + + pre-commit: + image: python:3.9 # or whichever version you prefer + volumes: + - .:/app + working_dir: /app + command: sh -c "pip install pre-commit && pre-commit run --all-files" + +volumes: + postgres_data: \ No newline at end of file From f9530cec48e2e3087d2653c60916e91a8cd14d30 Mon Sep 17 00:00:00 2001 From: mariogiampieri Date: Wed, 25 Sep 2024 07:51:31 -0400 Subject: [PATCH 04/50] Revert to classic districtr colors (#93) --- .../app/components/sidebar/ColorPicker.jsx | 4 +- .../sidebar/charts/HorizontalBarChart.tsx | 10 +- app/src/app/constants/colors.ts | 102 ++++++++---------- app/src/app/constants/layers.ts | 34 +++--- 4 files changed, 71 insertions(+), 79 deletions(-) diff --git a/app/src/app/components/sidebar/ColorPicker.jsx b/app/src/app/components/sidebar/ColorPicker.jsx index 0af44efbb..a9c1ca7ac 100644 --- a/app/src/app/components/sidebar/ColorPicker.jsx +++ b/app/src/app/components/sidebar/ColorPicker.jsx @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { palette, color10 } from "../../constants/colors"; +import { _colorScheme } from "../../constants/colors"; import { Button } from "@radix-ui/themes"; import { styled } from "@stitches/react"; import * as RadioGroup from "@radix-ui/react-radio-group"; @@ -22,7 +22,7 @@ export function ColorPicker() { accumulatedGeoids: state.accumulatedGeoids, resetAccumulatedBlockPopulations: state.resetAccumulatedBlockPopulations, })); - const colorArray = color10; + const colorArray = _colorScheme; if (!colorArray) return null; const handleRadioChange = (value) => { console.log( diff --git a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx index dfcf3ff68..58f9ffac8 100644 --- a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx +++ b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx @@ -9,7 +9,7 @@ import { YAxis, Cell, } from "recharts"; -import { color10 } from "@/app/constants/colors"; +import { colorScheme } from "@/app/constants/colors"; type TooltipInput = { active?: boolean; @@ -58,7 +58,8 @@ export const HorizontalBar = () => { { {mapMetrics.data .sort((a, b) => a.zone - b.zone) .map((entry, index) => ( - + ))} diff --git a/app/src/app/constants/colors.ts b/app/src/app/constants/colors.ts index 5c74147bd..d4249155b 100644 --- a/app/src/app/constants/colors.ts +++ b/app/src/app/constants/colors.ts @@ -27,63 +27,47 @@ import { sky, } from "@radix-ui/colors"; -export const palette = { - colors: { - ...(tomato as object), - ...(red as object), - ...(ruby as object), - ...(crimson as object), - ...(pink as object), - ...(plum as object), - ...(purple as object), - ...(violet as object), - ...(iris as object), - ...(indigo as object), - ...(blue as object), - ...(cyan as object), - ...(teal as object), - ...(jade as object), - ...(green as object), - ...(grass as object), - ...(orange as object), - ...(amber as object), - ...(yellow as object), - ...(gold as object), - ...(brown as object), - ...(bronze as object), - ...(gray as object), - ...(mint as object), - ...(lime as object), - ...(sky as object), - } as { [key: string]: { [key: string]: string } }, -}; - -// bright colors! -export const color10 = [ - tomato.tomato10, - red.red10, - ruby.ruby10, - crimson.crimson10, - pink.pink10, - plum.plum10, - purple.purple10, - violet.violet10, - iris.iris10, - indigo.indigo10, - blue.blue10, - cyan.cyan10, - teal.teal10, - jade.jade10, - green.green10, - grass.grass10, - orange.orange10, - amber.amber10, - yellow.yellow10, - gold.gold10, - brown.brown10, - bronze.bronze10, - gray.gray10, - mint.mint10, - lime.lime10, - sky.sky10, +export const colorScheme = [ + "#0099cd", + "#ffca5d", + "#00cd99", + "#99cd00", + "#cd0099", + "#aa44ef", // lighter, req from San Diego + // Color brewer: + "#8dd3c7", + "#bebada", + "#fb8072", + "#80b1d3", + "#fdb462", + "#b3de69", + "#fccde5", + "#bc80bd", + "#ccebc5", + "#ffed6f", + "#ffffb3", + // other color brewer scheme: + "#a6cee3", + "#1f78b4", + "#b2df8a", + "#33a02c", + "#fb9a99", + "#e31a1c", + "#fdbf6f", + "#ff7f00", + "#cab2d6", + "#6a3d9a", + "#b15928", + // random material design colors: + "#64ffda", + "#00B8D4", + "#A1887F", + "#76FF03", + "#DCE775", + "#B388FF", + "#FF80AB", + "#D81B60", + "#26A69A", + "#FFEA00", + "#6200EA", ]; diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index bc0d12e5c..5d79f404c 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -3,7 +3,7 @@ import { MutableRefObject } from "react"; import { Map } from "maplibre-gl"; import { getBlocksSource } from "./sources"; import { gerryDBView } from "../api/apiHandlers"; -import { color10 } from "./colors"; +import { colorScheme } from "./colors"; export const BLOCK_SOURCE_ID = "blocks"; export const BLOCK_LAYER_ID = "blocks"; @@ -23,10 +23,13 @@ export const COUNTY_LAYER_IDS: string[] = [ export const LABELS_BREAK_LAYER_ID = "places_subplace"; const colorStyleBaseline: any[] = ["case"]; -export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = color10.reduce((val, color, i) => { - val.push(["==", ["feature-state", "zone"], i + 1], color); // 1-indexed per mapStore.ts - return val; -}, colorStyleBaseline); +export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = _colorScheme.reduce( + (val, color, i) => { + val.push(["==", ["feature-state", "zone"], i + 1], color); // 1-indexed per mapStore.ts + return val; + }, + colorStyleBaseline +); ZONE_ASSIGNMENT_STYLE_DYNAMIC.push("#cecece"); // cast the above as an ExpressionSpecification @@ -35,7 +38,7 @@ export const ZONE_ASSIGNMENT_STYLE: ExpressionSpecification = ZONE_ASSIGNMENT_STYLE_DYNAMIC; export function getBlocksLayerSpecification( - sourceLayer: string, + sourceLayer: string ): LayerSpecification { return { id: BLOCK_LAYER_ID, @@ -58,7 +61,7 @@ export function getBlocksLayerSpecification( } export function getBlocksHoverLayerSpecification( - sourceLayer: string, + sourceLayer: string ): LayerSpecification { return { id: BLOCK_HOVER_LAYER_ID, @@ -70,7 +73,7 @@ export function getBlocksHoverLayerSpecification( }, paint: { "fill-opacity": [ - "case", + "case", // zone is selected and hover is true and hover is not null [ "all", @@ -81,7 +84,7 @@ export function getBlocksHoverLayerSpecification( // @ts-ignore ["!", ["==", ["feature-state", "hover"], null]], //< desired behavior but typerror ["boolean", ["feature-state", "hover"], true], - ] + ], ], 0.9, // zone is selected and hover is false, and hover is not null @@ -94,7 +97,7 @@ export function getBlocksHoverLayerSpecification( // @ts-ignore ["!", ["==", ["feature-state", "hover"], null]], //< desired behavior but typerror ["boolean", ["feature-state", "hover"], false], - ] + ], ], 0.7, // zone is selected, fallback, regardless of hover state @@ -102,8 +105,9 @@ export function getBlocksHoverLayerSpecification( ["!", ["==", ["feature-state", "zone"], null]], //< desired behavior but typerror 0.7, // hover is true, fallback, regardless of zone state - ["boolean", ["feature-state", "hover"], false], 0.6, - 0.2 + ["boolean", ["feature-state", "hover"], false], + 0.6, + 0.2, ], "fill-color": ZONE_ASSIGNMENT_STYLE || "#000000", }, @@ -112,18 +116,18 @@ export function getBlocksHoverLayerSpecification( const addBlockLayers = ( map: MutableRefObject, - gerryDBView: gerryDBView, + gerryDBView: gerryDBView ) => { const blockSource = getBlocksSource(gerryDBView.tiles_s3_path); removeBlockLayers(map); map.current?.addSource(BLOCK_SOURCE_ID, blockSource); map.current?.addLayer( getBlocksLayerSpecification(gerryDBView.name), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); map.current?.addLayer( getBlocksHoverLayerSpecification(gerryDBView.name), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); }; From 23c18696426957c22fbeb4c931f5198616d02f64 Mon Sep 17 00:00:00 2001 From: mariogiampieri Date: Thu, 26 Sep 2024 03:45:30 -0400 Subject: [PATCH 05/50] Fix/classic districtr colors (#113) --- app/src/app/constants/layers.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 5d79f404c..65c5668c7 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -23,7 +23,8 @@ export const COUNTY_LAYER_IDS: string[] = [ export const LABELS_BREAK_LAYER_ID = "places_subplace"; const colorStyleBaseline: any[] = ["case"]; -export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = _colorScheme.reduce( + +export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = colorScheme.reduce( (val, color, i) => { val.push(["==", ["feature-state", "zone"], i + 1], color); // 1-indexed per mapStore.ts return val; From 1b1aa5048f16404030431611997a8f8cc3b7f727 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 16 Oct 2024 12:11:21 -0400 Subject: [PATCH 06/50] Shattering (#84) Co-authored-by: nofurtherinformation --- .github/workflows/test-backend.yml | 9 +- app/next.config.mjs | 70 ++-- app/src/app/components/ContextMenu.tsx | 51 +++ app/src/app/components/Map.tsx | 160 ++------- .../components/sidebar/BrushSizeSelector.tsx | 6 +- .../app/components/sidebar/ColorPicker.jsx | 25 +- .../sidebar/GerryDBViewSelector.tsx | 65 ++-- app/src/app/components/sidebar/Layers.tsx | 13 +- .../components/sidebar/MapModeSelector.jsx | 8 +- .../app/components/sidebar/PaintByCounty.tsx | 10 +- app/src/app/components/sidebar/Picker.jsx | 15 +- app/src/app/components/sidebar/Sidebar.tsx | 4 +- .../sidebar/charts/HorizontalBarChart.tsx | 4 +- app/src/app/constants/layers.ts | 104 +++++- app/src/app/constants/types.ts | 6 + app/src/app/map/page.tsx | 5 +- app/src/app/page.tsx | 4 +- app/src/app/store/mapEditSubs.ts | 50 +++ app/src/app/store/mapRenderSubs.ts | 98 ++++++ app/src/app/store/mapStore.ts | 227 ++++++++++--- app/src/app/store/metricsSubs.ts | 17 + app/src/app/{ => utils}/api/apiHandlers.ts | 85 +++-- app/src/app/utils/api/mutations.ts | 66 ++++ app/src/app/utils/api/queries.ts | 51 +++ app/src/app/utils/api/queryClient.ts | 2 + app/src/app/utils/api/queryParamsListener.ts | 22 ++ app/src/app/utils/events/handlers.ts | 9 +- app/src/app/utils/events/mapEvents.ts | 155 +++++---- app/src/app/utils/helpers.ts | 182 ++++++++++- app/tsconfig.json | 24 +- backend/README.md | 136 +++++++- backend/app/alembic/env.py | 4 +- .../09d011c1b387_zones_can_be_null.py | 5 +- .../5ab466c5650a_migrate_to_districtrmap.py | 51 +++ .../versions/5c4028ff26df_update_pop_udf.py | 89 +++++ ...e954087_update_get_total_population_udf.py | 50 ++- .../versions/ccb2a6b81a8b_shattering.py | 119 +++++++ backend/app/main.py | 135 +++++--- backend/app/models.py | 83 ++++- backend/app/sql/create_districtr_map_udf.sql | 27 ++ .../sql/create_shatterable_gerrydb_view.sql | 42 +++ .../app/sql/parent_child_relationships.sql | 53 +++ backend/app/sql/shatter_parent.sql | 50 +++ backend/app/sql/total_pop_udf.sql | 14 +- backend/app/utils.py | 106 ++++++ backend/cli.py | 228 ++++++------- backend/load_data.py | 120 +++++++ backend/pyproject.toml | 4 +- backend/scripts/load_data.py | 66 ---- backend/tests/conftest.py | 72 +++++ backend/tests/constants.py | 25 ++ .../tests/fixtures/simple_child_geos.geojson | 13 + .../tests/fixtures/simple_parent_geos.geojson | 10 + backend/tests/test_main.py | 303 ++++++++---------- backend/tests/test_utils.py | 208 ++++++++++++ docker-compose.yml | 9 +- pipelines/simple_elt/README.md | 6 +- pipelines/simple_elt/files.py | 45 ++- pipelines/simple_elt/main.py | 145 ++++++++- pipelines/simple_elt/requirements.txt | 54 ++++ 60 files changed, 2948 insertions(+), 871 deletions(-) create mode 100644 app/src/app/components/ContextMenu.tsx create mode 100644 app/src/app/store/mapEditSubs.ts create mode 100644 app/src/app/store/mapRenderSubs.ts create mode 100644 app/src/app/store/metricsSubs.ts rename app/src/app/{ => utils}/api/apiHandlers.ts (69%) create mode 100644 app/src/app/utils/api/mutations.ts create mode 100644 app/src/app/utils/api/queries.ts create mode 100644 app/src/app/utils/api/queryClient.ts create mode 100644 app/src/app/utils/api/queryParamsListener.ts create mode 100644 backend/app/alembic/versions/5ab466c5650a_migrate_to_districtrmap.py create mode 100644 backend/app/alembic/versions/5c4028ff26df_update_pop_udf.py create mode 100644 backend/app/alembic/versions/ccb2a6b81a8b_shattering.py create mode 100644 backend/app/sql/create_districtr_map_udf.sql create mode 100644 backend/app/sql/create_shatterable_gerrydb_view.sql create mode 100644 backend/app/sql/parent_child_relationships.sql create mode 100644 backend/app/sql/shatter_parent.sql create mode 100644 backend/app/utils.py create mode 100755 backend/load_data.py delete mode 100755 backend/scripts/load_data.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/fixtures/simple_child_geos.geojson create mode 100644 backend/tests/fixtures/simple_parent_geos.geojson create mode 100644 backend/tests/test_utils.py diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index eb1443118..cb0754e42 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -10,7 +10,7 @@ jobs: container-job: runs-on: ubuntu-latest - container: python:3.12 + container: python:3.12.6 services: postgres: @@ -34,11 +34,6 @@ jobs: - name: Checkout repo code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install GDAL run: | apt-get update @@ -51,7 +46,7 @@ jobs: working-directory: backend - name: Run tests - run: pytest -v --cov=app + run: pytest -v --cov=. working-directory: backend env: DOMAIN: postgres diff --git a/app/next.config.mjs b/app/next.config.mjs index e0d4e14c6..5db6eadb4 100644 --- a/app/next.config.mjs +++ b/app/next.config.mjs @@ -1,4 +1,4 @@ -import {withSentryConfig} from "@sentry/nextjs"; +import { withSentryConfig } from "@sentry/nextjs"; /** @type {import('next').NextConfig} */ const nextConfig = { async redirects() { @@ -14,44 +14,54 @@ const nextConfig = { experimental: { missingSuspenseWithCSRBailout: false, }, + resolve: { + alias: { + "@src": "app/src", + "@components": "app/src/components", + "@utils": "app/src/utils", + "@api": "app/src/api", + "@store": "app/src/store", + "@constants": "app/src/constants", + }, + }, }; export default withSentryConfig(nextConfig, { -// For all available options, see: -// https://github.com/getsentry/sentry-webpack-plugin#options + // For all available options, see: + // https://github.com/getsentry/sentry-webpack-plugin#options -org: "mggg-districtr", -project: "districtr-v2-app", + org: "mggg-districtr", + project: "districtr-v2-app", -// Only print logs for uploading source maps in CI -silent: !process.env.CI, + // Only print logs for uploading source maps in CI + silent: !process.env.CI, -// For all available options, see: -// https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ + // For all available options, see: + // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ -// Upload a larger set of source maps for prettier stack traces (increases build time) -widenClientFileUpload: true, + // Upload a larger set of source maps for prettier stack traces (increases build time) + widenClientFileUpload: true, -// Automatically annotate React components to show their full name in breadcrumbs and session replay -reactComponentAnnotation: { -enabled: true, -}, + // Automatically annotate React components to show their full name in breadcrumbs and session replay + reactComponentAnnotation: { + enabled: true, + }, -// Route browser requests to Sentry through a Next.js rewrite to circumvent ad-blockers. -// This can increase your server load as well as your hosting bill. -// Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client- -// side errors will fail. -tunnelRoute: "/monitoring", + // Route browser requests to Sentry through a Next.js rewrite to circumvent ad-blockers. + // This can increase your server load as well as your hosting bill. + // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client- + // side errors will fail. + tunnelRoute: "/monitoring", -// Hides source maps from generated client bundles -hideSourceMaps: true, + // Hides source maps from generated client bundles + hideSourceMaps: true, -// Automatically tree-shake Sentry logger statements to reduce bundle size -disableLogger: true, + // Automatically tree-shake Sentry logger statements to reduce bundle size + disableLogger: true, -// Enables automatic instrumentation of Vercel Cron Monitors. (Does not yet work with App Router route handlers.) -// See the following for more information: -// https://docs.sentry.io/product/crons/ -// https://vercel.com/docs/cron-jobs -automaticVercelMonitors: true, -}); \ No newline at end of file + // Enables automatic instrumentation of Vercel Cron Monitors. (Does not yet work with App Router route handlers.) + // See the following for more information: + // https://docs.sentry.io/product/crons/ + // https://vercel.com/docs/cron-jobs + automaticVercelMonitors: true, +}); diff --git a/app/src/app/components/ContextMenu.tsx b/app/src/app/components/ContextMenu.tsx new file mode 100644 index 000000000..ed1a04f88 --- /dev/null +++ b/app/src/app/components/ContextMenu.tsx @@ -0,0 +1,51 @@ +import React from "react"; +import { ContextMenu, Text } from "@radix-ui/themes"; +import { useMapStore } from "@/app/store/mapStore"; + +export const MapContextMenu: React.FC = () => { + const mapDocument = useMapStore((state) => state.mapDocument); + const contextMenu = useMapStore((state) => state.contextMenu); + const handleShatter = useMapStore((state) => state.handleShatter); + if (!contextMenu) return null; + + const handleSelect = () => { + if (!mapDocument || contextMenu?.data?.id === undefined) return; + handleShatter(mapDocument.document_id, [contextMenu.data.id.toString()]); + contextMenu.close(); + }; + + return ( + + + {contextMenu.data.id && ( + + + {contextMenu.data.id} + + + )} + + Shatter + + + + ); +}; diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index 700a75a43..552c3cc55 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -6,75 +6,20 @@ import maplibregl, { import "maplibre-gl/dist/maplibre-gl.css"; import { Protocol } from "pmtiles"; import type { MutableRefObject } from "react"; -import React, { useEffect, useRef, useState } from "react"; +import React, { useEffect, useRef } from "react"; import { MAP_OPTIONS } from "../constants/configuration"; import { mapEvents, - useHoverFeatureIds, - handleResetMapSelectState, } from "../utils/events/mapEvents"; -import { BLOCK_HOVER_LAYER_ID, BLOCK_SOURCE_ID } from "../constants/layers"; -import { useSearchParams } from "next/navigation"; +import { INTERACTIVE_LAYERS } from "../constants/layers"; import { useMapStore } from "../store/mapStore"; -import { - FormatAssignments, - getDocument, - DocumentObject, - patchUpdateAssignments, - AssignmentsCreate, - getAssignments, - Assignment, - getZonePopulations, -} from "../api/apiHandlers"; -import { useMutation, useQuery, skipToken } from "@tanstack/react-query"; export const MapComponent: React.FC = () => { - const searchParams = useSearchParams(); const map: MutableRefObject = useRef(null); const mapContainer: MutableRefObject = useRef(null); - const [mapLoaded, setMapLoaded] = useState(false); - const hoverFeatureIds = useHoverFeatureIds(); - - const patchUpdates = useMutation({ - mutationFn: patchUpdateAssignments, - onMutate: () => { - console.log("Updating assignments"); - }, - onError: (error) => { - console.log("Error updating assignments: ", error); - }, - onSuccess: (data: AssignmentsCreate) => { - console.log( - `Successfully upserted ${data.assignments_upserted} assignments`, - ); - mapMetrics.refetch(); - }, - }); + const mapLock = useMapStore((state) => state.mapLock); - const { - activeTool, - freshMap, - zoneAssignments, - mapDocument, - setMapDocument, - setSelectedLayer, - setMapRef, - setMapMetrics, - } = useMapStore((state) => ({ - activeTool: state.activeTool, - freshMap: state.freshMap, - zoneAssignments: state.zoneAssignments, - mapDocument: state.mapDocument, - setMapDocument: state.setMapDocument, - setSelectedLayer: state.setSelectedLayer, - setMapRef: state.setMapRef, - setMapMetrics: state.setMapMetrics, - })); - - const mapMetrics = useQuery({ - queryKey: ["zonePopulations", mapDocument], - queryFn: mapDocument ? () => getZonePopulations(mapDocument) : skipToken, - }); + const setMapRef = useMapStore((state) => state.setMapRef); useEffect(() => { let protocol = new Protocol(); @@ -84,19 +29,6 @@ export const MapComponent: React.FC = () => { }; }, []); - useEffect(() => { - const document_id = searchParams.get("document_id"); - if (document_id && !useMapStore.getState().mapDocument) { - getDocument(document_id).then((res: DocumentObject) => { - setMapDocument(res); - }); - } - }, [searchParams, setMapDocument]); - - useEffect(() => { - setMapMetrics(mapMetrics); - }, [mapMetrics.data]); - useEffect(() => { if (map.current || !mapContainer.current) return; @@ -113,79 +45,37 @@ export const MapComponent: React.FC = () => { map.current.addControl(new maplibregl.NavigationControl()); map.current.on("load", () => { - setMapLoaded(true); setMapRef(map); - const mapDocument = useMapStore.getState().mapDocument; - - if (mapDocument?.tiles_s3_path) { - setSelectedLayer({ - name: mapDocument.gerrydb_table, - tiles_s3_path: mapDocument.tiles_s3_path, - }); - } - - if (mapDocument) { - console.log("fetching assignments"); - const sourceLayer = mapDocument.gerrydb_table; - getAssignments(mapDocument).then((res: Assignment[]) => { - console.log("got", res.length, "assignments"); - mapMetrics.refetch(); - res.forEach((assignment) => { - zoneAssignments.set(assignment.geo_id, assignment.zone); - map.current?.setFeatureState( - { - source: BLOCK_SOURCE_ID, - id: assignment.geo_id, - sourceLayer: sourceLayer, - }, - { - selected: true, - zone: assignment.zone, - }, - ); - }); - }); - } }); - - mapEvents.forEach((action) => { - if (map.current) { - map.current?.on( - action.action as keyof MapLayerEventType, - BLOCK_HOVER_LAYER_ID, // to be updated with the scale-agnostic layer id - (e: MapLayerMouseEvent | MapLayerTouchEvent) => { - action.handler(e, map, hoverFeatureIds); - }, - ); - } + INTERACTIVE_LAYERS.forEach((layer) => { + mapEvents.forEach((action) => { + if (map.current) { + map.current?.on( + action.action as keyof MapLayerEventType, + layer, // to be updated with the scale-agnostic layer id + (e: MapLayerMouseEvent | MapLayerTouchEvent) => { + action.handler(e, map); + } + ); + } + }); }); return () => { mapEvents.forEach((action) => { map.current?.off(action.action, (e) => { - action.handler(e, map, hoverFeatureIds); + action.handler(e, map); }); }); }; }); - /** - * send assignments to the server when zones change. - */ - useEffect(() => { - if (mapLoaded && map.current && zoneAssignments.size) { - if (activeTool === "brush" || activeTool === "eraser") { - const assignments = FormatAssignments(); - patchUpdates.mutate(assignments); - } - } - }, [mapLoaded, zoneAssignments]); - - useEffect(() => { - if (mapLoaded && map.current) { - handleResetMapSelectState(map); - } - }, [mapLoaded, freshMap]); - - return
; + return ( +
+ ); }; diff --git a/app/src/app/components/sidebar/BrushSizeSelector.tsx b/app/src/app/components/sidebar/BrushSizeSelector.tsx index 948b5fab9..e6efa0ebf 100644 --- a/app/src/app/components/sidebar/BrushSizeSelector.tsx +++ b/app/src/app/components/sidebar/BrushSizeSelector.tsx @@ -11,10 +11,8 @@ import { useMapStore } from "../../store/mapStore"; * @returns {JSX.Element} The component */ export function BrushSizeSelector() { - const { brushSize, setBrushSize } = useMapStore((state) => ({ - brushSize: state.brushSize, - setBrushSize: state.setBrushSize, - })); + const brushSize = useMapStore((state) => state.brushSize); + const setBrushSize = useMapStore((state) => state.setBrushSize); const handleChangeEnd = (value: Array) => { console.log("the final value size is", value); diff --git a/app/src/app/components/sidebar/ColorPicker.jsx b/app/src/app/components/sidebar/ColorPicker.jsx index a9c1ca7ac..ba3ef7c82 100644 --- a/app/src/app/components/sidebar/ColorPicker.jsx +++ b/app/src/app/components/sidebar/ColorPicker.jsx @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { _colorScheme } from "../../constants/colors"; +import { _colorScheme, colorScheme } from "../../constants/colors"; import { Button } from "@radix-ui/themes"; import { styled } from "@stitches/react"; import * as RadioGroup from "@radix-ui/react-radio-group"; @@ -7,22 +7,13 @@ import { blackA } from "@radix-ui/colors"; import { useMapStore } from "../../store/mapStore"; export function ColorPicker() { - const [color, setColor] = useState(null); - const [open, setOpen] = useState(false); - const { - selectedZone, - setSelectedZone, - setZoneAssignments, - accumulatedGeoids, - resetAccumulatedBlockPopulations, - } = useMapStore((state) => ({ - selectedZone: state.selectedZone, - setSelectedZone: state.setSelectedZone, - setZoneAssignments: state.setZoneAssignments, - accumulatedGeoids: state.accumulatedGeoids, - resetAccumulatedBlockPopulations: state.resetAccumulatedBlockPopulations, - })); - const colorArray = _colorScheme; + const selectedZone = useMapStore((state) => state.selectedZone); + const setSelectedZone = useMapStore((state) => state.setSelectedZone); + const setZoneAssignments = useMapStore((state) => state.setZoneAssignments); + const accumulatedGeoids = useMapStore((state) => state.accumulatedGeoids); + const resetAccumulatedBlockPopulations = useMapStore((state) => state.resetAccumulatedBlockPopulations); + + const colorArray = colorScheme; if (!colorArray) return null; const handleRadioChange = (value) => { console.log( diff --git a/app/src/app/components/sidebar/GerryDBViewSelector.tsx b/app/src/app/components/sidebar/GerryDBViewSelector.tsx index d7101c759..0cfc2520e 100644 --- a/app/src/app/components/sidebar/GerryDBViewSelector.tsx +++ b/app/src/app/components/sidebar/GerryDBViewSelector.tsx @@ -1,47 +1,37 @@ -import { useEffect, useState, useCallback } from "react"; +import { useState } from "react"; import { Select } from "@radix-ui/themes"; -import { getGerryDBViews } from "../../api/apiHandlers"; +import { getAvailableDistrictrMaps } from "../../utils/api/apiHandlers"; import { useMapStore } from "../../store/mapStore"; -import { createMapDocument } from "../../api/apiHandlers"; -import { useQuery, useMutation } from "@tanstack/react-query"; -import { useRouter, usePathname, useSearchParams } from "next/navigation"; +import { useQuery } from "@tanstack/react-query"; +import { document } from "@/app/utils/api/mutations"; export function GerryDBViewSelector() { - const router = useRouter(); - const pathname = usePathname(); - const searchParams = useSearchParams(); - const [limit, setLimit] = useState(20); + const [limit, setLimit] = useState(30); const [offset, setOffset] = useState(0); - const { selectedLayer, setMapDocument } = useMapStore((state) => ({ - selectedLayer: state.selectedLayer, - setMapDocument: state.setMapDocument, - })); - const document = useMutation({ - mutationFn: createMapDocument, - onMutate: () => { - console.log("Creating document"); - }, - onError: (error) => { - console.error("Error creating map document: ", error); - }, - onSuccess: (data) => { - setMapDocument(data); - const urlParams = new URLSearchParams(searchParams.toString()); - urlParams.set("document_id", data.document_id); - router.push(pathname + "?" + urlParams.toString()); - }, - }); + const mapDocument = useMapStore((state) => state.mapDocument); + const { isPending, isError, data, error } = useQuery({ queryKey: ["views", limit, offset], - queryFn: () => getGerryDBViews(limit, offset), + queryFn: () => getAvailableDistrictrMaps(limit, offset), }); + + const selectedView = data?.find( + (view) => view.gerrydb_table_name === mapDocument?.gerrydb_table, + ); const handleValueChange = (value: string) => { - const selectedLayer = data?.find((view) => view.name === value); - if (!selectedLayer || selectedLayer.name === document.data?.gerrydb_table) { + console.log("Value changed: ", value); + const selectedDistrictrMap = data?.find((view) => view.name === value); + console.log("Selected view: ", selectedDistrictrMap); + if ( + !selectedDistrictrMap || + selectedDistrictrMap.gerrydb_table_name === mapDocument?.gerrydb_table + ) { + console.log("No document or same document"); return; } - document.mutate({ gerrydb_table: selectedLayer.name }); + console.log("mutating to create new document"); + document.mutate({ gerrydb_table: selectedDistrictrMap.gerrydb_table_name }); }; if (isPending) return
Loading geographies... 🌎
; @@ -49,16 +39,11 @@ export function GerryDBViewSelector() { if (isError) return
Error loading geographies: {error.message}
; return ( - - + + - Select a geography + Districtr map options {data.map((view, index) => ( {view.name} diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index 9e00bf468..9a5afe981 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -16,13 +16,10 @@ import { toggleLayerVisibility } from "../../utils/helpers"; * - Support tribes and communities */ export default function Layers() { - const { mapRef, selectedLayer, visibleLayerIds, updateVisibleLayerIds } = - useMapStore((state) => ({ - mapRef: state.mapRef, - selectedLayer: state.selectedLayer, - visibleLayerIds: state.visibleLayerIds, - updateVisibleLayerIds: state.updateVisibleLayerIds, - })); + const mapRef = useMapStore((state) => state.mapRef); + const mapDocument = useMapStore((state) => state.mapDocument); + const visibleLayerIds = useMapStore((state) => state.visibleLayerIds); + const updateVisibleLayerIds = useMapStore((state) => state.updateVisibleLayerIds); const toggleLayers = (layerIds: string[]) => { if (!mapRef || !mapRef?.current) return; @@ -43,7 +40,7 @@ export default function Layers() { toggleLayers([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID])} - disabled={selectedLayer === null} + disabled={mapDocument === null} > Show painted districts diff --git a/app/src/app/components/sidebar/MapModeSelector.jsx b/app/src/app/components/sidebar/MapModeSelector.jsx index e2f9ddc88..5d6ff1130 100644 --- a/app/src/app/components/sidebar/MapModeSelector.jsx +++ b/app/src/app/components/sidebar/MapModeSelector.jsx @@ -8,11 +8,9 @@ import { EraserIcon, Pencil2Icon, HandIcon } from "@radix-ui/react-icons"; export function MapModeSelector() { const mapStore = useMapStore.getState(); - const { activeTool, setActiveTool } = useMapStore((state) => ({ - activeTool: state.activeTool, - setActiveTool: state.setActiveTool, - })); - + const activeTool = useMapStore((state) => state.activeTool); + const setActiveTool = useMapStore((state) => state.setActiveTool); + if (!activeTool) return null; const activeTools = [ { mode: "pan", disabled: false, label: "Pan", icon: }, diff --git a/app/src/app/components/sidebar/PaintByCounty.tsx b/app/src/app/components/sidebar/PaintByCounty.tsx index 1aa1c8dbb..f578118f9 100644 --- a/app/src/app/components/sidebar/PaintByCounty.tsx +++ b/app/src/app/components/sidebar/PaintByCounty.tsx @@ -8,13 +8,9 @@ import { } from "../../utils/helpers"; export default function PaintByCounty() { - const { mapRef, addVisibleLayerIds, setPaintFunction } = useMapStore( - (state) => ({ - mapRef: state.mapRef, - addVisibleLayerIds: state.addVisibleLayerIds, - setPaintFunction: state.setPaintFunction, - }), - ); + const mapRef = useMapStore((state) => state.mapRef); + const addVisibleLayerIds = useMapStore((state) => state.addVisibleLayerIds); + const setPaintFunction = useMapStore((state) => state.setPaintFunction); const [checked, setChecked] = useState(false); useEffect(() => { diff --git a/app/src/app/components/sidebar/Picker.jsx b/app/src/app/components/sidebar/Picker.jsx index 9282d8b62..d459866cb 100644 --- a/app/src/app/components/sidebar/Picker.jsx +++ b/app/src/app/components/sidebar/Picker.jsx @@ -3,17 +3,10 @@ import { Select } from "@radix-ui/themes"; import { useMapStore } from "../../store/mapStore"; export function ZoneTypeSelector() { - const { - selectedZone, - setSelectedZone, - setZoneAssignments, - accumulatedGeoids, - } = useMapStore((state) => ({ - selectedZone: state.selectedZone, - setSelectedZone: state.setSelectedZone, - setZoneAssignments: state.setZoneAssignments, - accumulatedGeoids: state.accumulatedGeoids, - })); + const selectedZone = useMapStore((state) => state.selectedZone); + const setSelectedZone = useMapStore((state) => state.setSelectedZone); + const setZoneAssignments = useMapStore((state) => state.setZoneAssignments); + const accumulatedGeoids = useMapStore((state) => state.accumulatedGeoids); const handlePickerValueChange = (value) => { console.log( diff --git a/app/src/app/components/sidebar/Sidebar.tsx b/app/src/app/components/sidebar/Sidebar.tsx index 17d00cd41..1c1aed307 100644 --- a/app/src/app/components/sidebar/Sidebar.tsx +++ b/app/src/app/components/sidebar/Sidebar.tsx @@ -11,9 +11,7 @@ import PaintByCounty from "./PaintByCounty"; import { BrushSizeSelector } from "./BrushSizeSelector"; export default function SidebarComponent() { - const { activeTool } = useMapStore((state) => ({ - activeTool: state.activeTool, - })); + const activeTool = useMapStore((state) => state.activeTool); return ( { }; export const HorizontalBar = () => { - const { mapMetrics } = useMapStore((state) => ({ - mapMetrics: state.mapMetrics, - })); + const mapMetrics = useMapStore((state) => state.mapMetrics); if (mapMetrics?.isPending) { return
Loading...
; diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 65c5668c7..508a3a9fc 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -1,13 +1,36 @@ -import { ExpressionSpecification, LayerSpecification } from "maplibre-gl"; +import { + ExpressionSpecification, + FilterSpecification, + LayerSpecification, +} from "maplibre-gl"; import { MutableRefObject } from "react"; import { Map } from "maplibre-gl"; import { getBlocksSource } from "./sources"; -import { gerryDBView } from "../api/apiHandlers"; +import { DocumentObject } from "../utils/api/apiHandlers"; +import { MapStore, useMapStore } from "../store/mapStore"; import { colorScheme } from "./colors"; export const BLOCK_SOURCE_ID = "blocks"; export const BLOCK_LAYER_ID = "blocks"; +export const BLOCK_LAYER_ID_CHILD = "blocks-child"; export const BLOCK_HOVER_LAYER_ID = `${BLOCK_LAYER_ID}-hover`; +export const BLOCK_HOVER_LAYER_ID_CHILD = `${BLOCK_LAYER_ID_CHILD}-hover`; + +export const INTERACTIVE_LAYERS = [ + BLOCK_HOVER_LAYER_ID, + BLOCK_HOVER_LAYER_ID_CHILD, +] + +export const PARENT_LAYERS = [ + BLOCK_LAYER_ID, + BLOCK_HOVER_LAYER_ID +] + +export const CHILD_LAYERS = [ + BLOCK_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD +] + export const DEFAULT_PAINT_STYLE: ExpressionSpecification = [ "case", ["boolean", ["feature-state", "hover"], false], @@ -38,17 +61,41 @@ ZONE_ASSIGNMENT_STYLE_DYNAMIC.push("#cecece"); export const ZONE_ASSIGNMENT_STYLE: ExpressionSpecification = ZONE_ASSIGNMENT_STYLE_DYNAMIC; +export function getLayerFilter( + layerId: string, + _shatterIds?: MapStore["shatterIds"] +) { + const shatterIds = _shatterIds || useMapStore.getState().shatterIds; + const isChildLayer = CHILD_LAYERS.includes(layerId); + const ids = isChildLayer ? shatterIds.children : shatterIds.parents; + const cleanIds = Boolean(ids) ? Array.from(ids) : []; + const filterBase: FilterSpecification = [ + "in", + ["get", "path"], + ["literal", cleanIds], + ]; + + if (isChildLayer) { + return filterBase; + } + const parentFilter: FilterSpecification = ["!", filterBase]; + return parentFilter; +} + export function getBlocksLayerSpecification( - sourceLayer: string + sourceLayer: string, + layerId: string, ): LayerSpecification { + const shatterIds = useMapStore.getState().shatterIds; return { - id: BLOCK_LAYER_ID, + id: layerId, source: BLOCK_SOURCE_ID, "source-layer": sourceLayer, type: "line", layout: { visibility: "visible", }, + filter: getLayerFilter(layerId), paint: { "line-opacity": [ "case", @@ -62,16 +109,18 @@ export function getBlocksLayerSpecification( } export function getBlocksHoverLayerSpecification( - sourceLayer: string + sourceLayer: string, + layerId: string, ): LayerSpecification { return { - id: BLOCK_HOVER_LAYER_ID, + id: layerId, source: BLOCK_SOURCE_ID, "source-layer": sourceLayer, type: "fill", layout: { visibility: "visible", }, + filter: getLayerFilter(layerId), paint: { "fill-opacity": [ "case", @@ -117,28 +166,59 @@ export function getBlocksHoverLayerSpecification( const addBlockLayers = ( map: MutableRefObject, - gerryDBView: gerryDBView + mapDocument: DocumentObject, ) => { - const blockSource = getBlocksSource(gerryDBView.tiles_s3_path); + if (!map.current || !mapDocument.tiles_s3_path) { + console.log("map or mapDocument not ready", mapDocument); + return; + } + const blockSource = getBlocksSource(mapDocument.tiles_s3_path); removeBlockLayers(map); map.current?.addSource(BLOCK_SOURCE_ID, blockSource); map.current?.addLayer( - getBlocksLayerSpecification(gerryDBView.name), - LABELS_BREAK_LAYER_ID + getBlocksLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID), + LABELS_BREAK_LAYER_ID, ); map.current?.addLayer( - getBlocksHoverLayerSpecification(gerryDBView.name), - LABELS_BREAK_LAYER_ID + getBlocksHoverLayerSpecification( + mapDocument.parent_layer, + BLOCK_HOVER_LAYER_ID, + ), + LABELS_BREAK_LAYER_ID, ); + if (mapDocument.child_layer) { + map.current?.addLayer( + getBlocksLayerSpecification( + mapDocument.child_layer, + BLOCK_LAYER_ID_CHILD, + ), + LABELS_BREAK_LAYER_ID, + ); + map.current?.addLayer( + getBlocksHoverLayerSpecification( + mapDocument.child_layer, + BLOCK_HOVER_LAYER_ID_CHILD, + ), + LABELS_BREAK_LAYER_ID, + ); + } + useMapStore.getState().setMapRenderingState("loaded") }; export function removeBlockLayers(map: MutableRefObject) { + useMapStore.getState().setMapRenderingState("loading") if (map.current?.getLayer(BLOCK_LAYER_ID)) { map.current?.removeLayer(BLOCK_LAYER_ID); } if (map.current?.getLayer(BLOCK_HOVER_LAYER_ID)) { map.current?.removeLayer(BLOCK_HOVER_LAYER_ID); } + if (map.current?.getLayer(BLOCK_LAYER_ID_CHILD)) { + map.current?.removeLayer(BLOCK_LAYER_ID_CHILD); + } + if (map.current?.getLayer(BLOCK_HOVER_LAYER_ID_CHILD)) { + map.current?.removeLayer(BLOCK_HOVER_LAYER_ID_CHILD); + } if (map.current?.getSource(BLOCK_SOURCE_ID)) { map.current?.removeSource(BLOCK_SOURCE_ID); } diff --git a/app/src/app/constants/types.ts b/app/src/app/constants/types.ts index b5c74534c..c1b70628a 100644 --- a/app/src/app/constants/types.ts +++ b/app/src/app/constants/types.ts @@ -36,3 +36,9 @@ export type ViewStateChangeEvent = | "pitchend"; viewState: MapOptions; }); + +export type MapFeatureInfo = { + source: string; + sourceLayer?: string; + id?: string | number; +}; diff --git a/app/src/app/map/page.tsx b/app/src/app/map/page.tsx index cbfeeffe4..916ac5d76 100644 --- a/app/src/app/map/page.tsx +++ b/app/src/app/map/page.tsx @@ -1,12 +1,12 @@ "use client"; import React from "react"; +import { MapContextMenu } from "../components/ContextMenu"; import { MapComponent } from "../components/Map"; import SidebarComponent from "../components/sidebar/Sidebar"; -import { QueryClient } from "@tanstack/react-query"; import { QueryClientProvider } from "@tanstack/react-query"; +import { queryClient } from "../utils/api/queryClient"; export default function Map() { - const queryClient = new QueryClient(); if (queryClient) { return ( @@ -14,6 +14,7 @@ export default function Map() {
+
); diff --git a/app/src/app/page.tsx b/app/src/app/page.tsx index 19c722a86..1f1f92158 100644 --- a/app/src/app/page.tsx +++ b/app/src/app/page.tsx @@ -1,9 +1,9 @@ "use client"; import { MapComponent } from "./components/Map"; import SidebarComponent from "./components/sidebar/Sidebar"; -import { QueryClientProvider, QueryClient } from "@tanstack/react-query"; +import { QueryClientProvider } from "@tanstack/react-query"; +import { queryClient } from "./utils/api/queryClient"; -const queryClient = new QueryClient(); export default function Home() { return ( diff --git a/app/src/app/store/mapEditSubs.ts b/app/src/app/store/mapEditSubs.ts new file mode 100644 index 000000000..e91fcff43 --- /dev/null +++ b/app/src/app/store/mapEditSubs.ts @@ -0,0 +1,50 @@ +import { debounce } from "lodash"; +import { + Assignment, + FormatAssignments, + getAssignments, +} from "../utils/api/apiHandlers"; +import { patchUpdates } from "../utils/api/mutations"; +import { useMapStore as _useMapStore, MapStore } from "./mapStore"; +import { shallowCompareArray } from "../utils/helpers"; + +const zoneUpdates = ({ + mapRef, + zoneAssignments, + appLoadingState, +}: Partial) => { + if ( + mapRef?.current && + zoneAssignments?.size && + appLoadingState === "loaded" + ) { + const assignments = FormatAssignments(); + patchUpdates.mutate(assignments); + } +}; +const debouncedZoneUpdate = debounce(zoneUpdates, 25); + +export const getMapEditSubs = (useMapStore: typeof _useMapStore) => { + const sendZonesOnMapRefSub = useMapStore.subscribe( + (state) => [state.mapRef, state.zoneAssignments], + () => { + const { mapRef, zoneAssignments, appLoadingState } = + useMapStore.getState(); + debouncedZoneUpdate({ mapRef, zoneAssignments, appLoadingState }); + }, + { equalityFn: shallowCompareArray} + ); + + const fetchAssignmentsSub = useMapStore.subscribe( + (state) => state.mapDocument, + (mapDocument) => { + if (mapDocument) { + getAssignments(mapDocument).then((res: Assignment[]) => { + useMapStore.getState().loadZoneAssignments(res); + }); + } + } + ); + + return [sendZonesOnMapRefSub, fetchAssignmentsSub]; +}; diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts new file mode 100644 index 000000000..caff6a2a7 --- /dev/null +++ b/app/src/app/store/mapRenderSubs.ts @@ -0,0 +1,98 @@ +import { + addBlockLayers, + BLOCK_LAYER_ID, + BLOCK_HOVER_LAYER_ID, + PARENT_LAYERS, + CHILD_LAYERS, + getLayerFilter, +} from "../constants/layers"; +import { + ColorZoneAssignmentsState, + colorZoneAssignments, + shallowCompareArray, +} from "../utils/helpers"; +import { useMapStore as _useMapStore, MapStore } from "./mapStore"; + +export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { + const addLayerSubMapDocument = useMapStore.subscribe< + [MapStore["mapDocument"], MapStore["mapRef"]] + >( + (state) => [state.mapDocument, state.mapRef], + ([mapDocument, mapRef]) => { + const mapStore = useMapStore.getState(); + if (mapRef?.current && mapDocument) { + addBlockLayers(mapRef, mapDocument); + mapStore.addVisibleLayerIds([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]); + } + }, + { equalityFn: shallowCompareArray }, + ); + + const _shatterMapSideEffectRender = useMapStore.subscribe< + [MapStore["shatterIds"], MapStore["mapRef"], MapStore["mapRenderingState"]] + >( + (state) => [state.shatterIds, state.mapRef, state.mapRenderingState], + ([shatterIds, mapRef, mapRenderingState]) => { + const state = useMapStore.getState(); + const setMapLock = state.setMapLock; + + if (!mapRef?.current || mapRenderingState !== "loaded") { + return; + } + + const layersToFilter = PARENT_LAYERS; + + if (state.mapDocument?.child_layer) layersToFilter.push(...CHILD_LAYERS); + + layersToFilter.forEach((layerId) => + mapRef.current?.setFilter(layerId, getLayerFilter(layerId, shatterIds)), + ); + + mapRef.current.once("render", () => { + setMapLock(false); + console.log(`Unlocked at`, performance.now()); + }); + }, + { equalityFn: shallowCompareArray }, + ); + + const _hoverMapSideEffectRender = useMapStore.subscribe( + (state) => state.hoverFeatures, + (hoverFeatures, previousHoverFeatures) => { + const mapRef = useMapStore.getState().mapRef; + + if (!mapRef?.current) { + return; + } + + previousHoverFeatures.forEach((feature) => { + mapRef.current?.setFeatureState(feature, { hover: false }); + }); + + hoverFeatures.forEach((feature) => { + mapRef.current?.setFeatureState(feature, { hover: true }); + }); + }, + ); + + const _zoneAssignmentMapSideEffectRender = + useMapStore.subscribe( + (state) => [ + state.zoneAssignments, + state.mapDocument, + state.mapRef, + state.shatterIds, + state.appLoadingState, + state.mapRenderingState, + ], + (curr, prev) => colorZoneAssignments(curr, prev), + { equalityFn: shallowCompareArray }, + ); + + return [ + addLayerSubMapDocument, + _shatterMapSideEffectRender, + _hoverMapSideEffectRender, + _zoneAssignmentMapSideEffectRender, + ]; +}; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index da4c5f0f2..bd281670e 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -1,34 +1,58 @@ -import type { MapOptions } from "maplibre-gl"; +import type { MapGeoJSONFeature, MapOptions } from "maplibre-gl"; import { create } from "zustand"; import { subscribeWithSelector } from "zustand/middleware"; -import type { ActiveTool, SpatialUnit } from "../constants/types"; +import type { + ActiveTool, + MapFeatureInfo, + SpatialUnit, +} from "../constants/types"; import { Zone, GDBPath } from "../constants/types"; import { - gerryDBView, + Assignment, DocumentObject, ZonePopulation, -} from "../api/apiHandlers"; +} from "../utils/api/apiHandlers"; import maplibregl from "maplibre-gl"; import type { MutableRefObject } from "react"; +import { UseQueryResult } from "@tanstack/react-query"; import { - addBlockLayers, - BLOCK_LAYER_ID, - BLOCK_HOVER_LAYER_ID, -} from "../constants/layers"; -import type { UseQueryResult } from "@tanstack/react-query"; -import { + ContextMenuState, LayerVisibility, PaintEventHandler, getFeaturesInBbox, + setZones, } from "../utils/helpers"; +import { getRenderSubscriptions } from "./mapRenderSubs"; +import { patchShatter } from "../utils/api/mutations"; +import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; +import { getMapMetricsSubs } from "./metricsSubs"; +import { getMapEditSubs } from "./mapEditSubs"; export interface MapStore { + appLoadingState: "loaded" | "initializing" | "loading"; + setAppLoadingState: (state: MapStore["appLoadingState"]) => void; + mapRenderingState: "loaded" | "initializing" | "loading"; + setMapRenderingState: (state: MapStore["mapRenderingState"]) => void; mapRef: MutableRefObject | null; setMapRef: (map: MutableRefObject) => void; + mapLock: boolean; + setMapLock: (lock: boolean) => void; mapDocument: DocumentObject | null; setMapDocument: (mapDocument: DocumentObject) => void; - selectedLayer: gerryDBView | null; - setSelectedLayer: (layer: gerryDBView) => void; + shatterIds: { + parents: Set; + children: Set; + }; + setShatterIds: ( + existingParents: Set, + existingChildren: Set, + newParent: string[], + newChildren: Set[], + multipleShattered: boolean + ) => void; + handleShatter: (document_id: string, geoids: string[]) => void; + hoverFeatures: Array; + setHoverFeatures: (features?: Array) => void; mapOptions: MapOptions; setMapOptions: (options: MapOptions) => void; activeTool: ActiveTool; @@ -41,6 +65,7 @@ export interface MapStore { resetAccumulatedBlockPopulations: () => void; zoneAssignments: Map; // geoid -> zone setZoneAssignments: (zone: Zone, gdbPaths: Set) => void; + loadZoneAssignments: (assigments: Assignment[]) => void; resetZoneAssignments: () => void; zonePopulations: Map; setZonePopulations: (zone: Zone, population: number) => void; @@ -56,33 +81,131 @@ export interface MapStore { setFreshMap: (resetMap: boolean) => void; mapMetrics: UseQueryResult | null; setMapMetrics: ( - metrics: UseQueryResult | null, + metrics: UseQueryResult | null ) => void; visibleLayerIds: string[]; setVisibleLayerIds: (layerIds: string[]) => void; addVisibleLayerIds: (layerIds: string[]) => void; updateVisibleLayerIds: (layerIds: LayerVisibility[]) => void; + contextMenu: ContextMenuState | null; + setContextMenu: (menu: ContextMenuState | null) => void; } +const initialLoadingState = typeof window !== 'undefined' && new URLSearchParams(window.location.search).has( + "document_id" +) + ? "loading" + : "initializing"; + export const useMapStore = create( - subscribeWithSelector((set) => ({ + subscribeWithSelector((set, get) => ({ + appLoadingState: initialLoadingState, + setAppLoadingState: (appLoadingState) => set({ appLoadingState }), + mapRenderingState: "initializing", + setMapRenderingState: (mapRenderingState) => set({ mapRenderingState }), mapRef: null, - setMapRef: (mapRef) => set({ mapRef }), + setMapRef: (mapRef) => + set({ + mapRef, + appLoadingState: + initialLoadingState === "initializing" + ? "loaded" + : get().appLoadingState, + }), + mapLock: false, + setMapLock: (mapLock) => set({ mapLock }), mapDocument: null, setMapDocument: (mapDocument) => set((state) => { - if (mapDocument.tiles_s3_path) { - state.setSelectedLayer({ - name: mapDocument.gerrydb_table, - tiles_s3_path: mapDocument.tiles_s3_path, - }); - } state.setFreshMap(true); state.resetZoneAssignments(); - return { mapDocument: mapDocument }; + return { + mapDocument: mapDocument, + shatterIds: { parents: new Set(), children: new Set() }, + }; }), - selectedLayer: null, - setSelectedLayer: (layer) => set({ selectedLayer: layer }), + shatterIds: { + parents: new Set(), + children: new Set(), + }, + handleShatter: async (document_id, geoids) => { + set({ mapLock: true }); + const shatterResult = await patchShatter.mutate({ + document_id, + geoids, + }); + + const zoneAssignments = new Map(get().zoneAssignments); + const shatterIds = get().shatterIds; + + let existingParents = new Set(shatterIds.parents); + let existingChildren = new Set(shatterIds.children); + + const newParent = shatterResult.parents.geoids; + const newChildren = new Set( + shatterResult.children.map((child) => child.geo_id) + ); + + const multipleShattered = shatterResult.parents.geoids.length > 1; + if (!multipleShattered) { + setZones(zoneAssignments, newParent[0], newChildren); + } else { + // todo handle multiple shattered case + } + newParent.forEach((parent) => existingParents.add(parent)); + // there may be a faster way to do this + [newChildren].forEach( + (children) => existingChildren = new Set([...existingChildren, ...children]) + ) + + set({ + shatterIds: { + parents: existingParents, + children: existingChildren, + }, + zoneAssignments, + }); + }, + setShatterIds: ( + existingParents, + existingChildren, + newParent, + newChildren, + multipleShattered + ) => { + const zoneAssignments = new Map(get().zoneAssignments); + + if (!multipleShattered) { + setZones(zoneAssignments, newParent[0], newChildren[0]); + } else { + // todo handle multiple shattered case + } + newParent.forEach((parent) => existingParents.add(parent)); + // there may be a faster way to do this + newChildren.forEach( + (children) => existingChildren = new Set([...existingChildren, ...children]) + ); + + set({ + shatterIds: { + parents: existingParents, + children: existingChildren, + }, + zoneAssignments, + }); + }, + hoverFeatures: [], + setHoverFeatures: (_features) => { + const hoverFeatures = _features + ? _features.map((f) => ({ + source: f.source, + sourceLayer: f.sourceLayer, + id: f.id, + })) + : []; + + set({ hoverFeatures }); + }, mapOptions: { center: [-98.5795, 39.8283], zoom: 3, @@ -99,17 +222,32 @@ export const useMapStore = create( setSelectedZone: (zone) => set({ selectedZone: zone }), zoneAssignments: new Map(), accumulatedGeoids: new Set(), - setZoneAssignments: (zone, geoids) => - set((state) => { - const newZoneAssignments = new Map(state.zoneAssignments); - geoids.forEach((geoid) => { - newZoneAssignments.set(geoid, zone); - }); - return { - zoneAssignments: newZoneAssignments, - accumulatedGeoids: new Set(), - }; - }), + setZoneAssignments: (zone, geoids) => { + const zoneAssignments = get().zoneAssignments; + const newZoneAssignments = new Map(zoneAssignments); + geoids.forEach((geoid) => { + newZoneAssignments.set(geoid, zone); + }); + set({ + zoneAssignments: newZoneAssignments, + accumulatedGeoids: new Set(), + }); + }, + loadZoneAssignments: (assignments) => { + const zoneAssignments = new Map(); + const shatterIds = { + parents: new Set(), + children: new Set(), + }; + assignments.forEach((assignment) => { + zoneAssignments.set(assignment.geo_id, assignment.zone); + if (assignment.parent_path) { + shatterIds.parents.add(assignment.parent_path); + shatterIds.children.add(assignment.geo_id); + } + }); + set({ zoneAssignments, shatterIds, appLoadingState: "loaded" }); + }, accumulatedBlockPopulations: new Map(), resetAccumulatedBlockPopulations: () => set({ accumulatedBlockPopulations: new Map() }), @@ -163,16 +301,13 @@ export const useMapStore = create( return { visibleLayerIds: Array.from(newVisibleLayerIds) }; }); }, - })), + contextMenu: null, + setContextMenu: (contextMenu) => set({ contextMenu }), + })) ); -useMapStore.subscribe( - (state) => state.selectedLayer, - (selectedLayer) => { - const mapStore = useMapStore.getState(); - if (mapStore.mapRef && selectedLayer) { - addBlockLayers(mapStore.mapRef, selectedLayer); - mapStore.addVisibleLayerIds([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]); - } - }, -); +// these need to initialize after the map store +getRenderSubscriptions(useMapStore); +getMapMetricsSubs(useMapStore); +getMapEditSubs(useMapStore); +getSearchParamsObersver(); \ No newline at end of file diff --git a/app/src/app/store/metricsSubs.ts b/app/src/app/store/metricsSubs.ts new file mode 100644 index 000000000..9bf573106 --- /dev/null +++ b/app/src/app/store/metricsSubs.ts @@ -0,0 +1,17 @@ +import { updateMapMetrics } from "../utils/api/queries"; +import { useMapStore as _useMapStore } from "./mapStore"; + +export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { + + const mapMetricsSub = useMapStore.subscribe( + (state) => state.mapDocument, + (mapDocument) => { + if (mapDocument) { + updateMapMetrics(mapDocument); + } + } + ); + return [ + mapMetricsSub + ] +} diff --git a/app/src/app/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts similarity index 69% rename from app/src/app/api/apiHandlers.ts rename to app/src/app/utils/api/apiHandlers.ts index adba6e7e9..93484c36c 100644 --- a/app/src/app/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -21,22 +21,48 @@ export const FormatAssignments = () => { return assignments; }; +/** + * DistrictrMap + * + * @interface + * @property {string} name - The name. + * @property {string} gerrydb_table_name - The gerrydb table name. + * @property {string} parent_layer - The parent layer. + * @property {string | null} child_layer - The child layer. + * @property {string | null} tiles_s3_path - The tiles s3 path. + * @property {number | null} num_districts - The number of districts. + */ +export interface DistrictrMap { + name: string; + gerrydb_table_name: string; + parent_layer: string; + child_layer: string | null; + tiles_s3_path: string | null; + num_districts: number | null; +} + /** * Document * * @interface * @property {string} document_id - The document id. * @property {string} gerrydb_table - The gerrydb table. + * @property {string} parent_layer_name - The parent layer name. + * @property {string | null} child_layer_name - The child layer name. + * @property {string | null} tiles_s3_path - The tiles s3 path. + * @property {number | null} num_districts - The number of districts to enforce. * @property {string} created_at - The created at. * @property {string} updated_at - The updated at. - * @property {string} tiles_s3_path - The tiles s3 path. */ export interface DocumentObject { document_id: string; gerrydb_table: string; - created_at: string; - updated_at: string; + parent_layer: string; + child_layer: string | null; tiles_s3_path: string | null; + num_districts: number | null; + created_at: string; + updated_at: string | null; } /** @@ -130,27 +156,15 @@ export const getZonePopulations: ( }; /** - * GerryDB view. - * - * @interface - * @property {string} name - Table name should match the name of the GerryDB table in Postgres and name of the layer in the tileset. - * @property {string} tiles_s3_path - the path to the tiles in the S3 bucket - */ -export interface gerryDBView { - name: string; - tiles_s3_path: string; -} - -/** - * Get available GerryDB views from the server. + * Get available DistrictrMap views from the server. * @param limit - number, the number of views to return (default 10, max 100) * @param offset - number, the number of views to skip (default 0) * @returns Promise */ -export const getGerryDBViews: ( +export const getAvailableDistrictrMaps: ( limit?: number, offset?: number, -) => Promise = async (limit = 10, offset = 0) => { +) => Promise = async (limit = 10, offset = 0) => { return await axios .get( `${process.env.NEXT_PUBLIC_API_URL}/api/gerrydb/views?limit=${limit}&offset=${offset}`, @@ -171,6 +185,7 @@ export interface Assignment { document_id: string; geo_id: string; zone: number; + parent_path?: string } /** @@ -198,3 +213,37 @@ export const patchUpdateAssignments: ( return res.data; }); }; + +/** + * Shatter result + * @interface + * @property {string[]} parents - The parents. + * @property {Assignment[]} children - The children. + */ +export interface ShatterResult { + parents: { geoids: string[] }; + children: Assignment[]; +} + +/** + * Shatter parents + * + * @param document_id - string, the document id + * @param geoids - string[], the geoids to shatter + * @returns list of child assignments results from shattered parents + */ +export const patchShatterParents: (params: { + document_id: string; + geoids: string[]; +}) => Promise = async ({ document_id, geoids }) => { + return await axios + .patch( + `${process.env.NEXT_PUBLIC_API_URL}/api/update_assignments/${document_id}/shatter_parents`, + { + geoids: geoids, + }, + ) + .then((res) => { + return res.data; + }); +}; diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts new file mode 100644 index 000000000..ed9ac5425 --- /dev/null +++ b/app/src/app/utils/api/mutations.ts @@ -0,0 +1,66 @@ +import { MutationObserver } from "@tanstack/query-core"; +import { queryClient } from "./queryClient"; +import { + AssignmentsCreate, + createMapDocument, + patchShatterParents, + patchUpdateAssignments, +} from "@/app/utils/api/apiHandlers"; +import { useMapStore } from "@/app/store/mapStore"; +import { mapMetrics } from "./queries"; + +export const patchShatter = new MutationObserver(queryClient, { + mutationFn: patchShatterParents, + onMutate: ({ document_id, geoids }) => { + useMapStore.getState().setMapLock(true); + console.log( + `Shattering parents for ${geoids} in document ${document_id}...`, + `Locked at `, + performance.now() + ); + }, + onError: (error) => { + console.log("Error updating assignments: ", error); + }, + onSuccess: (data) => { + console.log( + `Successfully shattered parents into ${data.children.length} children` + ); + return data; + }, +}); + +export const patchUpdates = new MutationObserver(queryClient, { + mutationFn: patchUpdateAssignments, + onMutate: () => { + console.log("Updating assignments"); + }, + onError: (error) => { + console.log("Error updating assignments: ", error); + }, + onSuccess: (data: AssignmentsCreate) => { + console.log( + `Successfully upserted ${data.assignments_upserted} assignments` + ); + mapMetrics.refetch(); + }, +}); + +export const document = new MutationObserver(queryClient, { + mutationFn: createMapDocument, + onMutate: () => { + console.log("Creating document"); + useMapStore.getState().setAppLoadingState('loading') + useMapStore.getState().resetZoneAssignments() + }, + onError: (error) => { + console.error("Error creating map document: ", error); + }, + onSuccess: (data) => { + useMapStore.getState().setMapDocument(data); + useMapStore.getState().setAppLoadingState('loaded') + const documentUrl = new URL(window.location.toString()) + documentUrl.searchParams.set("document_id", data.document_id); + history.pushState({}, "", documentUrl.toString()); + }, +}); \ No newline at end of file diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts new file mode 100644 index 000000000..99b7f0b54 --- /dev/null +++ b/app/src/app/utils/api/queries.ts @@ -0,0 +1,51 @@ +import { QueryObserver, skipToken } from "@tanstack/react-query"; +import { queryClient } from "./queryClient"; +import { + DocumentObject, + getDocument, + getZonePopulations, + ZonePopulation, +} from "./apiHandlers"; +import { useMapStore } from "@/app/store/mapStore"; + +export const mapMetrics = new QueryObserver(queryClient, { + queryKey: ["_zonePopulations"], + queryFn: skipToken, +}); + +export const updateMapMetrics = (mapDocument: DocumentObject) => { + mapMetrics.setOptions({ + queryKey: ["zonePopulations", mapDocument.document_id], + queryFn: mapDocument ? () => getZonePopulations(mapDocument) : skipToken, + }); +}; + +mapMetrics.subscribe((result) => { + useMapStore.getState().setMapMetrics(result); +}); + +export const updateDocumentFromId = new QueryObserver( + queryClient, + { + queryKey: ["mapDocument"], + queryFn: async () => { + const document_id = new URL(window.location.href).searchParams.get( + "document_id" + ); + const mapDocument = useMapStore.getState().mapDocument; + if (document_id && mapDocument?.document_id !== document_id) { + useMapStore.getState().setAppLoadingState('loading'); + return await getDocument(document_id); + } else { + return null; + } + }, + } +); + +updateDocumentFromId.subscribe((mapDocument) => { + if (mapDocument.data) { + useMapStore.getState().setMapDocument(mapDocument.data); + } +}); + diff --git a/app/src/app/utils/api/queryClient.ts b/app/src/app/utils/api/queryClient.ts new file mode 100644 index 000000000..7b3125501 --- /dev/null +++ b/app/src/app/utils/api/queryClient.ts @@ -0,0 +1,2 @@ +import { QueryClient } from "@tanstack/react-query"; +export const queryClient = new QueryClient(); \ No newline at end of file diff --git a/app/src/app/utils/api/queryParamsListener.ts b/app/src/app/utils/api/queryParamsListener.ts new file mode 100644 index 000000000..47495f233 --- /dev/null +++ b/app/src/app/utils/api/queryParamsListener.ts @@ -0,0 +1,22 @@ +import { updateDocumentFromId } from "./queries"; + +export const getSearchParamsObersver = () => { + // next ssr safety + if (typeof window === "undefined") { + return + } + let previousDocumentID = ""; + const observer = new MutationObserver(() => { + const documentId = new URLSearchParams(window.location.search).get( + "document_id" + ); + if (documentId && documentId !== previousDocumentID) { + previousDocumentID = documentId; + updateDocumentFromId.refetch(); + } + }); + const config = { subtree: true, childList: true }; + // start listening to changes + observer.observe(document, config); + return observer; +}; diff --git a/app/src/app/utils/events/handlers.ts b/app/src/app/utils/events/handlers.ts index 92b8f1878..af967610d 100644 --- a/app/src/app/utils/events/handlers.ts +++ b/app/src/app/utils/events/handlers.ts @@ -46,7 +46,6 @@ export const SelectMapFeatures = ( accumulatedGeoids, accumulatedBlockPopulations, activeTool, - selectedLayer, selectedZone, } = mapStoreRef; if (activeTool === "eraser") { @@ -58,7 +57,7 @@ export const SelectMapFeatures = ( { source: BLOCK_SOURCE_ID, id: feature?.id ?? undefined, - sourceLayer: selectedLayer?.name, + sourceLayer: feature.sourceLayer, }, { selected: true, zone: selectedZone }, ); @@ -103,6 +102,7 @@ export const SelectZoneAssignmentFeatures = (mapStoreRef: MapStore) => { * @param features - Array of MapGeoJSONFeature from QueryRenderedFeatures * @param map - MutableRefObject, the maplibre map instance * @param hoverGeoids - MutableRefObject>, used to keep track of geoids that have been hovered over + * @deprecated This function is no longer in use and will be removed in a future version. */ export const HighlightFeature = ( features: Array | undefined, @@ -130,8 +130,8 @@ export const HighlightFeature = ( map.current?.setFeatureState( { source: BLOCK_SOURCE_ID, - id: feature?.id ?? undefined, - sourceLayer: sourceLayer, + id: feature.id ?? undefined, + sourceLayer: feature.sourceLayer, }, { hover: true }, ); @@ -151,6 +151,7 @@ export const HighlightFeature = ( * called using `map.on("mouseleave", "blocks-hover", ...)` pattern. * @param map - MutableRefObject, the maplibre map instance * @param hoverFeatureIds - MutableRefObject>, used to keep track of geoids that have been hovered over + * @deprecated This function is no longer in use and will be removed in a future version. */ export const UnhighlightFeature = ( map: MutableRefObject, diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index 59cfa12dd..dfa9e26ee 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -2,39 +2,56 @@ Port over from map events declared at: https://github.com/uchicago-dsi/districtr-components/blob/2e8f9e5657b9f0fd2419b6f3258efd74ae310f32/src/Districtr/Districtr.tsx#L230 */ "use client"; -import type { Map, MapLayerMouseEvent, MapLayerTouchEvent } from "maplibre-gl"; -import { useMapStore } from "@/app/store/mapStore"; -import { MutableRefObject, useRef } from "react"; -import React from "react"; +import type { + Map as MapLibreMap, + MapLayerMouseEvent, + MapLayerTouchEvent, +} from "maplibre-gl"; +import { useMapStore } from "@store/mapStore"; +import { MutableRefObject } from "react"; +import { SelectMapFeatures, SelectZoneAssignmentFeatures } from "./handlers"; +import { ResetMapSelectState } from "@utils/events/handlers"; import { - HighlightFeature, - SelectMapFeatures, - SelectZoneAssignmentFeatures, - UnhighlightFeature, -} from "./handlers"; -import { ResetMapSelectState } from "@/app/utils/events/handlers"; + INTERACTIVE_LAYERS, + BLOCK_HOVER_LAYER_ID, + BLOCK_LAYER_ID, + BLOCK_LAYER_ID_CHILD, +} from "@constants/layers"; /* MapEvent handling; these functions are called by the event listeners in the MapComponent */ +/** + +*/ +function getLayerIdsToPaint(child_layer: string | undefined | null) { + return child_layer + ? [BLOCK_LAYER_ID, BLOCK_LAYER_ID_CHILD] + : [BLOCK_LAYER_ID]; +} + /** * What happens when the map is clicked on; incomplete implementation * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object * @param map - MutableRefObject, the maplibre map instance - * @param hoverFeatureIds - React.MutableRefObject>, used to keep track of geoids that have been hovered over */ export const handleMapClick = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; - const sourceLayer = mapStore.selectedLayer?.name; + const sourceLayer = mapStore.mapDocument?.parent_layer; if (activeTool === "brush" || activeTool === "eraser") { - const selectedFeatures = mapStore.paintFunction(map, e, mapStore.brushSize); + const paintLayers = getLayerIdsToPaint(mapStore.mapDocument?.child_layer); + const selectedFeatures = mapStore.paintFunction( + map, + e, + mapStore.brushSize, + paintLayers, + ); if (sourceLayer) { // select on both the map object and the store @@ -49,8 +66,7 @@ export const handleMapClick = ( export const handleMapMouseUp = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; @@ -65,8 +81,7 @@ export const handleMapMouseUp = ( export const handleMapMouseDown = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; @@ -83,59 +98,53 @@ export const handleMapMouseDown = ( export const handleMapMouseEnter = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => {}; export const handleMapMouseOver = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => {}; export const handleMapMouseLeave = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; - const sourceLayer = mapStore.selectedLayer?.name; - if ( - sourceLayer && - hoverFeatureIds.current.size && - (activeTool === "brush" || activeTool === "eraser") - ) { - UnhighlightFeature(map, hoverFeatureIds, sourceLayer); - } + const sourceLayer = mapStore.mapDocument?.parent_layer; + const setHoverFeatures = mapStore.setHoverFeatures; + setHoverFeatures([]); }; export const handleMapMouseOut = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> -) => { - // console.log("mouse out"); -}; + map: MutableRefObject, +) => {}; export const handleMapMouseMove = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; + const setHoverFeatures = mapStore.setHoverFeatures; const isPainting = mapStore.isPainting; - const sourceLayer = mapStore.selectedLayer?.name; - const selectedFeatures = mapStore.paintFunction(map, e, mapStore.brushSize); - if (sourceLayer && (activeTool === "brush" || activeTool === "eraser")) { - HighlightFeature(selectedFeatures, map, hoverFeatureIds, sourceLayer); + const sourceLayer = mapStore.mapDocument?.parent_layer; + const paintLayers = getLayerIdsToPaint(mapStore.mapDocument?.child_layer); + const selectedFeatures = mapStore.paintFunction( + map, + e, + mapStore.brushSize, + paintLayers, + ); + const isBrushingTool = + sourceLayer && ["brush", "eraser"].includes(activeTool); + if (isBrushingTool) { + setHoverFeatures(selectedFeatures); } - if ( - (activeTool === "brush" || activeTool === "eraser") && - isPainting && - sourceLayer - ) { + + if (isBrushingTool && isPainting) { // selects in the map object; the store object // is updated in the mouseup event SelectMapFeatures(selectedFeatures, map, mapStore); @@ -144,8 +153,7 @@ export const handleMapMouseMove = ( export const handleMapZoom = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => {}; export const handleMapIdle = () => {}; @@ -154,15 +162,14 @@ export const handleMapMoveEnd = () => {}; export const handleMapZoomEnd = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, - hoverFeatureIds: React.MutableRefObject> + map: MutableRefObject, ) => {}; export const handleResetMapSelectState = ( - map: MutableRefObject + map: MutableRefObject, ) => { const mapStore = useMapStore.getState(); - const sourceLayer = mapStore.selectedLayer?.name; + const sourceLayer = mapStore.mapDocument?.parent_layer; if (sourceLayer) { ResetMapSelectState(map, mapStore, sourceLayer); } else { @@ -170,9 +177,40 @@ export const handleResetMapSelectState = ( } }; -export const useHoverFeatureIds = () => { - const hoverFeatureIds = useRef(new Set()); - return hoverFeatureIds; +export const handleMapContextMenu = ( + e: MapLayerMouseEvent | MapLayerTouchEvent, + map: MutableRefObject, +) => { + const mapStore = useMapStore.getState(); + if (mapStore.activeTool !== "pan") { + return; + } + e.preventDefault(); + const setHoverFeatures = mapStore.setHoverFeatures; + const sourceLayer = mapStore.mapDocument?.parent_layer; + // Selects from the hover layers instead of the points + // Otherwise, its hard to select precisely + const paintLayers = mapStore.mapDocument?.child_layer + ? INTERACTIVE_LAYERS + : [BLOCK_HOVER_LAYER_ID]; + const selectedFeatures = mapStore.paintFunction(map, e, 0, paintLayers); + if (!selectedFeatures?.length || !map.current || !sourceLayer) return; + + setHoverFeatures(selectedFeatures.slice(0, 1)); + + const handleClose = () => { + mapStore.setContextMenu(null); + setHoverFeatures([]); + }; + + map.current.once("movestart", handleClose); + + mapStore.setContextMenu({ + x: e.point.x, + y: e.point.y, + data: selectedFeatures[0], + close: handleClose, + }); }; export const mapEvents = [ @@ -191,4 +229,5 @@ export const mapEvents = [ { action: "idle", handler: handleMapIdle }, { action: "moveend", handler: handleMapMoveEnd }, { action: "zoomend", handler: handleMapZoomEnd }, + { action: "contextmenu", handler: handleMapContextMenu }, ]; diff --git a/app/src/app/utils/helpers.ts b/app/src/app/utils/helpers.ts index 234bb442d..662aded3b 100644 --- a/app/src/app/utils/helpers.ts +++ b/app/src/app/utils/helpers.ts @@ -9,10 +9,16 @@ import { } from "maplibre-gl"; import { MutableRefObject } from "react"; import { Point } from "maplibre-gl"; -import { BLOCK_LAYER_ID } from "@/app/constants/layers"; +import { + BLOCK_HOVER_LAYER_ID, + BLOCK_LAYER_ID, + BLOCK_LAYER_ID_CHILD, + BLOCK_SOURCE_ID, +} from "@/app/constants/layers"; import { polygon, multiPolygon } from "@turf/helpers"; import { booleanWithin } from "@turf/boolean-within"; import { pointOnFeature } from "@turf/point-on-feature"; +import { MapStore, useMapStore } from "../store/mapStore"; /** * PaintEventHandler @@ -25,8 +31,26 @@ export type PaintEventHandler = ( map: React.MutableRefObject, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, + layers?: string[], ) => MapGeoJSONFeature[] | undefined; +/** + * ContextMenuState + * Represents the state of the context menu. + * @typedef {Object} ContextMenuState + * @property {number} x - The x-coordinate of the context menu. + * @property {number} y - The y-coordinate of the context menu. + * @property {Object} data - The data associated with the context menu. + * @property {string} data.geoid - The geographic ID. + * @property {string} data.name - The name associated with the geographic ID. + */ +export type ContextMenuState = { + x: number; + y: number; + data: MapGeoJSONFeature; + close: () => void; +}; + /** * boxAroundPoint * Create a bounding box around a point on the map. @@ -56,12 +80,11 @@ export const getFeaturesInBbox = ( map: MutableRefObject, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, + layers: string[] = [BLOCK_LAYER_ID], ): MapGeoJSONFeature[] | undefined => { const bbox = boxAroundPoint(e, brushSize); - return map.current?.queryRenderedFeatures(bbox, { - layers: [BLOCK_LAYER_ID], - }); + return map.current?.queryRenderedFeatures(bbox, { layers }); }; /** @@ -76,6 +99,7 @@ export const getFeaturesIntersectingCounties = ( map: MutableRefObject, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, + layers: string[] = [BLOCK_LAYER_ID], ): MapGeoJSONFeature[] | undefined => { if (!map.current) return; @@ -93,7 +117,7 @@ export const getFeaturesIntersectingCounties = ( const ne = map.current.project(featureBbox[1]); const features = map.current?.queryRenderedFeatures([sw, ne], { - layers: [BLOCK_LAYER_ID], + layers, }); let countyPoly; @@ -144,6 +168,7 @@ const getBoundingBoxFromFeatures = ( return [sw, ne]; }; + /** * mousePos * Get the position of the mouse on the map. @@ -210,3 +235,150 @@ export function getVisibleLayers(map: MutableRefObject) { return layer.layout?.visibility === "visible"; }); } + +export type ColorZoneAssignmentsState = [ + MapStore["zoneAssignments"], + MapStore["mapDocument"], + MapStore["mapRef"], + MapStore["shatterIds"], + MapStore["appLoadingState"], + MapStore["mapRenderingState"], +]; + +export const getMap = (_mapRef?: MapStore["mapRef"]) => { + const mapRef = _mapRef || useMapStore.getState().mapRef; + if ( + mapRef?.current && + mapRef.current + ?.getStyle() + .layers.findIndex((layer) => layer.id === BLOCK_HOVER_LAYER_ID) !== -1 + ) { + return null; + } + + return mapRef as MutableRefObject; +}; + +/** + * Assigns colors to zones on the map based on the current zone assignments. + * This function updates the feature state of map features to reflect their assigned zones. + * + * @function + * @name colorZoneAssignments + * @returns {void} + * + * @requires useMapStore + * @requires BLOCK_SOURCE_ID + * + * @description + * This function does the following: + * 1. Retrieves the current state from the map store. + * 2. Checks if the map reference and map document are available. + * 3. Iterates through the zone assignments. + * 4. Determines whether each assignment is for a parent or child layer. + * 5. Sets the feature state for each assigned feature on the map. + */ +export const colorZoneAssignments = ( + state: ColorZoneAssignmentsState, + previousState?: ColorZoneAssignmentsState, +) => { + const [ + zoneAssignments, + mapDocument, + mapRef, + _, + appLoadingState, + mapRenderingState, + ] = state; + const previousZoneAssignments = previousState?.[0] || null; + + if ( + !mapRef?.current || + !mapDocument || + appLoadingState !== "loaded" || + mapRenderingState !== "loaded" + ) { + return; + } + const isInitialRender = + previousState?.[4] !== "loaded" || previousState?.[5] !== "loaded"; + + zoneAssignments.forEach((zone, id) => { + if ( + !isInitialRender && + previousZoneAssignments?.get(id) === zoneAssignments.get(id) + ) { + return; + } + + // This is awful + // we need information on whether an assignment is parent or child + const isParent = id.toString().includes("vtd"); + const sourceLayer = isParent + ? mapDocument.parent_layer + : mapDocument.child_layer; + + if (!sourceLayer) { + return; + } + + mapRef.current?.setFeatureState( + { + source: BLOCK_SOURCE_ID, + id, + sourceLayer, + }, + { + selected: true, + zone, + }, + ); + }); +}; + +// property changes on which to re-color assignments +export const colorZoneAssignmentTriggers = [ + "zoneAssignments", + "mapDocument", + "mapRef", + "shatterIds", +] as Array; + +/** + * Sets zone assignments for child elements based on their parent's assignment. + * + * @param {MapStore['zoneAssignments']} zoneAssignments - The current map of zone assignments. + * @param {string} parent - The ID of the parent element. + * @param {string[]} children - An array of child element IDs. + * + * @description + * This function checks if the parent has a zone assignment. If it does: + * 1. It assigns the parent's zone to all the children. + * 2. It removes the parent's zone assignment. + * This is typically used when "shattering" a parent element into its constituent parts. + */ +export const setZones = ( + zoneAssignments: MapStore["zoneAssignments"], + parent: string, + children: Set, +) => { + const zone = zoneAssignments.get(parent); + if (zone) { + children.forEach((childId) => { + zoneAssignments.set(childId, zone); + }); + zoneAssignments.delete(parent); + } +}; + +export const shallowCompareArray = (curr: unknown[], prev: unknown[]) => { + if (curr.length !== prev.length) { + return false; + } + for (let i = 0; i < curr.length; i++) { + if (curr[i] !== prev[i]) { + return false; + } + } + return true; +}; diff --git a/app/tsconfig.json b/app/tsconfig.json index 2753f6753..c00ce254b 100644 --- a/app/tsconfig.json +++ b/app/tsconfig.json @@ -1,10 +1,6 @@ { "compilerOptions": { - "lib": [ - "dom", - "dom.iterable", - "esnext" - ], + "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -22,9 +18,12 @@ } ], "paths": { - "@/*": [ - "./src/*" - ] + "@/*": ["./src/*"], + "@components/*": ["./src/app/components/*"], + "@api/*": ["./src/app/api/*"], + "@constants/*": ["./src/app/constants/*"], + "@store/*": ["./src/app/store/*"], + "@utils/*": ["./src/app/utils/*"] }, "target": "ES6" }, @@ -35,8 +34,9 @@ ".next/types/**/*.ts", "src/app/components/Picker.jsx", "src/app/page.jsx", - "src/app/api/apiHandlers.test.js", "src/app/components/sidebar/MapModeSelector.jsx", "src/app/components/sidebar/ColorPicker.jsx", ], - "exclude": [ - "node_modules" - ] + "src/app/api/apiHandlers.test.js", + "src/app/components/sidebar/MapModeSelector.jsx", + "src/app/components/sidebar/ColorPicker.jsx" + ], + "exclude": ["node_modules"] } diff --git a/backend/README.md b/backend/README.md index cf996c088..d93105e76 100644 --- a/backend/README.md +++ b/backend/README.md @@ -87,7 +87,7 @@ If needed, create a user for yourself. ### Testing -`pytest --cov=app --cov-report=html` +`pytest --cov=.` Or with full coverage report: @@ -102,3 +102,137 @@ By default, the test database is created and destroyed for each test run. If you ### Useful reference apps - [full-stack-fastapi-template](https://github.com/tiangolo/full-stack-fastapi-template/tree/master) + +## Backend CLI + +Because all endpoints are public, we need to be careful about what we expose in the API. +As such, management commands are to be exclusively run using the CLI. + +To see which commands are available, run `python cli.py --help`. +You can also inspect individual commands by running `python cli.py --help`. + +These commands are fairly atomic. In the future we can create more complex commands that chain these together +to simplify set-up but for now that's a pretty low priority / devex need. + +### Adding a new Districtr Map + +A Districtr Map can either be shatterable or unshatterable. Both require GerryDB views to be loaded. + +#### Loading GerryDB views + +Run: +```sh +python cli.py import-gerrydb-view \ + --layer layer_name_in_geopackage \ + --gpkg /path/to/geopackage.gpkg \ + --replace +``` + +If creating a shatterable map, you'll need to load at least two views. +Make sure that the second view, the "child" view can be shattered by the "parent" view–meaning the child +geometries nest within the parent geometries. + +#### Creating an unshatterable map + +1. Load the GerryDB view for the map. See above. +1. Create a Districtr Map by running: + +```sh +python cli.py create-districtr-map \ + --name "My map name" \ + --gerrydb-table-name gerrydb_layer_name \ + --parent-layer-name gerrydb_layer_name \ + --tiles-s3-path path/to/my/tiles.pmtiles +``` + +**A few important notes on tilesets:** +- The `tiles-s3-path` should be the URL _path_ only, without the leading `/`. The scheme, subdomain, domain, TLD, and port (if any) are passed to the FE via the `NEXT_PUBLIC_S3_BUCKET_URL` environment variable. +- For more on tilesets, see [Tileset CLI](###tilesets-cli). + +#### Creating a shatterable map + +1. Load GerryDB parent layer. See above. +1. Load GerryDB child layer. See above. +1. Create shatterable `MATERIALIZED VIEW`, which is the union of the parent and child layers by running: + +```sh +python cli.py create-shatterable-districtr-view \ + --gerrydb-table-name gerrydb_layer_name \ + --parent-layer-name gerrydb_parent_layer_name \ + --child-layer-name gerrydb_child_layer_name +``` + +**Note:** The `gerrydb-table-name` must be unique across all shatterable maps. + +4. Create a shatterable Districtr Map by running: + +```sh +python cli.py create-districtr-map \ + --name "My shatterable map name" \ + --gerrydb-table-name gerrydb_layer_name \ + --parent-layer-name gerrydb_parent_layer_name \ + --child-layer-name gerrydb_child_layer_name \ + --tiles-s3-path path/to/my/joined/tiles.pmtiles +``` + +5. Create parent child edges by running `python cli.py create-parent-child-edges --districtr-map gerrydb_layer_name` + +You're done! (Assuming you also created your tilesets. See pipelines CLI for that.) + +## Tileset CLI + +In some of the Backend CLI commands, a tileset path must be provided. +These tilesets are not generated by the Backend CLI, but by the pipelines Simple ELT CLI. +These two CLIs are separate because generating tilesets is a resource-intensive process that is best done in a separate environment. +Separating dependencies also allows us to decouple tile generation fromt the backend, which has its pros/cons. + +To see which commands are available, run `python cli.py --help`. +You can also inspect individual commands by running `python cli.py --help`. + +### Generating individual tilesets + +Use the `create-gerrydb-tileset` command. + +### Generating tilesets for shatterable Districtr Maps + +Shatterable Districtr Maps send both the parent and child layers to the frontend in a single tileset. +You can create tilesets with multiple layers with the tippecannoe `tile-join` utility our our wrapped CLI command, `merge-gerrydb-tilesets`. + +### Production tilesets + +The following tilesets are available to the production environment: + +```txt +aws s3 ls s3://districtr-v2-dev/tilesets/ --endpoint-url=https://de4ecd9d308a46631d2b677af1d480a0.r2.cloudflarestorage.com --profile=cloudflare +2024-08-08 17:01:39 0 +2024-08-08 17:00:39 29505257 co_block_all_basic_pops.pmtiles +2024-08-08 15:49:48 36458896 co_block_p1_view.pmtiles +2024-08-08 16:56:02 29457733 co_block_p4_view.pmtiles +2024-09-21 19:33:26 38903491 co_p1_view.pmtiles +2024-08-08 16:40:45 3227745 co_vtd_all_basic_pops.pmtiles +2024-08-08 16:40:30 3218826 co_vtd_p1_view.pmtiles +2024-08-08 17:01:14 3213257 co_vtd_p4_view.pmtiles +2024-08-08 16:03:14 3836927 de_demo_view_census_blocks.pmtiles +2024-08-08 16:45:09 300671 de_demo_view_census_vtd.pmtiles +2024-08-08 15:58:44 42409135 ga_block_all_basic_pops.pmtiles +2024-08-08 16:10:31 42403512 ga_block_p1_view.pmtiles +2024-08-08 16:40:15 42322552 ga_block_p4_view.pmtiles +2024-08-08 17:00:58 4330016 ga_vtd_all_basic_pops.pmtiles +2024-08-08 16:10:50 4324852 ga_vtd_p1_view.pmtiles +2024-08-08 17:01:34 4319648 ga_vtd_p4_view.pmtiles +2024-08-08 16:45:05 20444228 ks_demo_view_census_blocks.pmtiles +2024-08-08 16:45:23 1583210 ks_demo_view_census_vtd.pmtiles +2024-08-08 16:52:11 49466105 pa_demo_view_census_blocks.pmtiles +2024-08-08 16:41:05 5445496 pa_demo_view_census_vtd.pmtiles +2024-08-08 16:56:52 5923235 ri_block_all_basic_pops.pmtiles +2024-08-08 16:41:30 5922335 ri_block_p1_view.pmtiles +2024-08-08 16:56:27 5905631 ri_block_p4_view.pmtiles +2024-08-08 17:01:39 308259 ri_vtd_all_basic_pops.pmtiles +2024-08-08 15:58:49 307905 ri_vtd_p1_view.pmtiles +2024-08-08 16:45:14 307582 ri_vtd_p4_view.pmtiles +``` + +Only `co_p1_view.pmtiles` is a shatterable tileset, +which has both `co_vtd_p1_view` and `co_block_p1_view` as separate layers. + +For now, @raphaellaude is the only maintainer that can add tilesets to the production environment. diff --git a/backend/app/alembic/env.py b/backend/app/alembic/env.py index f192eefab..de4a2c2e8 100644 --- a/backend/app/alembic/env.py +++ b/backend/app/alembic/env.py @@ -45,7 +45,9 @@ def get_url(): def include_object(object, name, type_, reflected, compare_to): if name and ( - name == "spatial_ref_sys" or re.match(r"document.assignments_.+", name) + name == "spatial_ref_sys" + or re.match(r"document.assignments_.+", name) + or re.match(r"parentchildedges_.+", name) ): return False return True diff --git a/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py b/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py index d795225bf..a1f15eb66 100644 --- a/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py +++ b/backend/app/alembic/versions/09d011c1b387_zones_can_be_null.py @@ -5,6 +5,7 @@ Create Date: 2024-09-09 13:34:59.347083 """ + from typing import Sequence, Union from alembic import op @@ -12,8 +13,8 @@ # revision identifiers, used by Alembic. -revision: str = '09d011c1b387' -down_revision: Union[str, None] = '8437ce954087' +revision: str = "09d011c1b387" +down_revision: Union[str, None] = "8437ce954087" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/backend/app/alembic/versions/5ab466c5650a_migrate_to_districtrmap.py b/backend/app/alembic/versions/5ab466c5650a_migrate_to_districtrmap.py new file mode 100644 index 000000000..dd8d78837 --- /dev/null +++ b/backend/app/alembic/versions/5ab466c5650a_migrate_to_districtrmap.py @@ -0,0 +1,51 @@ +"""migrate to districtrmap + +Revision ID: 5ab466c5650a +Revises: 5c4028ff26df +Create Date: 2024-10-09 22:41:59.024334 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "5ab466c5650a" +down_revision: Union[str, None] = "5c4028ff26df" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + sa.text(""" + INSERT INTO districtrmap + SELECT + now() as created_at, + now() AS updated_at, + gen_random_uuid() as uuid, + CONCAT( + UPPER(SUBSTRING(name, 1, 2)), + ' ', + REPLACE( + REPLACE(SUBSTRING(name, 4), 'vtd', 'VTD'), + '_', + ' ' + ) + ) as name, + name as gerrydb_table_name, + NULL as num_districts, + CONCAT('tilesets/', name, '.pmtiles') AS tiles_s3_path, + name AS parent_layer, + NULL as child_layer + FROM gerrydbtable; + """) + ) + + +def downgrade() -> None: + # Roll forwards only + pass diff --git a/backend/app/alembic/versions/5c4028ff26df_update_pop_udf.py b/backend/app/alembic/versions/5c4028ff26df_update_pop_udf.py new file mode 100644 index 000000000..3859f5cfd --- /dev/null +++ b/backend/app/alembic/versions/5c4028ff26df_update_pop_udf.py @@ -0,0 +1,89 @@ +"""update pop udf + +Revision ID: 5c4028ff26df +Revises: ccb2a6b81a8b +Create Date: 2024-10-07 20:30:15.230254 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "5c4028ff26df" +down_revision: Union[str, None] = "ccb2a6b81a8b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint("districtr_map_uuid_unique", "districtrmap", ["uuid"]) + op.create_unique_constraint( + "districtr_map_parent_child_edge_unique", + "parentchildedges", + ["districtr_map", "parent_path", "child_path"], + ) + with open(f"{SQL_DIR}/total_pop_udf.sql") as f: + query = f.read() + op.execute(sa.text(query)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "districtr_map_parent_child_edge_unique", "parentchildedges", type_="unique" + ) + op.drop_constraint("districtr_map_uuid_unique", "districtrmap", type_="unique") + op.execute( + sa.text(""" + CREATE OR REPLACE FUNCTION get_total_population(document_id UUID) + RETURNS TABLE (zone TEXT, total_pop BIGINT) AS $$ + DECLARE + gerrydb_table_name TEXT; + sql_query TEXT; + total_pop_column_name TEXT; + BEGIN + SELECT districtrmap.parent_layer INTO gerrydb_table_name + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + SELECT column_name INTO total_pop_column_name + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('total_pop', 'total_vap') + ORDER BY column_name ASC + LIMIT 1; + + IF total_pop_column_name IS NULL THEN + RAISE EXCEPTION 'Population column not found for gerrydbview %', gerrydb_table_name; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.%I, 0))::BIGINT AS total_pop + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', total_pop_column_name, gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; + END; + $$ LANGUAGE plpgsql; + """) + ) + # ### end Alembic commands ### diff --git a/backend/app/alembic/versions/8437ce954087_update_get_total_population_udf.py b/backend/app/alembic/versions/8437ce954087_update_get_total_population_udf.py index 66280a192..3777c131d 100644 --- a/backend/app/alembic/versions/8437ce954087_update_get_total_population_udf.py +++ b/backend/app/alembic/versions/8437ce954087_update_get_total_population_udf.py @@ -10,7 +10,6 @@ from alembic import op import sqlalchemy as sa -from app.constants import SQL_DIR # revision identifiers, used by Alembic. @@ -22,9 +21,52 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - with open(f"{SQL_DIR}/total_pop_udf.sql") as f: - query = f.read() - op.execute(sa.text(query)) + op.execute( + sa.text(""" + CREATE OR REPLACE FUNCTION get_total_population(document_id UUID) + RETURNS TABLE (zone TEXT, total_pop BIGINT) AS $$ + DECLARE + gerrydb_table_name TEXT; + sql_query TEXT; + total_pop_column_name TEXT; + BEGIN + SELECT districtrmap.parent_layer INTO gerrydb_table_name + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + SELECT column_name INTO total_pop_column_name + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('total_pop', 'total_vap') + ORDER BY column_name ASC + LIMIT 1; + + IF total_pop_column_name IS NULL THEN + RAISE EXCEPTION 'Population column not found for gerrydbview %', gerrydb_table_name; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.%I, 0))::BIGINT AS total_pop + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', total_pop_column_name, gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; + END; + $$ LANGUAGE plpgsql; + """) + ) # ### end Alembic commands ### diff --git a/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py b/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py new file mode 100644 index 000000000..1866a3875 --- /dev/null +++ b/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py @@ -0,0 +1,119 @@ +"""shattering + +Revision ID: ccb2a6b81a8b +Revises: 09d011c1b387 +Create Date: 2024-09-13 09:44:34.534198 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import app.models +import sqlmodel.sql.sqltypes +from pathlib import Path + +SQL_PATH = Path(__file__).parent.parent.parent / "sql" + + +# revision identifiers, used by Alembic. +revision: str = "ccb2a6b81a8b" +down_revision: Union[str, None] = "09d011c1b387" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "districtrmap", + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("uuid", app.models.UUIDType(), nullable=False), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column( + "gerrydb_table_name", sqlmodel.sql.sqltypes.AutoString(), nullable=True + ), + sa.Column("num_districts", sa.Integer(), nullable=True), + sa.Column("tiles_s3_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("parent_layer", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("child_layer", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.ForeignKeyConstraint( + ["child_layer"], + ["gerrydbtable.name"], + ), + sa.ForeignKeyConstraint( + ["parent_layer"], + ["gerrydbtable.name"], + ), + sa.PrimaryKeyConstraint("uuid"), + sa.UniqueConstraint("uuid"), + sa.UniqueConstraint("gerrydb_table_name"), + ) + op.create_table( + "parentchildedges", + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("districtr_map", app.models.UUIDType(), nullable=False), + sa.Column("parent_path", sa.String(), nullable=False), + sa.Column("child_path", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["districtr_map"], + ["districtrmap.uuid"], + ), + sa.PrimaryKeyConstraint("districtr_map", "parent_path", "child_path"), + postgresql_partition_by="LIST (districtr_map)", + ) + op.drop_column("gerrydbtable", "tiles_s3_path") + # ### end Alembic commands ### + + for file_name in [ + "parent_child_relationships.sql", + "create_shatterable_gerrydb_view.sql", + "create_districtr_map_udf.sql", + "shatter_parent.sql", + ]: + with open(SQL_PATH / file_name, "r") as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "gerrydbtable", + sa.Column("tiles_s3_path", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.drop_table("parentchildedges") + op.drop_table("districtrmap") + # ### end Alembic commands ### + op.execute("DROP PROCEDURE IF EXISTS add_parent_child_relationships(TEXT)") + op.execute( + "DROP PROCEDURE IF EXISTS create_shatterable_gerrydb_view(TEXT, TEXT, TEXT)" + ) + for func_name in [ + "create_districtr_map", + "shatter_parent", + ]: + op.execute(f"DROP FUNCTION IF EXISTS {func_name}") diff --git a/backend/app/main.py b/backend/app/main.py index 011d381b2..aceeada77 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,11 +1,12 @@ from fastapi import FastAPI, status, Depends, HTTPException, Query -from pydantic import UUID4 from sqlalchemy import text from sqlalchemy.exc import ProgrammingError -from sqlmodel import Session, select +from sqlmodel import Session, String, select from starlette.middleware.cors import CORSMiddleware from sqlalchemy.dialects.postgresql import insert import logging +from sqlalchemy import bindparam +from sqlmodel import ARRAY import sentry_sdk from app.core.db import engine @@ -13,12 +14,17 @@ from app.models import ( Assignments, AssignmentsCreate, + AssignmentsResponse, + DistrictrMap, Document, DocumentCreate, DocumentPublic, + GEOIDS, + UUIDType, ZonePopulation, - GerryDBTable, - GerryDBViewPublic, + DistrictrMapPublic, + ParentChildEdges, + ShatterResult, ) if settings.ENVIRONMENT == "production": @@ -86,17 +92,33 @@ async def create_document( select( Document.document_id, Document.created_at, - Document.updated_at, Document.gerrydb_table, - GerryDBTable.tiles_s3_path, + Document.updated_at, + DistrictrMap.uuid.label("map_uuid"), # pyright: ignore + DistrictrMap.parent_layer.label("parent_layer"), # pyright: ignore + DistrictrMap.child_layer.label("child_layer"), # pyright: ignore + DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore + DistrictrMap.num_districts.label("num_districts"), # pyright: ignore ) .where(Document.document_id == document_id) - .join(GerryDBTable, Document.gerrydb_table == GerryDBTable.name, isouter=True) + .join( + DistrictrMap, + Document.gerrydb_table == DistrictrMap.gerrydb_table_name, + isouter=True, + ) .limit(1) ) + # Document id has a unique constraint so I'm not sure we need to hit the DB again here + # more valuable would be to check that the assignments table doc = session.exec( stmt ).one() # again if we've got more than one, we have problems. + if not doc.map_uuid: + session.rollback() + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"DistrictrMap matching {data.gerrydb_table} does not exist.", + ) if not doc.document_id: session.rollback() raise HTTPException( @@ -107,30 +129,6 @@ async def create_document( return doc -@app.patch("/api/update_document/{document_id}", response_model=DocumentPublic) -async def update_document( - document_id: UUID4, data: DocumentCreate, session: Session = Depends(get_session) -): - # Validate that gerrydb_table exists? - stmt = text("""UPDATE document.document - SET - gerrydb_table = :gerrydb_table_name, - updated_at = now() - WHERE document_id = :document_id - RETURNING *""") - results = session.execute( - stmt, {"document_id": document_id, "gerrydb_table_name": data.gerrydb_table} - ) - db_document = results.first() - if not db_document: - session.rollback() - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Document not found" - ) - session.commit() - return db_document - - @app.patch("/api/update_assignments") async def update_assignments( data: AssignmentsCreate, session: Session = Depends(get_session) @@ -139,15 +137,61 @@ async def update_assignments( stmt = stmt.on_conflict_do_update( constraint=Assignments.__table__.primary_key, set_={"zone": stmt.excluded.zone} ) - session.exec(stmt) + session.execute(stmt) session.commit() return {"assignments_upserted": len(data.assignments)} +@app.patch( + "/api/update_assignments/{document_id}/shatter_parents", + response_model=ShatterResult, +) +async def shatter_parent( + document_id: str, data: GEOIDS, session: Session = Depends(get_session) +): + stmt = text("""SELECT * + FROM shatter_parent(:input_document_id, :parent_geoids)""").bindparams( + bindparam(key="input_document_id", type_=UUIDType), + bindparam(key="parent_geoids", type_=ARRAY(String)), + ) + results = session.execute( + statement=stmt, + params={ + "input_document_id": document_id, + "parent_geoids": data.geoids, + }, + ) + # :( was getting validation errors so am just going to loop + assignments = [ + Assignments(document_id=str(document_id), geo_id=geo_id, zone=zone) + for document_id, geo_id, zone in results + ] + result = ShatterResult(parents=data, children=assignments) + session.commit() + return result + + # called by getAssignments in apiHandlers.ts -@app.get("/api/get_assignments/{document_id}", response_model=list[Assignments]) +@app.get("/api/get_assignments/{document_id}", response_model=list[AssignmentsResponse]) async def get_assignments(document_id: str, session: Session = Depends(get_session)): - stmt = select(Assignments).where(Assignments.document_id == document_id) + stmt = ( + select( + Assignments.geo_id, + Assignments.zone, + Assignments.document_id, + ParentChildEdges.parent_path, + ) + .join(Document, Assignments.document_id == Document.document_id) + .join( + DistrictrMap, + Document.gerrydb_table == DistrictrMap.gerrydb_table_name, + ) + .outerjoin(ParentChildEdges, Assignments.geo_id == ParentChildEdges.child_path) + .where( + Assignments.document_id == document_id, + ) + ) + results = session.exec(stmt) return results @@ -160,10 +204,17 @@ async def get_document(document_id: str, session: Session = Depends(get_session) Document.created_at, Document.gerrydb_table, Document.updated_at, - GerryDBTable.tiles_s3_path.label("tiles_s3_path"), - ) + DistrictrMap.parent_layer.label("parent_layer"), # pyright: ignore + DistrictrMap.child_layer.label("child_layer"), # pyright: ignore + DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore + DistrictrMap.num_districts.label("num_districts"), # pyright: ignore + ) # pyright: ignore .where(Document.document_id == document_id) - .join(GerryDBTable, Document.gerrydb_table == GerryDBTable.name, isouter=True) + .join( + DistrictrMap, + Document.gerrydb_table == DistrictrMap.gerrydb_table_name, + isouter=True, + ) .limit(1) ) result = session.exec(stmt) @@ -174,7 +225,9 @@ async def get_document(document_id: str, session: Session = Depends(get_session) async def get_total_population( document_id: str, session: Session = Depends(get_session) ): - stmt = text("SELECT * from get_total_population(:document_id) WHERE zone IS NOT NULL") + stmt = text( + "SELECT * from get_total_population(:document_id) WHERE zone IS NOT NULL" + ) try: result = session.execute(stmt, {"document_id": document_id}) return [ @@ -195,7 +248,7 @@ async def get_total_population( ) -@app.get("/api/gerrydb/views", response_model=list[GerryDBViewPublic]) +@app.get("/api/gerrydb/views", response_model=list[DistrictrMapPublic]) async def get_projects( *, session: Session = Depends(get_session), @@ -203,8 +256,8 @@ async def get_projects( limit: int = Query(default=100, le=100), ): gerrydb_views = session.exec( - select(GerryDBTable) - .order_by(GerryDBTable.created_at.asc()) + select(DistrictrMap) + .order_by(DistrictrMap.created_at.asc()) # pyright: ignore .offset(offset) .limit(limit) ).all() diff --git a/backend/app/models.py b/backend/app/models.py index df0829df7..faf842126 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -3,6 +3,7 @@ from pydantic import UUID4, BaseModel from sqlmodel import ( Field, + ForeignKey, SQLModel, UUID, TIMESTAMP, @@ -10,6 +11,7 @@ text, Column, MetaData, + String, ) from app.constants import DOCUMENT_SCHEMA @@ -40,15 +42,67 @@ class TimeStampMixin(SQLModel): ) -class GerryDBTable(TimeStampMixin, SQLModel, table=True): +class DistrictrMap(TimeStampMixin, SQLModel, table=True): uuid: str = Field(sa_column=Column(UUIDType, unique=True, primary_key=True)) - name: str = Field(nullable=False, unique=True) + name: str = Field(nullable=False) + # This is intentionally not a foreign key on `GerryDBTable` because in some cases + # this may be the GerryDBTable but in others the pop table may be a materialized + # view of two GerryDBTables in the case of shatterable maps. + # We'll want to enforce the constraint tha the gerrydb_table_name is either in + # GerrydbTable.name or a materialized view of two GerryDBTables some other way. + gerrydb_table_name: str | None = Field(nullable=True, unique=True) + # Null means default number of districts? Should we have a sensible default? + num_districts: int | None = Field(nullable=True, default=None) tiles_s3_path: str | None = Field(nullable=True) + parent_layer: str = Field( + sa_column=Column(String, ForeignKey("gerrydbtable.name"), nullable=False) + ) + child_layer: str | None = Field( + sa_column=Column( + String, ForeignKey("gerrydbtable.name"), default=None, nullable=True + ) + ) + # schema? will need to contrain the schema + # where does this go? + # when you create the view, pull the columns that you need + # we'll want discrete management steps -class GerryDBViewPublic(BaseModel): +class DistrictrMapPublic(BaseModel): name: str - tiles_s3_path: str | None + gerrydb_table_name: str + parent_layer: str + child_layer: str | None = None + tiles_s3_path: str | None = None + num_districts: int | None = None + + +class GerryDBTable(TimeStampMixin, SQLModel, table=True): + uuid: str = Field(sa_column=Column(UUIDType, unique=True, primary_key=True)) + # Must correspond to the layer name in the tileset + name: str = Field(nullable=False, unique=True) + + +class ParentChildEdges(TimeStampMixin, SQLModel, table=True): + __table_args__ = ( + UniqueConstraint( + "districtr_map", + "parent_path", + "child_path", + name="districtr_map_parent_child_edge_unique", + ), + {"postgresql_partition_by": "LIST (districtr_map)"}, + ) + districtr_map: str = Field( + sa_column=Column( + UUIDType, + ForeignKey("districtrmap.uuid"), + nullable=False, + primary_key=True, + ) + ) + parent_path: str = Field(sa_column=Column(String, nullable=False, primary_key=True)) + child_path: str = Field(sa_column=Column(String, nullable=False, primary_key=True)) class Document(TimeStampMixin, SQLModel, table=True): @@ -66,9 +120,12 @@ class DocumentCreate(BaseModel): class DocumentPublic(BaseModel): document_id: UUID4 gerrydb_table: str | None + parent_layer: str + child_layer: str | None + tiles_s3_path: str | None = None + num_districts: int | None = None created_at: datetime updated_at: datetime - tiles_s3_path: str | None = None class AssignmentsBase(SQLModel): @@ -91,6 +148,22 @@ class AssignmentsCreate(BaseModel): assignments: list[Assignments] +class AssignmentsResponse(SQLModel): + geo_id: str + zone: int | None + parent_path: str | None + document_id: str + + +class GEOIDS(BaseModel): + geoids: list[str] + + +class ShatterResult(BaseModel): + parents: GEOIDS + children: list[Assignments] + + class ZonePopulation(BaseModel): zone: int total_pop: int diff --git a/backend/app/sql/create_districtr_map_udf.sql b/backend/app/sql/create_districtr_map_udf.sql new file mode 100644 index 000000000..08806ea82 --- /dev/null +++ b/backend/app/sql/create_districtr_map_udf.sql @@ -0,0 +1,27 @@ +CREATE OR REPLACE FUNCTION create_districtr_map( + map_name VARCHAR, + gerrydb_table_name VARCHAR, + num_districts INTEGER, + tiles_s3_path VARCHAR, + parent_layer_name VARCHAR, + child_layer_name VARCHAR +) +RETURNS UUID AS $$ +DECLARE + inserted_districtr_uuid UUID; +BEGIN + INSERT INTO districtrmap ( + created_at, + uuid, + name, + gerrydb_table_name, + num_districts, + tiles_s3_path, + parent_layer, + child_layer + ) + VALUES (now(), gen_random_uuid(), $1, $2, $3, $4, $5, $6) + RETURNING uuid INTO inserted_districtr_uuid; + RETURN inserted_districtr_uuid; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/create_shatterable_gerrydb_view.sql b/backend/app/sql/create_shatterable_gerrydb_view.sql new file mode 100644 index 000000000..438513309 --- /dev/null +++ b/backend/app/sql/create_shatterable_gerrydb_view.sql @@ -0,0 +1,42 @@ +/* +NOTE: This enforces identical schemas but requires that the child contain all the columns +except geometry that the parent has. In the future we may want to allow subsetting but +for now I think this is fine. It does make sure the columns are properly ordered which is nice. +*/ +CREATE OR REPLACE PROCEDURE create_shatterable_gerrydb_view( + parent_gerrydb_table_name TEXT, + child_gerrydb_table_name TEXT, + gerrydb_table_name TEXT +) +LANGUAGE plpgsql +AS $$ +DECLARE + sql_query TEXT; + col_list TEXT := '*'; -- Don't think we really need a default + +BEGIN + -- Insert into the gerrydb table first so that we fail early if + -- the name is already taken (due to unique gerrydbtable name constraint) + INSERT INTO gerrydbtable (created_at, uuid, name) + VALUES (now(), gen_random_uuid(), gerrydb_table_name); + + SELECT string_agg(quote_ident(column_name), ', ') + INTO col_list + FROM information_schema.columns + WHERE table_schema = 'gerrydb' + AND table_name = parent_gerrydb_table_name + AND data_type != 'USER-DEFINED' + AND udt_name != 'geometry' -- on second thought, we want the geometry for some types of operations? + AND column_name != 'ogc_fid'; -- ogr2ogr adds a primary key if one is not present + + sql_query := format(' + CREATE MATERIALIZED VIEW gerrydb.%I AS + SELECT %s FROM gerrydb.%I + UNION ALL + SELECT %s + FROM gerrydb.%I + ', gerrydb_table_name, col_list, parent_gerrydb_table_name, col_list, child_gerrydb_table_name); + + EXECUTE sql_query; +END; +$$; diff --git a/backend/app/sql/parent_child_relationships.sql b/backend/app/sql/parent_child_relationships.sql new file mode 100644 index 000000000..e995602ab --- /dev/null +++ b/backend/app/sql/parent_child_relationships.sql @@ -0,0 +1,53 @@ +/* +TODO: Should add additional validations +- Check that in the result set all parents are present once and only once +- Check that in the result set all children are present once +*/ + +CREATE OR REPLACE PROCEDURE add_parent_child_relationships( + districtr_map_uuid UUID +) +LANGUAGE plpgsql +AS $$ +DECLARE + previously_loaded BOOLEAN; + districtr_map RECORD; + +BEGIN + SELECT uuid, parent_layer, child_layer INTO districtr_map + FROM districtrmap + WHERE uuid = districtr_map_uuid; + + IF districtr_map IS NULL THEN + RAISE EXCEPTION 'No districtrmap found for: %', districtr_map_uuid; + END IF; + + SELECT COUNT(*) > 0 INTO previously_loaded + FROM parentchildedges edges + WHERE edges.districtr_map = districtr_map.uuid; + + IF previously_loaded THEN + RAISE EXCEPTION 'Relationships for districtr_map % already loaded', districtr_map_uuid; + END IF; + + EXECUTE format(' + CREATE TABLE "parentchildedges_%s" + PARTITION OF parentchildedges FOR VALUES IN (%L); + + INSERT INTO "parentchildedges_%s" (created_at, districtr_map, parent_path, child_path) + SELECT + now() AS created_at, + $1 as districtr_map, + parent.path as parent_path, + child.path as child_path + FROM + gerrydb.%I AS parent + JOIN + gerrydb.%I AS child + ON -- NOTE: All geometry column aliases must be geometry. This should be enforced on load. + ST_Contains(parent.geometry, ST_PointOnSurface(child.geometry)); + ', districtr_map_uuid, districtr_map_uuid, districtr_map_uuid, districtr_map.parent_layer, districtr_map.child_layer) + USING districtr_map.uuid; + +END; +$$; diff --git a/backend/app/sql/shatter_parent.sql b/backend/app/sql/shatter_parent.sql new file mode 100644 index 000000000..e2e1349e4 --- /dev/null +++ b/backend/app/sql/shatter_parent.sql @@ -0,0 +1,50 @@ +CREATE OR REPLACE FUNCTION shatter_parent( + input_document_id UUID, + parent_geoids VARCHAR[] +) +RETURNS TABLE ( + output_document_id UUID, + output_child_path VARCHAR, + output_zone INTEGER +) AS $$ +DECLARE + districtr_map_uuid UUID; + +BEGIN + SELECT districtrmap.uuid INTO districtr_map_uuid + FROM document.document + INNER JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF districtr_map_uuid IS NULL THEN + RAISE EXCEPTION 'District map uuid not found for document_id: %', input_document_id; + END IF; + + RETURN QUERY + WITH inserted_child_geoids AS ( + INSERT INTO document.assignments (document_id, geo_id, zone) + SELECT $1, child_geoids.child_path, child_geoids.zone + FROM ( + SELECT $1 as document_id, edges.child_path, a.zone + FROM parentchildedges edges + LEFT JOIN document.assignments a + ON edges.parent_path = a.geo_id + AND a.document_id = $1 + WHERE edges.parent_path = ANY(parent_geoids) + AND edges.districtr_map = districtr_map_uuid + ) child_geoids + ON CONFLICT (document_id, geo_id) DO UPDATE SET zone = EXCLUDED.zone + RETURNING geo_id, zone + ) + SELECT + $1 AS document_id, + geo_id, + zone + FROM inserted_child_geoids; + + DELETE FROM document.assignments a + WHERE a.document_id = $1 AND geo_id = ANY(parent_geoids); + +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/total_pop_udf.sql b/backend/app/sql/total_pop_udf.sql index e91a7ad93..80066ec53 100644 --- a/backend/app/sql/total_pop_udf.sql +++ b/backend/app/sql/total_pop_udf.sql @@ -1,28 +1,30 @@ CREATE OR REPLACE FUNCTION get_total_population(document_id UUID) RETURNS TABLE (zone TEXT, total_pop BIGINT) AS $$ DECLARE - gerrydb_table_name TEXT; + doc_districtrmap RECORD; sql_query TEXT; total_pop_column_name TEXT; BEGIN - SELECT gerrydb_table INTO gerrydb_table_name + SELECT districtrmap.* INTO doc_districtrmap FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name WHERE document.document_id = $1; - IF gerrydb_table_name IS NULL THEN + IF doc_districtrmap.gerrydb_table_name IS NULL THEN RAISE EXCEPTION 'Table name not found for document_id: %', $1; END IF; SELECT column_name INTO total_pop_column_name FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = gerrydb_table_name + WHERE table_name = doc_districtrmap.parent_layer AND table_schema = 'gerrydb' AND column_name IN ('total_pop', 'total_vap') ORDER BY column_name ASC LIMIT 1; IF total_pop_column_name IS NULL THEN - RAISE EXCEPTION 'Population column not found for gerrydbview %', gerrydb_table_name; + RAISE EXCEPTION 'Population column not found for gerrydbview %', doc_districtrmap.gerrydb_table_name; END IF; sql_query := format(' @@ -34,7 +36,7 @@ BEGIN ON blocks.path = assignments.geo_id WHERE assignments.document_id = $1 GROUP BY assignments.zone - ', total_pop_column_name, gerrydb_table_name); + ', total_pop_column_name, doc_districtrmap.gerrydb_table_name); RETURN QUERY EXECUTE sql_query USING $1; END; $$ LANGUAGE plpgsql; diff --git a/backend/app/utils.py b/backend/app/utils.py new file mode 100644 index 000000000..0ac723b76 --- /dev/null +++ b/backend/app/utils.py @@ -0,0 +1,106 @@ +from sqlalchemy import text +from sqlalchemy import bindparam, Integer, String, Text +from sqlmodel import Session + +from app.models import UUIDType + + +def create_districtr_map( + session: Session, + name: str, + parent_layer_name: str, + child_layer_name: str | None = None, + gerrydb_table_name: str | None = None, + num_districts: int | None = None, + tiles_s3_path: str | None = None, +) -> str: + """ + Create a new districtr map. + + Args: + session: The database session. + name: The name of the map. + parent_layer_name: The name of the parent layer. + child_layer_name: The name of the child layer. + gerrydb_table_name: The name of the gerrydb table. + num_districts: The number of districts. + tiles_s3_path: The S3 path to the tiles. + + Returns: + The UUID of the inserted map. + """ + stmt = text(""" + SELECT * + FROM create_districtr_map( + :map_name, + :gerrydb_table_name, + :num_districts, + :tiles_s3_path, + :parent_layer_name, + :child_layer_name + )""").bindparams( + bindparam(key="map_name", type_=String), + bindparam(key="gerrydb_table_name", type_=String), + bindparam(key="num_districts", type_=Integer), + bindparam(key="tiles_s3_path", type_=String), + bindparam(key="parent_layer_name", type_=String), + bindparam(key="child_layer_name", type_=String), + ) + + (inserted_map_uuid,) = session.execute( + stmt, + { + "map_name": name, + "gerrydb_table_name": gerrydb_table_name, + "num_districts": num_districts, + "tiles_s3_path": tiles_s3_path, + "parent_layer_name": parent_layer_name, + "child_layer_name": child_layer_name, + }, + ) + return inserted_map_uuid # pyright: ignore + + +def create_shatterable_gerrydb_view( + session: Session, + parent_layer_name: str, + child_layer_name: str, + gerrydb_table_name: str, +) -> None: + stmt = text( + "CALL create_shatterable_gerrydb_view(:parent_layer_name, :child_layer_name, :gerrydb_table_name)" + ).bindparams( + bindparam(key="parent_layer_name", type_=Text), + bindparam(key="child_layer_name", type_=Text), + bindparam(key="gerrydb_table_name", type_=Text), + ) + session.execute( + stmt, + { + "parent_layer_name": parent_layer_name, + "child_layer_name": child_layer_name, + "gerrydb_table_name": gerrydb_table_name, + }, + ) + + +def create_parent_child_edges( + session: Session, + districtr_map_uuid: str, +) -> None: + """ + Create the parent child edges for a given gerrydb map. + + Args: + session: The database session. + districtr_map_uuid: The UUID of the districtr map. + """ + stmt = text("CALL add_parent_child_relationships(:districtr_map_uuid)").bindparams( + bindparam(key="districtr_map_uuid", type_=UUIDType), + ) + session.execute( + stmt, + { + "districtr_map_uuid": districtr_map_uuid, + }, + ) diff --git a/backend/cli.py b/backend/cli.py index ad8d6689a..9f51d91c7 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1,13 +1,18 @@ import os -from typing import Iterable import click import logging + from app.main import get_session from app.core.config import settings import subprocess from urllib.parse import urlparse, ParseResult from sqlalchemy import text from app.constants import GERRY_DB_SCHEMA +from app.utils import ( + create_districtr_map as _create_districtr_map, + create_shatterable_gerrydb_view as _create_shatterable_gerrydb_view, + create_parent_child_edges as _create_parent_child_edges, +) logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -19,6 +24,16 @@ def cli(): def download_file_from_s3(s3, url: ParseResult, replace=False) -> str: + """ + Download a file from S3 to the local volume path. + + Args: + s3: S3 client + url (ParseResult): URL of the file to download + replace (bool): If True, replace the file if it already exists + + Returns the path to the downloaded file. + """ if not s3: raise ValueError("S3 client is not available") @@ -33,7 +48,6 @@ def download_file_from_s3(s3, url: ParseResult, replace=False) -> str: logger.info("Downloading GerryDB view. Got response:\n%s", object_information) - # Download to settings.VOLUME_PATH path = os.path.join(settings.VOLUME_PATH, file_name) if os.path.exists(path) and not replace: @@ -73,6 +87,10 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): layer, # must match layer name in gpkg "-lco", "OVERWRITE=yes", + "-lco", + "GEOMETRY_NAME=geometry", + "-nlt", + "MULTIPOLYGON", "-nln", f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema ], @@ -118,138 +136,100 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): session.close() -@cli.command("create-gerrydb-tileset") -@click.option("--layer", "-n", help="layer of the view", required=True) -@click.option("--gpkg", "-g", help="Path or URL to GeoPackage file", required=True) -@click.option("--replace", "-f", help="Replace the file if it exists", is_flag=True) -@click.option( - "--column", - "-c", - help="Column to use for tileset", - multiple=True, - default=[ - "path", - "geography", - "total_pop", - ], -) -@click.option( - "--rm", "-r", help="Delete tileset after loading to postgres", is_flag=True -) -def create_gerrydb_tileset( - layer: str, gpkg: str, replace: bool, column: Iterable[str], rm: bool -) -> None: - logger.info("Creating GerryDB tileset...") - s3 = settings.get_s3_client() - assert s3, "S3 client is not available" +@cli.command("create-parent-child-edges") +@click.option("--districtr-map", "-d", help="Districtr map name", required=True) +def create_parent_child_edges(districtr_map: str): + logger.info("Creating parent-child edges...") - url = urlparse(gpkg) - logger.info("URL: %s", url) + session = next(get_session()) + stmt = text( + "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" + ) + (districtr_map_uuid,) = session.execute( + stmt, params={"districtrmap_name": districtr_map} + ).one() + print(f"Found districtmap uuid: {districtr_map_uuid}") + _create_parent_child_edges(session=session, districtr_map_uuid=districtr_map_uuid) + session.commit() + logger.info("Parent-child relationship upserted successfully.") - if url.scheme == "s3": - path = download_file_from_s3(s3, url, replace) - else: - path = gpkg + session.close() - fbg_path = f"{settings.VOLUME_PATH}/{layer}.fgb" - logger.info("Creating flatgeobuf...") - if os.path.exists(fbg_path) and not replace: - logger.info("File already exists. Skipping creation.") - else: - result = subprocess.run( - args=[ - "ogr2ogr", - "-f", - "FlatGeobuf", - "-select", - ",".join(column), - "-t_srs", - "EPSG:4326", - fbg_path, - path, - layer, - ] - ) - if result.returncode != 0: - logger.error("ogr2ogr failed. Got %s", result) - raise ValueError(f"ogr2ogr failed with return code {result.returncode}") +@cli.command("delete-parent-child-edges") +@click.option("--districtr-map", "-d", help="Districtr map name", required=True) +def delete_parent_child_edges(districtr_map: str): + logger.info("Deleting parent-child edges...") - logger.info("Creating tileset...") - tileset_path = f"{settings.VOLUME_PATH}/{layer}.pmtiles" + session = next(get_session()) - if os.path.exists(tileset_path) and not replace: - logger.info("File already exists. Skipping creation.") - else: - args = [ - "tippecanoe", - "-zg", - "--coalesce-smallest-as-needed", - "--extend-zooms-if-still-dropping", - # "--drop-densest-as-needed", - # "--drop-smallest-as-needed", - # "-pf", # --no-feature-limit: Don't limit tiles to 200,000 features - # "-pk", # --no-tile-size-limit: Don't limit tiles to 500K bytes - # "-ps", # --no-line-simplification - "-o", - tileset_path, - "-l", - layer, - fbg_path, - ] - if replace: - args.append("--force") - result = subprocess.run(args=args) - - if result.returncode != 0: - logger.error("tippecanoe failed. Got %s", result) - raise ValueError(f"tippecanoe failed with return code {result.returncode}") - - logger.info("Uploading to R2") - s3.put_object(Bucket=settings.R2_BUCKET_NAME, Key="tilesets/") - s3_path = f"tilesets/{layer}.pmtiles" - s3.upload_file( - tileset_path, - settings.R2_BUCKET_NAME, - s3_path, + delete_query = text(""" + DELETE FROM parentchildedges + WHERE districtr_map = :districtr_map + """) + session.execute( + delete_query, + { + "districtr_map": districtr_map, + }, ) + session.commit() + logger.info("Parent-child relationship upserted successfully.") - if rm: - os.remove(path) - os.remove(fbg_path) - os.remove(tileset_path) - logger.info("Deleted files %s, %s, %s", path, fbg_path, tileset_path) - - logger.info("GerryDB tileset uploaded successfully") - - logger.info("Updating GerryDBTiles") - _session = get_session() - session = next(_session) - - upsert_query = text(""" - UPDATE gerrydbtable SET - updated_at = now(), - tiles_s3_path = :tiles_s3_path - WHERE name = :name - RETURNING uuid - """) + session.close() - try: - result = session.execute( - upsert_query, - { - "name": layer, - "tiles_s3_path": s3_path, - }, - ) - session.commit() - logger.info("GerryDB tileset added to view:\n%s", result.fetchone()) - except Exception as e: - session.rollback() - logger.error("Failed to upsert GerryDB tiles. Got %s", e) - raise ValueError(f"Failed to upsert GerryDB tiles. Got {e}") - session.close() +@cli.command("create-districtr-map") +@click.option("--name", help="Name of the districtr map", required=True) +@click.option("--parent-layer-name", help="Parent gerrydb layer name", required=True) +@click.option("--child-layer-name", help="Child gerrydb layer name", required=False) +@click.option("--gerrydb-table-name", help="Name of the GerryDB table", required=True) +@click.option("--num-districts", help="Number of districts", required=False) +@click.option("--tiles-s3-path", help="S3 path to the tileset", required=False) +def create_districtr_map( + name: str, + parent_layer_name: str, + child_layer_name: str | None, + gerrydb_table_name: str, + num_districts: int | None, + tiles_s3_path: str | None, +): + logger.info("Creating districtr map...") + session = next(get_session()) + districtr_map_uuid = _create_districtr_map( + session=session, + name=name, + parent_layer_name=parent_layer_name, + child_layer_name=child_layer_name, + gerrydb_table_name=gerrydb_table_name, + num_districts=num_districts, + tiles_s3_path=tiles_s3_path, + ) + session.commit() + logger.info(f"Districtr map created successfully {districtr_map_uuid}") + + +@cli.command("create-shatterable-districtr-view") +@click.option("--parent-layer-name", help="Parent gerrydb layer name", required=True) +@click.option("--child-layer-name", help="Child gerrydb layer name", required=False) +@click.option("--gerrydb-table-name", help="Name of the GerryDB table", required=False) +def create_shatterable_gerrydb_view( + parent_layer_name: str, + child_layer_name: str, + gerrydb_table_name: str, +): + logger.info("Creating materialized shatterable gerrydb view...") + session = next(get_session()) + inserted_uuid = _create_shatterable_gerrydb_view( + session=session, + parent_layer_name=parent_layer_name, + child_layer_name=child_layer_name, + gerrydb_table_name=gerrydb_table_name, + ) + session.commit() + logger.info( + f"Materialized shatterable gerrydb view created successfully {inserted_uuid}" + ) if __name__ == "__main__": diff --git a/backend/load_data.py b/backend/load_data.py new file mode 100755 index 000000000..2e9faf981 --- /dev/null +++ b/backend/load_data.py @@ -0,0 +1,120 @@ +import subprocess +from os import environ, path +import json # Assuming this import is needed for the JSON operations + +# TODO refactor to these utilities +# from app.utils import create_parent_child_edges, create_districtr_map, create_shatterable_gerrydb_view +from app.main import get_session +from sqlalchemy import text # Add this import at the top of the file + +# Optionally, set a data directory to load in +DATA_DIR = environ.get("GPKG_DATA_DIR", "sample_data") +# flag to load data, by default, will load data +LOAD_DATA = environ.get("LOAD_GERRY_DB_DATA", "false").lower() == "true" + + +def load_sample_data(config): + """ + Load sample data from the specified data directory. + + This function iterates through all files with a '.gpkg' extension in the + specified data directory, and for each file, it runs a script to load the + GerryDB view. + + Args: + None + Returns: + None + """ + + subprocess.run(["alembic", "upgrade", "head"]) + + for view in config["gerrydb_views"]: + subprocess.run( + [ + "python3", + "cli.py", + "import-gerrydb-view", + "--layer", + view["layer"], + "--gpkg", + path.join(DATA_DIR, view["gpkg"]), + "--replace", + ] + ) + + for view in config["shatterable_views"]: + session = next(get_session()) + exists_query = text( + f"SELECT EXISTS (SELECT 1 FROM pg_matviews WHERE matviewname = '{view['gerrydb_table_name']}')" + ) + result = session.execute(exists_query).scalar() + if result: + print( + f"###\nMaterialized view {view['gerrydb_table_name']} already exists.\n###" + ) + else: + subprocess.run( + [ + "python3", + "cli.py", + "create-shatterable-districtr-view", + "--gerrydb-table-name", + view["gerrydb_table_name"], + "--parent-layer-name", + view["parent_layer_name"], + "--child-layer-name", + view["child_layer_name"], + ] + ) + + for view in config["districtr_map"]: + session = next(get_session()) + name = view["name"] + exists_query = text( + f"SELECT count(*) FROM public.districtrmap WHERE name = '{name}'" + ) + result = session.execute(exists_query).scalar() + + if result > 0: + print(f"###\Districtr map {name} already exists.\n###") + else: + subprocess.run( + [ + "python3", + "cli.py", + "create-districtr-map", + "--name", + view["name"], + "--parent-layer-name", + view["parent_layer_name"], + "--child-layer-name", + view["child_layer_name"], + "--gerrydb-table-name", + view["gerrydb_table_name"], + "--tiles-s3-path", + view["tiles_s3_path"], + ] + ) + subprocess.run( + [ + "python3", + "cli.py", + "create-parent-child-edges", + "--districtr-map", + view["gerrydb_table_name"], + ] + ) + + +if __name__ == "__main__": + # Correctly open the config.json file and read its contents + with open(path.join(DATA_DIR, "config.json")) as config_file: + config = json.load(config_file) + + if LOAD_DATA: + load_sample_data(config) + else: + print( + "App startup will not perform data loading.\nTo load, run `export LOAD_GERRY_DB_DATA='True' && python3 load_data.py`\nor change the environment variable in `docker-compose.yml`" + ) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index dbbda6cd8..94e8e4f43 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -2,6 +2,4 @@ exclude = [".venv"] [tool.coverage.run] -omit = [ - "app/**/test_*.py", -] +omit = ["app/**/test_*.py", "app/alembic/**", "**/__init__.py", "tests/**"] diff --git a/backend/scripts/load_data.py b/backend/scripts/load_data.py deleted file mode 100755 index cc27a2b56..000000000 --- a/backend/scripts/load_data.py +++ /dev/null @@ -1,66 +0,0 @@ -import subprocess -import sqlalchemy as sa -from os import environ -from glob import glob - -# Optionally, set a data directory to load in -DATA_DIR = environ.get("GPKG_DATA_DIR", "sample_data") -# flag to load data, by default, will load data -LOAD_DATA = environ.get("LOAD_GERRY_DB_DATA", "true") - - -def update_tile_column(engine): - """ - Update the 'tiles_s3_path' column in the 'gerrydbtable' of the public schema. - - This function connects to the database using the provided SQLAlchemy engine - and executes an UPDATE query. It sets the 'tiles_s3_path' column to a - concatenated string based on the 'name' column. - - Args: - engine (sqlalchemy.engine.Engine): SQLAlchemy engine instance for database connection. - - Prints: - Success message with the number of updated rows or an error message if the update fails. - - Raises: - SQLAlchemyError: If there's an error during the database operation. - """ - print("UPDATING GERRYDB COLUMN") - with engine.connect() as connection: - try: - result = connection.execute( - sa.text( - "UPDATE public.gerrydbtable SET tiles_s3_path = CONCAT('tilesets/', name, '.pmtiles')" - ) - ) - updated_rows = result.rowcount - print(f"Successfully updated {updated_rows} rows in gerrydbtable.") - connection.commit() - except sa.exc.SQLAlchemyError as e: - print(f"Error updating gerrydbtable: {str(e)}") - connection.rollback() - - -def load_sample_data(): - """ - Load sample data from the specified data directory. - - This function iterates through all files with a '.gpkg' extension in the - specified data directory, and for each file, it runs a script to load the - GerryDB view. - - Args: - None - Returns: - None - """ - for gpkg in glob(f"{DATA_DIR}/*.gpkg"): - subprocess.run(["bash", "./scripts/load_gerrydb_view.sh", gpkg]) - - -if __name__ == "__main__": - if LOAD_DATA == "true": - load_sample_data() - engine = sa.create_engine(environ.get("DATABASE_URL")) - update_tile_column(engine) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 000000000..cc651b3ab --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,72 @@ +import os +import pytest +from app.main import app, get_session +from fastapi.testclient import TestClient +from sqlmodel import create_engine, Session +from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError +import subprocess +from tests.constants import ( + POSTGRES_TEST_DB, + TEARDOWN_TEST_DB, + TEST_SQLALCHEMY_DATABASE_URI, + TEST_POSTGRES_CONNECTION_STRING, +) + +client = TestClient(app) + + +@pytest.fixture(name="client") +def client_fixture(session: Session): + def get_session_override(): + return session + + app.dependency_overrides[get_session] = get_session_override + + client = TestClient(app, headers={"origin": "http://localhost:5173"}) + yield client + app.dependency_overrides.clear() + + +my_env = os.environ.copy() +my_env["POSTGRES_DB"] = POSTGRES_TEST_DB + + +@pytest.fixture(scope="session", name="engine") +def engine_fixture(request): + url = TEST_POSTGRES_CONNECTION_STRING + _engine = create_engine(url) + conn = _engine.connect() + conn.execute(text("commit")) + try: + if conn.in_transaction(): + conn.rollback() + conn.execution_options(isolation_level="AUTOCOMMIT").execute( + text(f"CREATE DATABASE {POSTGRES_TEST_DB}") + ) + except (OperationalError, ProgrammingError): + pass + + subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) + + def teardown(): + if TEARDOWN_TEST_DB: + close_connections_query = f""" + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '{POSTGRES_TEST_DB}' + AND pid <> pg_backend_pid(); + """ + conn.execute(text(close_connections_query)) + conn.execute(text(f"DROP DATABASE {POSTGRES_TEST_DB}")) + conn.close() + + request.addfinalizer(teardown) + + return create_engine(str(TEST_SQLALCHEMY_DATABASE_URI), echo=True) + + +@pytest.fixture(name="session") +def session_fixture(engine): + with Session(engine, expire_on_commit=True) as session: + yield session diff --git a/backend/tests/constants.py b/backend/tests/constants.py index ec194ec21..c64415e3a 100644 --- a/backend/tests/constants.py +++ b/backend/tests/constants.py @@ -1,4 +1,29 @@ +import os from pathlib import Path +from tests.utils import string_to_bool +from pydantic_core import MultiHostUrl FIXTURES_PATH = Path(__file__).parent / "fixtures" + + +ENVIRONMENT = os.environ.get("ENVIRONMENT") +POSTGRES_TEST_DB = os.environ.get("POSTGRES_TEST_DB", "districtr_test") +POSTGRES_SCHEME = "postgresql+psycopg" +POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres") +POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "postgres") +POSTGRES_SERVER = os.environ.get("POSTGRES_SERVER", "localhost") +POSTGRES_PORT = os.environ.get("POSTGRES_PORT", 5432) +TEARDOWN_TEST_DB = string_to_bool(os.environ.get("TEARDOWN_TEST_DB", "true")) + +TEST_SQLALCHEMY_DATABASE_URI = MultiHostUrl.build( + scheme=POSTGRES_SCHEME, + username=POSTGRES_USER, + host=POSTGRES_SERVER, + port=int(POSTGRES_PORT), + path=POSTGRES_TEST_DB, + password=POSTGRES_PASSWORD, +) + +TEST_POSTGRES_CONNECTION_STRING = f"{POSTGRES_SCHEME}://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_SERVER}/postgres" +OGR2OGR_PG_CONNECTION_STRING = f"PG:host={POSTGRES_SERVER} port={POSTGRES_PORT} dbname={POSTGRES_TEST_DB} user={POSTGRES_USER} password={POSTGRES_PASSWORD}" diff --git a/backend/tests/fixtures/simple_child_geos.geojson b/backend/tests/fixtures/simple_child_geos.geojson new file mode 100644 index 000000000..cd527879d --- /dev/null +++ b/backend/tests/fixtures/simple_child_geos.geojson @@ -0,0 +1,13 @@ +{ +"type": "FeatureCollection", +"name": "simple_child_geos", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "a", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.863070848232667, 39.182339005832596 ], [ -96.864144926911308, 39.185919268094722 ], [ -96.859234852951801, 39.186737613754637 ], [ -96.859132559744324, 39.183771110737446 ], [ -96.863070848232667, 39.182339005832596 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "b", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.859234852951801, 39.186737613754637 ], [ -96.859714352361905, 39.189045604248619 ], [ -96.856798995948466, 39.190094109625392 ], [ -96.855443610949223, 39.187204326513815 ], [ -96.857642914910244, 39.185644355099598 ], [ -96.859234852951801, 39.186737613754637 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "c", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.856517689627864, 39.1838797972704 ], [ -96.859132559744324, 39.183771110737446 ], [ -96.859234852951801, 39.186737613754637 ], [ -96.857642914910244, 39.185644355099598 ], [ -96.856517689627864, 39.1838797972704 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "d", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.855136731326752, 39.184212250194747 ], [ -96.856517689627864, 39.1838797972704 ], [ -96.857642914910244, 39.185644355099598 ], [ -96.855443610949223, 39.187204326513815 ], [ -96.855136731326752, 39.184212250194747 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "e", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.864291973397059, 39.189224617361731 ], [ -96.859714352361905, 39.189045604248619 ], [ -96.859234852951801, 39.186737613754637 ], [ -96.864144926911308, 39.185919268094722 ], [ -96.864291973397059, 39.189224617361731 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "f", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.864291973397059, 39.189224617361731 ], [ -96.860993017455527, 39.189940669814156 ], [ -96.856798995948466, 39.190094109625392 ], [ -96.859714352361905, 39.189045604248619 ], [ -96.864291973397059, 39.189224617361731 ] ] ] ] } } +] +} diff --git a/backend/tests/fixtures/simple_parent_geos.geojson b/backend/tests/fixtures/simple_parent_geos.geojson new file mode 100644 index 000000000..bdd359136 --- /dev/null +++ b/backend/tests/fixtures/simple_parent_geos.geojson @@ -0,0 +1,10 @@ +{ +"type": "FeatureCollection", +"name": "simple_parent_geos", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "A", "total_pop": 2 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.864144926911308, 39.185919268094722 ], [ -96.864291973397059, 39.189224617361731 ], [ -96.859714352361905, 39.189045604248619 ], [ -96.859234852951801, 39.186737613754637 ], [ -96.859132559744324, 39.183771110737446 ], [ -96.863070848232667, 39.182339005832596 ], [ -96.864144926911308, 39.185919268094722 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "B", "total_pop": 3 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.856798995948466, 39.190094109625392 ], [ -96.855443610949223, 39.187204326513815 ], [ -96.855136731326752, 39.184212250194747 ], [ -96.856517689627864, 39.1838797972704 ], [ -96.859132559744324, 39.183771110737446 ], [ -96.859234852951801, 39.186737613754637 ], [ -96.859714352361905, 39.189045604248619 ], [ -96.856798995948466, 39.190094109625392 ] ] ] ] } }, +{ "type": "Feature", "properties": { "path": "C", "total_pop": 1 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -96.864291973397059, 39.189224617361731 ], [ -96.860993017455527, 39.189940669814156 ], [ -96.856798995948466, 39.190094109625392 ], [ -96.859714352361905, 39.189045604248619 ], [ -96.864291973397059, 39.189224617361731 ] ] ] ] } } +] +} diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 8144f5293..8ada690af 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -1,45 +1,20 @@ import os import pytest -from fastapi.testclient import TestClient -from sqlmodel import Session, create_engine +from sqlmodel import Session -from app.main import app, get_session +from app.main import get_session from app.constants import GERRY_DB_SCHEMA -from pydantic_core import MultiHostUrl from sqlalchemy import text -from sqlalchemy.exc import OperationalError, ProgrammingError import subprocess import uuid -from tests.constants import FIXTURES_PATH -from tests.utils import string_to_bool - - -client = TestClient(app) - -ENVIRONMENT = os.environ.get("ENVIRONMENT") -POSTGRES_TEST_DB = os.environ.get("POSTGRES_TEST_DB", "districtr_test") -POSTGRES_SCHEME = "postgresql+psycopg" -POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres") -POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "postgres") -POSTGRES_SERVER = os.environ.get("POSTGRES_SERVER", "localhost") -POSTGRES_PORT = os.environ.get("POSTGRES_PORT", 5432) -TEARDOWN_TEST_DB = string_to_bool(os.environ.get("TEARDOWN_TEST_DB", "true")) - -my_env = os.environ.copy() - -my_env["POSTGRES_DB"] = POSTGRES_TEST_DB - -TEST_SQLALCHEMY_DATABASE_URI = MultiHostUrl.build( - scheme=POSTGRES_SCHEME, - username=POSTGRES_USER, - host=POSTGRES_SERVER, - port=int(POSTGRES_PORT), - path=POSTGRES_TEST_DB, - password=POSTGRES_PASSWORD, +from tests.constants import ( + OGR2OGR_PG_CONNECTION_STRING, + FIXTURES_PATH, ) +from app.utils import create_districtr_map -def test_read_main(): +def test_read_main(client): response = client.get("/") assert response.status_code == 200 assert response.json() == {"message": "Hello World"} @@ -60,70 +35,15 @@ def test_get_session(): ## Test DB -@pytest.fixture(scope="session", name="engine") -def engine_fixture(request): - url = f"{POSTGRES_SCHEME}://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_SERVER}/postgres" - _engine = create_engine(url) - conn = _engine.connect() - conn.execute(text("commit")) - try: - if conn.in_transaction(): - conn.rollback() - conn.execution_options(isolation_level="AUTOCOMMIT").execute( - text(f"CREATE DATABASE {POSTGRES_TEST_DB}") - ) - except (OperationalError, ProgrammingError): - pass - - subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) - - def teardown(): - if TEARDOWN_TEST_DB: - close_connections_query = f""" - SELECT pg_terminate_backend(pg_stat_activity.pid) - FROM pg_stat_activity - WHERE pg_stat_activity.datname = '{POSTGRES_TEST_DB}' - AND pid <> pg_backend_pid(); - """ - conn.execute(text(close_connections_query)) - conn.execute(text(f"DROP DATABASE {POSTGRES_TEST_DB}")) - conn.close() - - request.addfinalizer(teardown) - - return create_engine(str(TEST_SQLALCHEMY_DATABASE_URI), echo=True) - - -@pytest.fixture(name="session") -def session_fixture(engine): - with Session(engine) as session: - yield session - - -@pytest.fixture(name="client") -def client_fixture(session: Session): - def get_session_override(): - return session - - def get_auth_result_override(): - return True - - app.dependency_overrides[get_session] = get_session_override - - client = TestClient(app, headers={"origin": "http://localhost:5173"}) - yield client - app.dependency_overrides.clear() - - @pytest.fixture(name=GERRY_DB_FIXTURE_NAME) def ks_demo_view_census_blocks_fixture(session: Session): layer = GERRY_DB_FIXTURE_NAME - result = subprocess.run( + subprocess.run( args=[ "ogr2ogr", "-f", "PostgreSQL", - f"PG:host={POSTGRES_SERVER} port={POSTGRES_PORT} dbname={POSTGRES_TEST_DB} user={POSTGRES_USER} password={POSTGRES_PASSWORD}", + OGR2OGR_PG_CONNECTION_STRING, os.path.join(FIXTURES_PATH, f"{layer}.geojson"), "-lco", "OVERWRITE=yes", @@ -132,20 +52,39 @@ def ks_demo_view_census_blocks_fixture(session: Session): ], ) - if result.returncode != 0: - print(f"ogr2ogr failed. Got {result}") - raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + +@pytest.fixture(name="ks_demo_view_census_blocks_districtrmap") +def ks_demo_view_census_blocks_districtrmap_fixture( + session: Session, ks_demo_view_census_blocks_total_vap: None +): + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.begin() + session.execute(upsert_query, {"name": GERRY_DB_FIXTURE_NAME}) + create_districtr_map( + session=session, + name=f"Districtr map {GERRY_DB_FIXTURE_NAME}", + gerrydb_table_name=GERRY_DB_FIXTURE_NAME, + parent_layer_name=GERRY_DB_FIXTURE_NAME, + ) + session.commit() @pytest.fixture(name=GERRY_DB_TOTAL_VAP_FIXTURE_NAME) def ks_demo_view_census_blocks_total_vap_fixture(session: Session): layer = GERRY_DB_TOTAL_VAP_FIXTURE_NAME - result = subprocess.run( + subprocess.run( args=[ "ogr2ogr", "-f", "PostgreSQL", - f"PG:host={POSTGRES_SERVER} port={POSTGRES_PORT} dbname={POSTGRES_TEST_DB} user={POSTGRES_USER} password={POSTGRES_PASSWORD}", + OGR2OGR_PG_CONNECTION_STRING, os.path.join(FIXTURES_PATH, f"{layer}.geojson"), "-lco", "OVERWRITE=yes", @@ -154,20 +93,39 @@ def ks_demo_view_census_blocks_total_vap_fixture(session: Session): ], ) - if result.returncode != 0: - print(f"ogr2ogr failed. Got {result}") - raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + +@pytest.fixture(name="ks_demo_view_census_total_vap_blocks_districtrmap") +def ks_demo_view_census_blocks_total_vap_districtrmap_fixture( + session: Session, ks_demo_view_census_blocks_total_vap: None +): + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.begin() + session.execute(upsert_query, {"name": GERRY_DB_TOTAL_VAP_FIXTURE_NAME}) + create_districtr_map( + session=session, + name=f"Districtr map {GERRY_DB_TOTAL_VAP_FIXTURE_NAME}", + gerrydb_table_name=GERRY_DB_TOTAL_VAP_FIXTURE_NAME, + parent_layer_name=GERRY_DB_TOTAL_VAP_FIXTURE_NAME, + ) + session.commit() @pytest.fixture(name=GERRY_DB_NO_POP_FIXTURE_NAME) def ks_demo_view_census_blocks_no_pop_fixture(session: Session): layer = GERRY_DB_NO_POP_FIXTURE_NAME - result = subprocess.run( + subprocess.run( args=[ "ogr2ogr", "-f", "PostgreSQL", - f"PG:host={POSTGRES_SERVER} port={POSTGRES_PORT} dbname={POSTGRES_TEST_DB} user={POSTGRES_USER} password={POSTGRES_PASSWORD}", + OGR2OGR_PG_CONNECTION_STRING, os.path.join(FIXTURES_PATH, f"{layer}.geojson"), "-lco", "OVERWRITE=yes", @@ -176,13 +134,11 @@ def ks_demo_view_census_blocks_no_pop_fixture(session: Session): ], ) - if result.returncode != 0: - print(f"ogr2ogr failed. Got {result}") - raise ValueError(f"ogr2ogr failed with return code {result.returncode}") - -@pytest.fixture(name="gerrydbtable") -def gerrydbtable_fixture(session: Session): +@pytest.fixture(name="ks_demo_view_census_no_pop_blocks_districtrmap") +def ks_demo_view_census_blocks_no_pop_districtrmap_fixture( + session: Session, ks_demo_view_census_blocks_no_pop: None +): upsert_query = text(""" INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) @@ -190,22 +146,29 @@ def gerrydbtable_fixture(session: Session): DO UPDATE SET updated_at = now() """) + session.begin() - session.execute(upsert_query, {"name": GERRY_DB_FIXTURE_NAME}) + session.execute(upsert_query, {"name": GERRY_DB_NO_POP_FIXTURE_NAME}) + create_districtr_map( + session=session, + name=f"Districtr map {GERRY_DB_NO_POP_FIXTURE_NAME}", + gerrydb_table_name=GERRY_DB_NO_POP_FIXTURE_NAME, + parent_layer_name=GERRY_DB_NO_POP_FIXTURE_NAME, + ) session.commit() -@pytest.fixture(name="second_gerrydbtable") -def second_gerrydbtable_fixture(session: Session): - upsert_query = text(""" - INSERT INTO gerrydbtable (uuid, name, updated_at) - VALUES (gen_random_uuid(), :name, now()) - ON CONFLICT (name) - DO UPDATE SET - updated_at = now() - """) - session.begin() - session.execute(upsert_query, {"name": "bleh"}) +@pytest.fixture(name="districtr_maps") +def districtr_map_fixtures( + session: Session, ks_demo_view_census_blocks_districtrmap: None +): + for i in range(4): + create_districtr_map( + session=session, + name=f"Districtr map {i}", + gerrydb_table_name=f"districtr_map_{i}", + parent_layer_name=GERRY_DB_FIXTURE_NAME, + ) session.commit() @@ -233,18 +196,6 @@ def document_total_vap_fixture(client): return document_id -@pytest.fixture(name="document_no_gerrydb_id") -def document_no_gerrydb_fixture(client): - response = client.post( - "/api/create_document", - json={ - "gerrydb_table": None, - }, - ) - document_id = response.json()["document_id"] - return document_id - - @pytest.fixture(name="document_no_gerrydb_pop") def document_no_gerrydb_pop_fixture(client): response = client.post( @@ -290,23 +241,6 @@ def assignments_total_vap_fixture(client, document_id_total_vap): return document_id -@pytest.fixture(name="assignments_document_no_gerrydb_id") -def assignments_no_gerrydb_fixture(client, document_no_gerrydb_id): - document_id = document_no_gerrydb_id - response = client.patch( - "/api/update_assignments", - json={ - "assignments": [ - {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, - {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, - {"document_id": document_id, "geo_id": "202090434001003", "zone": 2}, - ] - }, - ) - assert response.status_code == 200 - return document_id - - @pytest.fixture(name="assignments_document_no_gerrydb_pop_id") def assignments_no_gerrydb_pop_fixture(client, document_no_gerrydb_pop): document_id = document_no_gerrydb_pop @@ -330,6 +264,23 @@ def test_db_is_alive(client): assert response.json() == {"message": "DB is alive"} +def test_setup( + client, + districtr_maps, + ks_demo_view_census_blocks_districtrmap, + ks_demo_view_census_total_vap_blocks_districtrmap, + ks_demo_view_census_no_pop_blocks_districtrmap, +): + """ + This is a really ugly way of setting up fixtures that can result in + integrity errors due esp. from unique violations. + + TODO: Really we should run all tests in rollbacked transactions so the dev + doesn't need to think about the global state of the database between tests. + """ + pass + + def test_new_document(client): response = client.post( "/api/create_document", @@ -353,19 +304,7 @@ def test_get_document(client, document_id): assert data.get("gerrydb_table") == GERRY_DB_FIXTURE_NAME assert data.get("updated_at") assert data.get("created_at") - assert data.get("tiles_s3_path") is None - - -def test_patch_document(client, document_id): - response = client.patch( - f"/api/update_document/{document_id}", - json={"gerrydb_table": "foo"}, - ) - assert response.status_code == 200 - data = response.json() - assert data.get("document_id") == document_id - assert data.get("gerrydb_table") == "foo" - assert data.get("updated_at") + # assert data.get("tiles_s3_path") is None def test_patch_assignments(client, document_id): @@ -435,6 +374,28 @@ def test_patch_assignments_twice(client, document_id): assert data[1]["geo_id"] == "202090434001003" +def test_get_document_population_totals_null_assignments( + client, document_id, ks_demo_view_census_blocks +): + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": None}, + ] + }, + ) + assert response.status_code == 200 + assert response.json() == {"assignments_upserted": 3} + doc_uuid = str(uuid.UUID(document_id)) + result = client.get(f"/api/document/{doc_uuid}/total_pop") + assert result.status_code == 200 + data = result.json() + assert data == [{"zone": 1, "total_pop": 67}] + + def test_get_document_population_totals( client, assignments_document_id, ks_demo_view_census_blocks ): @@ -481,15 +442,6 @@ def test_get_document_vap_totals( assert data == [{"zone": 1, "total_pop": 67}, {"zone": 2, "total_pop": 130}] -def test_get_document_population_totals_no_gerrydb_view( - client, assignments_document_no_gerrydb_id -): - doc_uuid = str(uuid.UUID(assignments_document_no_gerrydb_id)) - result = client.get(f"/api/document/{doc_uuid}/total_pop") - assert result.status_code == 404 - assert result.json() == {"detail": f"Document with ID {doc_uuid} not found"} - - def test_get_document_population_totals_no_gerrydb_pop_view( client, assignments_document_no_gerrydb_pop_id, ks_demo_view_census_blocks ): @@ -499,32 +451,29 @@ def test_get_document_population_totals_no_gerrydb_pop_view( assert result.json() == {"detail": "Population column not found in GerryDB view"} -def test_list_gerydb_views(client, gerrydbtable): +def test_list_gerydb_views(client): response = client.get("/api/gerrydb/views") assert response.status_code == 200 data = response.json() - assert len(data) == 1 - assert data[0]["name"] == GERRY_DB_FIXTURE_NAME + assert len(data) == 7 -def test_list_gerydb_views_limit(client, gerrydbtable): +def test_list_gerydb_views_limit(client): response = client.get("/api/gerrydb/views?limit=0") assert response.status_code == 200 data = response.json() assert len(data) == 0 -def test_list_gerydb_views_offset(client, gerrydbtable, second_gerrydbtable): +def test_list_gerydb_views_offset(client): response = client.get("/api/gerrydb/views?offset=1") assert response.status_code == 200 data = response.json() - assert len(data) == 1 - assert data[0]["name"] == "bleh" + assert len(data) == 6 -def test_list_gerydb_views_offset_and_limit(client, gerrydbtable, second_gerrydbtable): +def test_list_gerydb_views_offset_and_limit(client): response = client.get("/api/gerrydb/views?offset=1&limit=1") assert response.status_code == 200 data = response.json() assert len(data) == 1 - assert data[0]["name"] == "bleh" diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py new file mode 100644 index 000000000..cebda92a4 --- /dev/null +++ b/backend/tests/test_utils.py @@ -0,0 +1,208 @@ +import pytest +import os +from app.utils import ( + create_districtr_map, + create_shatterable_gerrydb_view, + create_parent_child_edges, +) +from sqlmodel import Session +import subprocess +from app.constants import GERRY_DB_SCHEMA +from tests.constants import OGR2OGR_PG_CONNECTION_STRING, FIXTURES_PATH +from sqlalchemy import text + + +@pytest.fixture(name="simple_parent_geos") +def simple_parent_geos_fixture(session: Session): + layer = "simple_parent_geos" + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema + "-nlt", + "MULTIPOLYGON", + "-lco", + "GEOMETRY_NAME=geometry", + ], + ) + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="simple_child_geos") +def simple_child_geos_fixture(session: Session): + layer = "simple_child_geos" + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema + "-nlt", + "MULTIPOLYGON", + "-lco", + "GEOMETRY_NAME=geometry", + ], + ) + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="simple_parent_geos_gerrydb") +def simple_parent_geos_gerrydb_fixture(session: Session, simple_parent_geos): + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + session.begin() + session.execute(upsert_query, {"name": "simple_parent_geos"}) + session.commit() + session.close() + + +@pytest.fixture(name="simple_child_geos_gerrydb") +def simple_child_geos_gerrydb_fixture(session: Session, simple_child_geos): + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + session.begin() + session.execute(upsert_query, {"name": "simple_child_geos"}) + session.commit() + session.close() + + +@pytest.fixture(name="gerrydb_simple_geos_view") +def gerrydb_simple_geos_view_fixture( + session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb +): + create_shatterable_gerrydb_view( + session, + parent_layer_name="simple_parent_geos", + child_layer_name="simple_child_geos", + gerrydb_table_name="simple_geos", + ) + session.commit() + return + + +@pytest.fixture(name="districtr_map") +def districtr_map_fixture( + session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb +): + (inserted_districtr_map,) = create_districtr_map( + session, + name="Simple shatterable layer", + gerrydb_table_name="simple_geos", + num_districts=10, + tiles_s3_path="tilesets/simple_shatterable_layer.pmtiles", + parent_layer_name="simple_parent_geos", + child_layer_name="simple_child_geos", + ) + session.commit() + return inserted_districtr_map + + +# FOR THE TESTS BELOW I NEED TO ADD ACTUAL ASSERTIONS + + +def test_create_districtr_map( + session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb +): + (inserted_districtr_map,) = create_districtr_map( + session, + name="Simple shatterable layer", + gerrydb_table_name="simple_geos_test", + num_districts=10, + tiles_s3_path="tilesets/simple_shatterable_layer.pmtiles", + parent_layer_name="simple_parent_geos", + child_layer_name="simple_child_geos", + ) + session.commit() + + +def test_create_districtr_map_some_nulls( + session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb +): + # This is also an example of a districtr map before other set-up operations + # are performed, such as creating a tileset and a shatterable view + (inserted_districtr_map,) = create_districtr_map( + session, + name="Simple non-shatterable layer", + gerrydb_table_name="simple_parent_geos_some_nulls", + parent_layer_name="simple_parent_geos", + ) + session.commit() + + +def test_create_shatterable_gerrydb_view( + session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb +): + create_shatterable_gerrydb_view( + session, + parent_layer_name="simple_parent_geos", + child_layer_name="simple_child_geos", + gerrydb_table_name="simple_geos_test", + ) + session.commit() + + +def test_create_parent_child_edges( + session: Session, districtr_map: str, gerrydb_simple_geos_view +): + create_parent_child_edges(session=session, districtr_map_uuid=districtr_map) + session.commit() + + +def test_shattering(client): + # Set-up + response = client.post( + "/api/create_document", + json={ + "gerrydb_table": "simple_geos", + }, + ) + assert response.status_code == 201 + doc = response.json() + document_id = doc["document_id"] + + response = client.patch( + "/api/update_assignments", + json={"assignments": [{"document_id": document_id, "geo_id": "A", "zone": 1}]}, + ) + assert response.status_code == 200 + + # Test + response = client.patch( + f"/api/update_assignments/{document_id}/shatter_parents", json={"geoids": ["A"]} + ) + assert response.status_code == 200 + data = response.json() + assert len(data["parents"]) == 1 + assert data["parents"]["geoids"][0] == "A" + assert len(data["children"]) == 2 + assert len({d["document_id"] for d in data["children"]}) == 1 + assert {d["geo_id"] for d in data["children"]} == {"a", "e"} + assert all(d["zone"] == 1 for d in data["children"]) diff --git a/docker-compose.yml b/docker-compose.yml index fe52aa450..c39585b5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,7 @@ services: dockerfile: Dockerfile.dev container_name: backend volumes: - - ./backend/app:/districtr-backend/app # Adjust volumes to point to the backend folder - - ./backend/scripts:/districtr-backend/scripts # Ensure scripts are mounted from backend directory + - ./backend:/districtr-backend # Adjust volumes to point to the backend folder - ./sample_data:/districtr-backend/sample_data env_file: @@ -18,11 +17,7 @@ services: db: condition: service_healthy command: bash -c " - until alembic upgrade head; do - echo 'Alembic failed, retrying in 5 seconds...' - sleep 5; - done && - python3 scripts/load_data.py && + python3 load_data.py && uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload --reload-exclude '.venv/**/*.py' " ports: diff --git a/pipelines/simple_elt/README.md b/pipelines/simple_elt/README.md index 3bbf94247..ad36fe86c 100644 --- a/pipelines/simple_elt/README.md +++ b/pipelines/simple_elt/README.md @@ -12,14 +12,16 @@ Dependencies are managed with uv as noted in the root README. Follow set-up inst ### Geospatial libraries +#### GDAL + Follow the [installation instructions](https://docs.djangoproject.com/en/5.0/ref/contrib/gis/install/geolibs/) for GeoDjango. Although we are not using Django, the instructions are super useful / kept up-to-date. You'll need `ogr2ogr` installed, part of GDAL. You can test that it was installed properly with `which ogr2ogr` or `ogr2ogr --version`. -### DuckDB +#### DuckDB Follow [DuckDB installation instructions](https://duckdb.org/docs/installation/) -### Tippecanoe +#### Tippecanoe Follow [Tippecanoe installation instructions](https://github.com/felt/tippecanoe?tab=readme-ov-file#installation). diff --git a/pipelines/simple_elt/files.py b/pipelines/simple_elt/files.py index dc9534e4b..60869d911 100644 --- a/pipelines/simple_elt/files.py +++ b/pipelines/simple_elt/files.py @@ -1,7 +1,48 @@ +import os +import zipfile from urllib.request import urlretrieve -from urllib.parse import urlparse +from urllib.parse import urlparse, ParseResult from pathlib import Path -import zipfile +from settings import settings + +import logging + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + + +def download_file_from_s3(s3, url: ParseResult, replace=False) -> str: + """ + Download a file from S3 to the local volume path. + + Args: + s3: S3 client + url (ParseResult): URL of the file to download + replace (bool): If True, replace the file if it already exists + + Returns the path to the downloaded file. + """ + if not s3: + raise ValueError("S3 client is not available") + + file_name = url.path.lstrip("/") + LOGGER.debug("File name: %s", file_name) + object_information = s3.head_object(Bucket=url.netloc, Key=file_name) + LOGGER.debug("Object information: %s", object_information) + + if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: + raise ValueError( + f"GeoPackage file {file_name} not found in S3 bucket {url.netloc}" + ) + + path = os.path.join(settings.OUT_SCRATCH, file_name) + LOGGER.debug("Path: %s", path) + + if not os.path.exists(path) or replace: + LOGGER.debug("Downloading file...") + s3.download_file(url.netloc, file_name, path) + + return path def download_and_unzip_zipfile(zip_file_url: str, out_dir: Path | str) -> Path: diff --git a/pipelines/simple_elt/main.py b/pipelines/simple_elt/main.py index ff0802bd9..268a24c27 100644 --- a/pipelines/simple_elt/main.py +++ b/pipelines/simple_elt/main.py @@ -7,8 +7,9 @@ import logging from urllib.parse import urlparse from subprocess import run +from typing import Iterable -from files import download_and_unzip_zipfile, exists_in_s3 +from files import download_and_unzip_zipfile, exists_in_s3, download_file_from_s3 from settings import settings TIGER_YEAR = 2023 @@ -168,5 +169,147 @@ def wi_blocks(): ) +@cli.command("create-gerrydb-tileset") +@click.option( + "--layer", "-n", help="Name of the layer in the gerrydb view to load", required=True +) +@click.option( + "--gpkg", + "-g", + help="Path or URL to GeoPackage file. If URL, must be s3 URI", + required=True, +) +@click.option("--replace", "-f", help="Replace files they exist", is_flag=True) +@click.option( + "--column", + "-c", + help="Column to include in tileset", + multiple=True, + default=[ + "path", + "geography", + "total_pop", + ], +) +def create_gerrydb_tileset( + layer: str, gpkg: str, replace: bool, column: Iterable[str] +) -> None: + """ + Create a tileset from a GeoPackage file. Does not upload the tileset to S3. Use the s3 cli for that. + """ + LOGGER.info("Creating GerryDB tileset...") + s3 = settings.get_s3_client() + + url = urlparse(gpkg) + LOGGER.info("URL: %s", url) + + path = gpkg + + if url.scheme == "s3": + assert s3, "S3 client is not available" + path = download_file_from_s3(s3, url, replace) + + fbg_path = f"{settings.OUT_SCRATCH}/{layer}.fgb" + LOGGER.info("Creating flatgeobuf...") + if os.path.exists(fbg_path) and not replace: + LOGGER.info("File already exists. Skipping creation.") + else: + result = run( + args=[ + "ogr2ogr", + "-f", + "FlatGeobuf", + "-select", + ",".join(column), + "-t_srs", + "EPSG:4326", + fbg_path, + path, + layer, + ] + ) + + if result.returncode != 0: + LOGGER.error("ogr2ogr failed. Got %s", result) + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + LOGGER.info("Creating tileset...") + tileset_path = f"{settings.OUT_SCRATCH}/{layer}.pmtiles" + + args = [ + "tippecanoe", + "-zg", + "--coalesce-smallest-as-needed", + "--extend-zooms-if-still-dropping", + "-o", + tileset_path, + "-l", + layer, + fbg_path, + ] + if replace: + args.append("--force") + + result = run(args=args) + + if result.returncode != 0: + LOGGER.error("tippecanoe failed. Got %s", result) + raise ValueError(f"tippecanoe failed with return code {result.returncode}") + + +@cli.command("merge-gerrydb-tilesets") +@click.option("--out-name", "-o", help="Name of the output tileset", required=True) +@click.option( + "--parent-layer", + help="Path to the parent layer to load. Can be an S3 URI", + required=True, +) +@click.option( + "--child-layer", + help="Path to the child layer to load. Can be an S3 URI", + required=True, +) +@click.option("--replace", "-f", help="Replace files they exist", is_flag=True) +def merge_gerrydb_tilesets( + out_name: str, parent_layer: str, child_layer: str, replace: bool +) -> None: + """ + Merge two tilesets. Does not upload the tileset to S3. Use the s3 cli for that. + """ + LOGGER.info("Merging GerryDB tilesets...") + + s3 = settings.get_s3_client() + + parent_url = urlparse(parent_layer) + LOGGER.info("Parent URL: %s", parent_url) + + parent_path = parent_layer + + if parent_url.scheme == "s3": + assert s3, "S3 client is not available" + parent_path = download_file_from_s3(s3, parent_url, replace) + + child_url = urlparse(child_layer) + LOGGER.info("Child URL: %s", child_url) + + child_path = child_layer + + if child_url.scheme == "s3": + assert s3, "S3 client is not available" + child_path = download_file_from_s3(s3, child_url, replace) + + run( + [ + "tile-join", + "-o", + f"{settings.OUT_SCRATCH}/{out_name}.pmtiles", + parent_path, + child_path, + "--no-tile-size-limit", + "--force", + ] + ) + + if __name__ == "__main__": cli() diff --git a/pipelines/simple_elt/requirements.txt b/pipelines/simple_elt/requirements.txt index 06276e3f1..2c4596d70 100644 --- a/pipelines/simple_elt/requirements.txt +++ b/pipelines/simple_elt/requirements.txt @@ -1,33 +1,87 @@ +# This file was generated by uv via the following command: +# uv pip freeze | uv pip compile - -o requirements.txt annotated-types==0.7.0 + # via pydantic attrs==23.2.0 + # via fiona boto3==1.35.5 botocore==1.35.5 + # via + # boto3 + # s3transfer certifi==2024.2.2 + # via + # fiona + # pyogrio + # pyproj + # requests charset-normalizer==3.3.2 + # via requests click==8.1.7 + # via + # click-plugins + # cligj + # fiona click-plugins==1.1.1 + # via fiona cligj==0.7.2 + # via fiona duckdb==1.0.0 fiona==1.9.6 + # via geopandas geopandas==0.14.4 idna==3.7 + # via requests jmespath==1.0.1 + # via + # boto3 + # botocore numpy==1.26.4 + # via + # geopandas + # pandas + # pyarrow + # pyogrio + # shapely packaging==24.0 + # via + # geopandas + # pyogrio pandas==2.2.2 + # via geopandas pyarrow==16.1.0 pydantic==2.8.2 + # via pydantic-settings pydantic-core==2.20.1 + # via pydantic pydantic-settings==2.4.0 pyogrio==0.8.0 pyproj==3.6.1 + # via geopandas python-dateutil==2.9.0.post0 + # via + # botocore + # pandas python-dotenv==1.0.1 + # via pydantic-settings pytz==2024.1 + # via pandas requests==2.32.3 s3transfer==0.10.2 + # via boto3 shapely==2.0.4 + # via geopandas six==1.16.0 + # via + # fiona + # python-dateutil typing-extensions==4.12.2 + # via + # pydantic + # pydantic-core tzdata==2024.1 + # via pandas urllib3==2.2.1 + # via + # botocore + # requests From 2c0826b03dbd8b4c97662cf85bc5173398ff6cdf Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 17 Oct 2024 09:02:18 -0500 Subject: [PATCH 07/50] State enhancements: serialize and devtools (#125) --- app/src/app/components/Map.tsx | 4 +- app/src/app/components/sidebar/Layers.tsx | 4 +- .../app/components/sidebar/PaintByCounty.tsx | 6 +- app/src/app/constants/layers.ts | 40 ++--- app/src/app/constants/types.ts | 3 +- app/src/app/store/mapEditSubs.ts | 21 +-- app/src/app/store/mapRenderSubs.ts | 32 ++-- app/src/app/store/mapStore.ts | 52 ++++--- app/src/app/utils/api/queries.ts | 30 +++- app/src/app/utils/api/queryParamsListener.ts | 10 ++ app/src/app/utils/arrays.ts | 3 + app/src/app/utils/events/handlers.ts | 145 ++++-------------- app/src/app/utils/events/mapEvents.ts | 37 +++-- app/src/app/utils/helpers.ts | 80 +++++----- 14 files changed, 216 insertions(+), 251 deletions(-) create mode 100644 app/src/app/utils/arrays.ts diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index 552c3cc55..cb522fe04 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -54,7 +54,7 @@ export const MapComponent: React.FC = () => { action.action as keyof MapLayerEventType, layer, // to be updated with the scale-agnostic layer id (e: MapLayerMouseEvent | MapLayerTouchEvent) => { - action.handler(e, map); + action.handler(e, map.current); } ); } @@ -64,7 +64,7 @@ export const MapComponent: React.FC = () => { return () => { mapEvents.forEach((action) => { map.current?.off(action.action, (e) => { - action.handler(e, map); + action.handler(e, map.current); }); }); }; diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index 9a5afe981..7ba84ee49 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -16,13 +16,13 @@ import { toggleLayerVisibility } from "../../utils/helpers"; * - Support tribes and communities */ export default function Layers() { - const mapRef = useMapStore((state) => state.mapRef); + const mapRef = useMapStore((state) => state.getMapRef()); const mapDocument = useMapStore((state) => state.mapDocument); const visibleLayerIds = useMapStore((state) => state.visibleLayerIds); const updateVisibleLayerIds = useMapStore((state) => state.updateVisibleLayerIds); const toggleLayers = (layerIds: string[]) => { - if (!mapRef || !mapRef?.current) return; + if (!mapRef) return; const layerUpdates = toggleLayerVisibility(mapRef, layerIds); updateVisibleLayerIds(layerUpdates); }; diff --git a/app/src/app/components/sidebar/PaintByCounty.tsx b/app/src/app/components/sidebar/PaintByCounty.tsx index f578118f9..4dee72031 100644 --- a/app/src/app/components/sidebar/PaintByCounty.tsx +++ b/app/src/app/components/sidebar/PaintByCounty.tsx @@ -8,17 +8,17 @@ import { } from "../../utils/helpers"; export default function PaintByCounty() { - const mapRef = useMapStore((state) => state.mapRef); + const mapRef = useMapStore((state) => state.getMapRef()); const addVisibleLayerIds = useMapStore((state) => state.addVisibleLayerIds); const setPaintFunction = useMapStore((state) => state.setPaintFunction); const [checked, setChecked] = useState(false); useEffect(() => { - if (!mapRef || !mapRef.current) return; + if (!mapRef) return; if (checked) { COUNTY_LAYER_IDS.forEach((layerId) => { - mapRef.current?.setLayoutProperty(layerId, "visibility", "visible"); + mapRef.setLayoutProperty(layerId, "visibility", "visible"); }); addVisibleLayerIds(COUNTY_LAYER_IDS); setPaintFunction(getFeaturesIntersectingCounties); diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 508a3a9fc..9cf524a5d 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -3,7 +3,6 @@ import { FilterSpecification, LayerSpecification, } from "maplibre-gl"; -import { MutableRefObject } from "react"; import { Map } from "maplibre-gl"; import { getBlocksSource } from "./sources"; import { DocumentObject } from "../utils/api/apiHandlers"; @@ -165,21 +164,21 @@ export function getBlocksHoverLayerSpecification( } const addBlockLayers = ( - map: MutableRefObject, + map: Map | null, mapDocument: DocumentObject, ) => { - if (!map.current || !mapDocument.tiles_s3_path) { + if (!map || !mapDocument.tiles_s3_path) { console.log("map or mapDocument not ready", mapDocument); return; } const blockSource = getBlocksSource(mapDocument.tiles_s3_path); removeBlockLayers(map); - map.current?.addSource(BLOCK_SOURCE_ID, blockSource); - map.current?.addLayer( + map?.addSource(BLOCK_SOURCE_ID, blockSource); + map?.addLayer( getBlocksLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID), LABELS_BREAK_LAYER_ID, ); - map.current?.addLayer( + map?.addLayer( getBlocksHoverLayerSpecification( mapDocument.parent_layer, BLOCK_HOVER_LAYER_ID, @@ -187,14 +186,14 @@ const addBlockLayers = ( LABELS_BREAK_LAYER_ID, ); if (mapDocument.child_layer) { - map.current?.addLayer( + map?.addLayer( getBlocksLayerSpecification( mapDocument.child_layer, BLOCK_LAYER_ID_CHILD, ), LABELS_BREAK_LAYER_ID, ); - map.current?.addLayer( + map?.addLayer( getBlocksHoverLayerSpecification( mapDocument.child_layer, BLOCK_HOVER_LAYER_ID_CHILD, @@ -205,22 +204,25 @@ const addBlockLayers = ( useMapStore.getState().setMapRenderingState("loaded") }; -export function removeBlockLayers(map: MutableRefObject) { +export function removeBlockLayers(map: Map | null) { + if (!map) { + return + } useMapStore.getState().setMapRenderingState("loading") - if (map.current?.getLayer(BLOCK_LAYER_ID)) { - map.current?.removeLayer(BLOCK_LAYER_ID); + if (map.getLayer(BLOCK_LAYER_ID)) { + map.removeLayer(BLOCK_LAYER_ID); } - if (map.current?.getLayer(BLOCK_HOVER_LAYER_ID)) { - map.current?.removeLayer(BLOCK_HOVER_LAYER_ID); + if (map.getLayer(BLOCK_HOVER_LAYER_ID)) { + map.removeLayer(BLOCK_HOVER_LAYER_ID); } - if (map.current?.getLayer(BLOCK_LAYER_ID_CHILD)) { - map.current?.removeLayer(BLOCK_LAYER_ID_CHILD); + if (map.getLayer(BLOCK_LAYER_ID_CHILD)) { + map.removeLayer(BLOCK_LAYER_ID_CHILD); } - if (map.current?.getLayer(BLOCK_HOVER_LAYER_ID_CHILD)) { - map.current?.removeLayer(BLOCK_HOVER_LAYER_ID_CHILD); + if (map.getLayer(BLOCK_HOVER_LAYER_ID_CHILD)) { + map.removeLayer(BLOCK_HOVER_LAYER_ID_CHILD); } - if (map.current?.getSource(BLOCK_SOURCE_ID)) { - map.current?.removeSource(BLOCK_SOURCE_ID); + if (map.getSource(BLOCK_SOURCE_ID)) { + map.removeSource(BLOCK_SOURCE_ID); } } diff --git a/app/src/app/constants/types.ts b/app/src/app/constants/types.ts index c1b70628a..1073d5130 100644 --- a/app/src/app/constants/types.ts +++ b/app/src/app/constants/types.ts @@ -1,6 +1,7 @@ import type { MapOptions, MapLibreEvent } from "maplibre-gl"; -export type Zone = number | null; +export type Zone = number; +export type NullableZone = Zone | null export type GEOID = string; diff --git a/app/src/app/store/mapEditSubs.ts b/app/src/app/store/mapEditSubs.ts index e91fcff43..cd3741735 100644 --- a/app/src/app/store/mapEditSubs.ts +++ b/app/src/app/store/mapEditSubs.ts @@ -7,15 +7,16 @@ import { import { patchUpdates } from "../utils/api/mutations"; import { useMapStore as _useMapStore, MapStore } from "./mapStore"; import { shallowCompareArray } from "../utils/helpers"; +import { updateAssignments } from "../utils/api/queries"; const zoneUpdates = ({ - mapRef, + getMapRef, zoneAssignments, appLoadingState, }: Partial) => { if ( - mapRef?.current && - zoneAssignments?.size && + getMapRef?.() && + (zoneAssignments?.size) && appLoadingState === "loaded" ) { const assignments = FormatAssignments(); @@ -26,24 +27,18 @@ const debouncedZoneUpdate = debounce(zoneUpdates, 25); export const getMapEditSubs = (useMapStore: typeof _useMapStore) => { const sendZonesOnMapRefSub = useMapStore.subscribe( - (state) => [state.mapRef, state.zoneAssignments], + (state) => [state.getMapRef, state.zoneAssignments], () => { - const { mapRef, zoneAssignments, appLoadingState } = + const { getMapRef, zoneAssignments, appLoadingState } = useMapStore.getState(); - debouncedZoneUpdate({ mapRef, zoneAssignments, appLoadingState }); + debouncedZoneUpdate({ getMapRef, zoneAssignments, appLoadingState }); }, { equalityFn: shallowCompareArray} ); const fetchAssignmentsSub = useMapStore.subscribe( (state) => state.mapDocument, - (mapDocument) => { - if (mapDocument) { - getAssignments(mapDocument).then((res: Assignment[]) => { - useMapStore.getState().loadZoneAssignments(res); - }); - } - } + (mapDocument) => mapDocument && updateAssignments(mapDocument) ); return [sendZonesOnMapRefSub, fetchAssignmentsSub]; diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index caff6a2a7..8762c85f1 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -15,12 +15,13 @@ import { useMapStore as _useMapStore, MapStore } from "./mapStore"; export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { const addLayerSubMapDocument = useMapStore.subscribe< - [MapStore["mapDocument"], MapStore["mapRef"]] + [MapStore["mapDocument"], MapStore["getMapRef"]] >( - (state) => [state.mapDocument, state.mapRef], - ([mapDocument, mapRef]) => { + (state) => [state.mapDocument, state.getMapRef], + ([mapDocument, getMapRef]) => { const mapStore = useMapStore.getState(); - if (mapRef?.current && mapDocument) { + const mapRef = getMapRef() + if (mapRef && mapDocument) { addBlockLayers(mapRef, mapDocument); mapStore.addVisibleLayerIds([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]); } @@ -29,14 +30,15 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ); const _shatterMapSideEffectRender = useMapStore.subscribe< - [MapStore["shatterIds"], MapStore["mapRef"], MapStore["mapRenderingState"]] + [MapStore["shatterIds"], MapStore["getMapRef"], MapStore["mapRenderingState"]] >( - (state) => [state.shatterIds, state.mapRef, state.mapRenderingState], - ([shatterIds, mapRef, mapRenderingState]) => { + (state) => [state.shatterIds, state.getMapRef, state.mapRenderingState], + ([shatterIds, getMapRef, mapRenderingState]) => { const state = useMapStore.getState(); + const mapRef = getMapRef() const setMapLock = state.setMapLock; - if (!mapRef?.current || mapRenderingState !== "loaded") { + if (!mapRef || mapRenderingState !== "loaded") { return; } @@ -45,10 +47,10 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { if (state.mapDocument?.child_layer) layersToFilter.push(...CHILD_LAYERS); layersToFilter.forEach((layerId) => - mapRef.current?.setFilter(layerId, getLayerFilter(layerId, shatterIds)), + mapRef.setFilter(layerId, getLayerFilter(layerId, shatterIds)), ); - mapRef.current.once("render", () => { + mapRef.once("render", () => { setMapLock(false); console.log(`Unlocked at`, performance.now()); }); @@ -59,18 +61,18 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { const _hoverMapSideEffectRender = useMapStore.subscribe( (state) => state.hoverFeatures, (hoverFeatures, previousHoverFeatures) => { - const mapRef = useMapStore.getState().mapRef; + const mapRef = useMapStore.getState().getMapRef(); - if (!mapRef?.current) { + if (!mapRef) { return; } previousHoverFeatures.forEach((feature) => { - mapRef.current?.setFeatureState(feature, { hover: false }); + mapRef.setFeatureState(feature, { hover: false }); }); hoverFeatures.forEach((feature) => { - mapRef.current?.setFeatureState(feature, { hover: true }); + mapRef.setFeatureState(feature, { hover: true }); }); }, ); @@ -80,7 +82,7 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { (state) => [ state.zoneAssignments, state.mapDocument, - state.mapRef, + state.getMapRef, state.shatterIds, state.appLoadingState, state.mapRenderingState, diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index bd281670e..8f66c9df9 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -1,9 +1,10 @@ import type { MapGeoJSONFeature, MapOptions } from "maplibre-gl"; import { create } from "zustand"; -import { subscribeWithSelector } from "zustand/middleware"; +import { devtools, subscribeWithSelector } from "zustand/middleware"; import type { ActiveTool, MapFeatureInfo, + NullableZone, SpatialUnit, } from "../constants/types"; import { Zone, GDBPath } from "../constants/types"; @@ -28,12 +29,16 @@ import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; import { getMapMetricsSubs } from "./metricsSubs"; import { getMapEditSubs } from "./mapEditSubs"; + +const prodWrapper: typeof devtools = (store: any) => store +const devwrapper = process.env.NODE_ENV === 'development' ? devtools : prodWrapper + export interface MapStore { appLoadingState: "loaded" | "initializing" | "loading"; setAppLoadingState: (state: MapStore["appLoadingState"]) => void; mapRenderingState: "loaded" | "initializing" | "loading"; setMapRenderingState: (state: MapStore["mapRenderingState"]) => void; - mapRef: MutableRefObject | null; + getMapRef: () => maplibregl.Map | null; setMapRef: (map: MutableRefObject) => void; mapLock: boolean; setMapLock: (lock: boolean) => void; @@ -63,13 +68,14 @@ export interface MapStore { setSelectedZone: (zone: Zone) => void; accumulatedBlockPopulations: Map; resetAccumulatedBlockPopulations: () => void; - zoneAssignments: Map; // geoid -> zone - setZoneAssignments: (zone: Zone, gdbPaths: Set) => void; + zoneAssignments: Map; // geoid -> zone + setZoneAssignments: (zone: NullableZone, gdbPaths: Set) => void; loadZoneAssignments: (assigments: Assignment[]) => void; resetZoneAssignments: () => void; zonePopulations: Map; setZonePopulations: (zone: Zone, population: number) => void; accumulatedGeoids: Set; + setAccumulatedGeoids: (geoids: MapStore['accumulatedGeoids']) => void; brushSize: number; setBrushSize: (size: number) => void; isPainting: boolean; @@ -97,33 +103,38 @@ const initialLoadingState = typeof window !== 'undefined' && new URLSearchParams ? "loading" : "initializing"; -export const useMapStore = create( +export const useMapStore = create(devtools( subscribeWithSelector((set, get) => ({ appLoadingState: initialLoadingState, setAppLoadingState: (appLoadingState) => set({ appLoadingState }), mapRenderingState: "initializing", setMapRenderingState: (mapRenderingState) => set({ mapRenderingState }), - mapRef: null, - setMapRef: (mapRef) => + getMapRef: () => null, + setMapRef: (mapRef) => { set({ - mapRef, + getMapRef: () => mapRef.current, appLoadingState: - initialLoadingState === "initializing" + initialLoadingState === "initializing" ? "loaded" : get().appLoadingState, - }), + }) + }, mapLock: false, setMapLock: (mapLock) => set({ mapLock }), mapDocument: null, - setMapDocument: (mapDocument) => - set((state) => { - state.setFreshMap(true); - state.resetZoneAssignments(); - return { - mapDocument: mapDocument, - shatterIds: { parents: new Set(), children: new Set() }, - }; - }), + setMapDocument: (mapDocument) => { + const currentMapDocument = get().mapDocument + if (currentMapDocument?.document_id === mapDocument.document_id) { + return + } + get().setFreshMap(true); + get().resetZoneAssignments(); + + set({ + mapDocument: mapDocument, + shatterIds: { parents: new Set(), children: new Set() }, + }); + }, shatterIds: { parents: new Set(), children: new Set(), @@ -222,6 +233,7 @@ export const useMapStore = create( setSelectedZone: (zone) => set({ selectedZone: zone }), zoneAssignments: new Map(), accumulatedGeoids: new Set(), + setAccumulatedGeoids: (accumulatedGeoids) => set({accumulatedGeoids}), setZoneAssignments: (zone, geoids) => { const zoneAssignments = get().zoneAssignments; const newZoneAssignments = new Map(zoneAssignments); @@ -304,7 +316,7 @@ export const useMapStore = create( contextMenu: null, setContextMenu: (contextMenu) => set({ contextMenu }), })) -); +)); // these need to initialize after the map store getRenderSubscriptions(useMapStore); diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index 99b7f0b54..b5d04db1e 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -1,12 +1,14 @@ import { QueryObserver, skipToken } from "@tanstack/react-query"; import { queryClient } from "./queryClient"; import { + Assignment, DocumentObject, + getAssignments, getDocument, getZonePopulations, ZonePopulation, } from "./apiHandlers"; -import { useMapStore } from "@/app/store/mapStore"; +import { MapStore, useMapStore } from "@/app/store/mapStore"; export const mapMetrics = new QueryObserver(queryClient, { queryKey: ["_zonePopulations"], @@ -49,3 +51,29 @@ updateDocumentFromId.subscribe((mapDocument) => { } }); +const getFetchAssignmentsQuery = (mapDocument?: MapStore['mapDocument']) => { + if (!mapDocument) return () => null + return async () => await getAssignments(mapDocument) +} + +export const fetchAssignments = new QueryObserver( + queryClient, + { + queryKey: ["assignments"], + queryFn: getFetchAssignmentsQuery(), + } +) + +export const updateAssignments = (mapDocument: DocumentObject) => { + fetchAssignments.setOptions({ + queryFn: getFetchAssignmentsQuery(mapDocument), + queryKey: ['assignments', performance.now()] + }) +} + + +fetchAssignments.subscribe((assignments) => { + if (assignments.data) { + useMapStore.getState().loadZoneAssignments(assignments.data); + } +}); \ No newline at end of file diff --git a/app/src/app/utils/api/queryParamsListener.ts b/app/src/app/utils/api/queryParamsListener.ts index 47495f233..42e947dce 100644 --- a/app/src/app/utils/api/queryParamsListener.ts +++ b/app/src/app/utils/api/queryParamsListener.ts @@ -5,6 +5,16 @@ export const getSearchParamsObersver = () => { if (typeof window === "undefined") { return } + + // listener for tab refocus + const handleVisibilityChange = () => { + if (document.visibilityState === 'visible') { + updateDocumentFromId.refetch(); + } + }; + + document.addEventListener('visibilitychange', handleVisibilityChange); + let previousDocumentID = ""; const observer = new MutationObserver(() => { const documentId = new URLSearchParams(window.location.search).get( diff --git a/app/src/app/utils/arrays.ts b/app/src/app/utils/arrays.ts new file mode 100644 index 000000000..e6990774a --- /dev/null +++ b/app/src/app/utils/arrays.ts @@ -0,0 +1,3 @@ +export const onlyUnique = (value: unknown, index: number, self: unknown[]) => { + return self.indexOf(value) === index; +}; diff --git a/app/src/app/utils/events/handlers.ts b/app/src/app/utils/events/handlers.ts index af967610d..18c29e4c1 100644 --- a/app/src/app/utils/events/handlers.ts +++ b/app/src/app/utils/events/handlers.ts @@ -2,26 +2,26 @@ import { BLOCK_SOURCE_ID } from "@/app/constants/layers"; import { MutableRefObject } from "react"; import { Map, MapGeoJSONFeature } from "maplibre-gl"; import { debounce } from "lodash"; -import { Zone } from "@/app/constants/types"; +import { NullableZone, Zone } from "@/app/constants/types"; import { MapStore } from "@/app/store/mapStore"; /** * Debounced function to set zone assignments in the store without resetting the state every time the mouse moves (assuming onhover event). - * @param mapStoreRef - MutableRefObject, the zone store reference from zustand + * @param mapStoreRef - MapStore | null, the zone store reference from zustand * @param geoids - Set, the set of geoids to assign to the selected zone * @returns void - but updates the zoneAssignments and zonePopulations in the store */ const debouncedSetZoneAssignments = debounce( - (mapStoreRef: MapStore, selectedZone: Zone, geoids: Set) => { + (mapStoreRef: MapStore, selectedZone: NullableZone, geoids: Set) => { mapStoreRef.setZoneAssignments(selectedZone, geoids); const accumulatedBlockPopulations = mapStoreRef.accumulatedBlockPopulations; - const population = Array.from(accumulatedBlockPopulations.values()).reduce( + const population = Array.from(Object.values(accumulatedBlockPopulations)).reduce( (acc, val) => acc + Number(val), 0, ); - mapStoreRef.setZonePopulations(selectedZone, population); + selectedZone && mapStoreRef.setZonePopulations(selectedZone, population); }, 1, // 1ms debounce ); @@ -31,7 +31,7 @@ const debouncedSetZoneAssignments = debounce( * called using mapEvent handlers. * * @param features - Array of MapGeoJSONFeature from QueryRenderedFeatures - * @param map - MutableRefObject, the maplibre map instance + * @param map - Map | null, the maplibre map instance * @param mapStoreRef - MutableRefObject, the map store reference from zustand * @returns Promise - resolves after the function completes * Selects the features and sets the state of the map features to be selected. @@ -39,38 +39,31 @@ const debouncedSetZoneAssignments = debounce( * */ export const SelectMapFeatures = ( features: Array | undefined, - map: MutableRefObject, + map: Map | null, mapStoreRef: MapStore, ) => { - let { - accumulatedGeoids, - accumulatedBlockPopulations, - activeTool, - selectedZone, - } = mapStoreRef; - if (activeTool === "eraser") { - selectedZone = null; - } + if (map) { + let { accumulatedGeoids, accumulatedBlockPopulations, activeTool } = + mapStoreRef; + const selectedZone = + activeTool === "eraser" ? null : mapStoreRef.selectedZone; - features?.forEach((feature) => { - map.current?.setFeatureState( - { - source: BLOCK_SOURCE_ID, - id: feature?.id ?? undefined, - sourceLayer: feature.sourceLayer, - }, - { selected: true, zone: selectedZone }, - ); - }); - if (features?.length) { - features.forEach((feature) => { - accumulatedGeoids.add(feature.properties?.path); - - accumulatedBlockPopulations.set( - feature.properties?.path, - feature.properties?.total_pop, + features?.forEach((feature) => { + map.setFeatureState( + { + source: BLOCK_SOURCE_ID, + id: feature?.id ?? undefined, + sourceLayer: feature.sourceLayer, + }, + { selected: true, zone: selectedZone } ); }); + if (features?.length) { + features.forEach((feature) => { + accumulatedGeoids.add(feature.properties?.path); + accumulatedBlockPopulations.set(feature.properties?.path, feature.properties?.total_pop) + }); + } } return new Promise((resolve) => { // Resolve the Promise after the function completes @@ -97,100 +90,24 @@ export const SelectZoneAssignmentFeatures = (mapStoreRef: MapStore) => { } }; -/** - * Highlight features based on hover mouseEvent. called using `map.on("mousemove", "blocks-hover", ...)` pattern. - * @param features - Array of MapGeoJSONFeature from QueryRenderedFeatures - * @param map - MutableRefObject, the maplibre map instance - * @param hoverGeoids - MutableRefObject>, used to keep track of geoids that have been hovered over - * @deprecated This function is no longer in use and will be removed in a future version. - */ -export const HighlightFeature = ( - features: Array | undefined, - map: MutableRefObject, - hoverGeoids: MutableRefObject>, - sourceLayer: string, -) => { - if (features?.length) { - if (hoverGeoids.current.size) { - hoverGeoids.current.forEach((Id) => { - map.current?.setFeatureState( - { - source: BLOCK_SOURCE_ID, - id: Id, - sourceLayer: sourceLayer, - }, - { hover: false }, - ); - }); - hoverGeoids.current.clear(); - } - } - - features?.forEach((feature) => { - map.current?.setFeatureState( - { - source: BLOCK_SOURCE_ID, - id: feature.id ?? undefined, - sourceLayer: feature.sourceLayer, - }, - { hover: true }, - ); - }); - - if (features?.length) { - features.forEach((feature) => { - if (feature?.id) { - hoverGeoids.current.add(feature.id.toString()); - } - }); - } -}; - -/** - * Un-highlight features based on mouseleave event. - * called using `map.on("mouseleave", "blocks-hover", ...)` pattern. - * @param map - MutableRefObject, the maplibre map instance - * @param hoverFeatureIds - MutableRefObject>, used to keep track of geoids that have been hovered over - * @deprecated This function is no longer in use and will be removed in a future version. - */ -export const UnhighlightFeature = ( - map: MutableRefObject, - hoverFeatureIds: MutableRefObject>, - sourceLayer: string, -) => { - if (hoverFeatureIds.current.size) { - hoverFeatureIds.current.forEach((Id) => { - map.current?.setFeatureState( - { - source: BLOCK_SOURCE_ID, - id: Id, - sourceLayer: sourceLayer, - }, - { hover: false }, - ); - }); - hoverFeatureIds.current.clear(); - } -}; - /** * Resets the selection status of the map to be able to clear all and start over. * - * @param map - MutableRefObject + * @param map - Map | null * @param mapStoreRef - MapStore */ export const ResetMapSelectState = ( - map: MutableRefObject, + map: Map | null, mapStoreRef: MapStore, sourceLayer: string, ) => { - if (mapStoreRef.zoneAssignments.size) { - map.current?.removeFeatureState({ + if (map && Object.keys(mapStoreRef.zoneAssignments).length) { + map.removeFeatureState({ source: BLOCK_SOURCE_ID, sourceLayer: sourceLayer, }); - mapStoreRef.accumulatedGeoids.clear(); + mapStoreRef.setAccumulatedGeoids(new Set()) // reset zoneAssignments mapStoreRef.resetZoneAssignments(); // confirm the map has been reset diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index dfa9e26ee..4851baaf3 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -8,7 +8,6 @@ import type { MapLayerTouchEvent, } from "maplibre-gl"; import { useMapStore } from "@store/mapStore"; -import { MutableRefObject } from "react"; import { SelectMapFeatures, SelectZoneAssignmentFeatures } from "./handlers"; import { ResetMapSelectState } from "@utils/events/handlers"; import { @@ -34,11 +33,11 @@ function getLayerIdsToPaint(child_layer: string | undefined | null) { /** * What happens when the map is clicked on; incomplete implementation * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object - * @param map - MutableRefObject, the maplibre map instance + * @param map - Map | null, the maplibre map instance */ export const handleMapClick = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; @@ -53,7 +52,7 @@ export const handleMapClick = ( paintLayers, ); - if (sourceLayer) { + if (sourceLayer && selectedFeatures && map && mapStore) { // select on both the map object and the store SelectMapFeatures(selectedFeatures, map, mapStore).then(() => { SelectZoneAssignmentFeatures(mapStore); @@ -66,7 +65,7 @@ export const handleMapClick = ( export const handleMapMouseUp = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; @@ -81,34 +80,34 @@ export const handleMapMouseUp = ( export const handleMapMouseDown = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; if (activeTool === "pan") { // enable drag pan - map.current?.dragPan.enable(); + map?.dragPan.enable(); } else if (activeTool === "brush" || activeTool === "eraser") { // disable drag pan - map.current?.dragPan.disable(); + map?.dragPan.disable(); mapStore.setIsPainting(true); } }; export const handleMapMouseEnter = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => {}; export const handleMapMouseOver = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => {}; export const handleMapMouseLeave = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; @@ -119,12 +118,12 @@ export const handleMapMouseLeave = ( export const handleMapMouseOut = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => {}; export const handleMapMouseMove = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; @@ -153,7 +152,7 @@ export const handleMapMouseMove = ( export const handleMapZoom = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => {}; export const handleMapIdle = () => {}; @@ -162,11 +161,11 @@ export const handleMapMoveEnd = () => {}; export const handleMapZoomEnd = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => {}; export const handleResetMapSelectState = ( - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); const sourceLayer = mapStore.mapDocument?.parent_layer; @@ -179,7 +178,7 @@ export const handleResetMapSelectState = ( export const handleMapContextMenu = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - map: MutableRefObject, + map: MapLibreMap | null, ) => { const mapStore = useMapStore.getState(); if (mapStore.activeTool !== "pan") { @@ -194,7 +193,7 @@ export const handleMapContextMenu = ( ? INTERACTIVE_LAYERS : [BLOCK_HOVER_LAYER_ID]; const selectedFeatures = mapStore.paintFunction(map, e, 0, paintLayers); - if (!selectedFeatures?.length || !map.current || !sourceLayer) return; + if (!selectedFeatures?.length || !map || !sourceLayer) return; setHoverFeatures(selectedFeatures.slice(0, 1)); @@ -203,7 +202,7 @@ export const handleMapContextMenu = ( setHoverFeatures([]); }; - map.current.once("movestart", handleClose); + map.once("movestart", handleClose); mapStore.setContextMenu({ x: e.point.x, diff --git a/app/src/app/utils/helpers.ts b/app/src/app/utils/helpers.ts index 662aded3b..b415028a1 100644 --- a/app/src/app/utils/helpers.ts +++ b/app/src/app/utils/helpers.ts @@ -7,7 +7,6 @@ import { LngLat, LngLatLike, } from "maplibre-gl"; -import { MutableRefObject } from "react"; import { Point } from "maplibre-gl"; import { BLOCK_HOVER_LAYER_ID, @@ -23,12 +22,12 @@ import { MapStore, useMapStore } from "../store/mapStore"; /** * PaintEventHandler * A function that takes a map reference, a map event object, and a brush size. - * @param map - MutableRefObject, the maplibre map instance + * @param map - Map | null, the maplibre map instance * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object * @param brushSize - number, the size of the brush */ export type PaintEventHandler = ( - map: React.MutableRefObject, + map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, layers?: string[], @@ -71,39 +70,39 @@ export const boxAroundPoint = ( /** * getFeaturesInBbox * Get the features in a bounding box on the map. - * @param map - MutableRefObject, the maplibre map instance + * @param map - Map | null, the maplibre map instance * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object * @param brushSize - number, the size of the brush * @returns MapGeoJSONFeature[] | undefined - An array of map features or undefined */ export const getFeaturesInBbox = ( - map: MutableRefObject, + map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, layers: string[] = [BLOCK_LAYER_ID], ): MapGeoJSONFeature[] | undefined => { const bbox = boxAroundPoint(e, brushSize); - return map.current?.queryRenderedFeatures(bbox, { layers }); + return map?.queryRenderedFeatures(bbox, { layers }); }; /** * getFeaturesIntersectingCounties * Get the features intersecting counties on the map. - * @param map - MutableRefObject, the maplibre map instance + * @param map - Map | null, the maplibre map instance * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object * @param brushSize - number, the size of the brush * @returns MapGeoJSONFeature[] | undefined - An array of map features or undefined */ export const getFeaturesIntersectingCounties = ( - map: MutableRefObject, + map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, layers: string[] = [BLOCK_LAYER_ID], ): MapGeoJSONFeature[] | undefined => { - if (!map.current) return; + if (!map) return; - const countyFeatures = map.current.queryRenderedFeatures(e.point, { + const countyFeatures = map.queryRenderedFeatures(e.point, { layers: ["counties_fill"], }); @@ -113,10 +112,9 @@ export const getFeaturesIntersectingCounties = ( if (!featureBbox) return; - const sw = map.current.project(featureBbox[0]); - const ne = map.current.project(featureBbox[1]); - - const features = map.current?.queryRenderedFeatures([sw, ne], { + const sw = map.project(featureBbox[0]); + const ne = map.project(featureBbox[1]); + const features = map.queryRenderedFeatures([sw, ne], { layers, }); @@ -172,15 +170,15 @@ const getBoundingBoxFromFeatures = ( /** * mousePos * Get the position of the mouse on the map. - * @param map - MutableRefObject, the maplibre map instance + * @param map - Map | null, the maplibre map instance * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object * @returns Point - The position of the mouse on the map */ export const mousePos = ( - map: MutableRefObject, + map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, ) => { - const canvas = map.current?.getCanvasContainer(); + const canvas = map?.getCanvasContainer(); if (!canvas) return new Point(0, 0); const rect = canvas.getBoundingClientRect(); return new Point( @@ -207,18 +205,19 @@ export interface LayerVisibility { * @returns {LayerVisibility[]} - An array of objects containing the layer ID and the new visibility state. */ export function toggleLayerVisibility( - mapRef: MutableRefObject, + mapRef: maplibregl.Map, layerIds: string[], ): LayerVisibility[] { + const activeLayerIds = getVisibleLayers(mapRef)?.map((layer) => layer.id); if (!activeLayerIds) return []; return layerIds.map((layerId) => { if (activeLayerIds && activeLayerIds.includes(layerId)) { - mapRef.current?.setLayoutProperty(layerId, "visibility", "none"); + mapRef.setLayoutProperty(layerId, "visibility", "none"); return { layerId: layerId, visibility: "none" }; } else { - mapRef.current?.setLayoutProperty(layerId, "visibility", "visible"); + mapRef.setLayoutProperty(layerId, "visibility", "visible"); return { layerId: layerId, visibility: "visible" }; } }, {}); @@ -228,10 +227,10 @@ export function toggleLayerVisibility( * getVisibleLayers * Returning an array of visible layers on the map based on the visibility layout property. * i.e. it's not based on what the user actually sees. - * @param {MutableRefObject} map - The map reference. + * @param {maplibregl.Map} map - The map reference. */ -export function getVisibleLayers(map: MutableRefObject) { - return map.current?.getStyle().layers.filter((layer) => { +export function getVisibleLayers(map: Map | null) { + return map?.getStyle().layers.filter((layer) => { return layer.layout?.visibility === "visible"; }); } @@ -239,24 +238,22 @@ export function getVisibleLayers(map: MutableRefObject) { export type ColorZoneAssignmentsState = [ MapStore["zoneAssignments"], MapStore["mapDocument"], - MapStore["mapRef"], + MapStore["getMapRef"], MapStore["shatterIds"], MapStore["appLoadingState"], MapStore["mapRenderingState"], ]; -export const getMap = (_mapRef?: MapStore["mapRef"]) => { - const mapRef = _mapRef || useMapStore.getState().mapRef; +export const getMap = (_getMapRef?: MapStore["getMapRef"]) => { + const mapRef = _getMapRef?.() || useMapStore.getState().getMapRef(); if ( - mapRef?.current && - mapRef.current - ?.getStyle() + mapRef?.getStyle() .layers.findIndex((layer) => layer.id === BLOCK_HOVER_LAYER_ID) !== -1 ) { return null; } - return mapRef as MutableRefObject; + return mapRef as maplibregl.Map; }; /** @@ -285,15 +282,16 @@ export const colorZoneAssignments = ( const [ zoneAssignments, mapDocument, - mapRef, + getMapRef, _, appLoadingState, mapRenderingState, ] = state; const previousZoneAssignments = previousState?.[0] || null; - + const mapRef = getMapRef() + const shatterIds = useMapStore.getState().shatterIds if ( - !mapRef?.current || + !mapRef || !mapDocument || appLoadingState !== "loaded" || mapRenderingState !== "loaded" @@ -305,24 +303,22 @@ export const colorZoneAssignments = ( zoneAssignments.forEach((zone, id) => { if ( - !isInitialRender && - previousZoneAssignments?.get(id) === zoneAssignments.get(id) + (id && !isInitialRender && + previousZoneAssignments?.get(id) === zoneAssignments.get(id)) || (!id) ) { return; } - // This is awful - // we need information on whether an assignment is parent or child - const isParent = id.toString().includes("vtd"); - const sourceLayer = isParent - ? mapDocument.parent_layer - : mapDocument.child_layer; + const isChild = shatterIds.children.has(id) + const sourceLayer = isChild + ? mapDocument.child_layer + : mapDocument.parent_layer; if (!sourceLayer) { return; } - mapRef.current?.setFeatureState( + mapRef?.setFeatureState( { source: BLOCK_SOURCE_ID, id, From 2421bd0adc20c795b801521dc87a5905aad85c03 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 21 Oct 2024 07:42:45 -0400 Subject: [PATCH 08/50] Pull tiles from s3 (#132) --- app/.env.dev | 2 +- app/.env.docker | 2 +- app/.env.production | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/.env.dev b/app/.env.dev index 4e3f04c60..839c644a9 100644 --- a/app/.env.dev +++ b/app/.env.dev @@ -1,2 +1,2 @@ NEXT_PUBLIC_API_URL=http://localhost:8000 -NEXT_PUBLIC_S3_BUCKET_URL=https://pub-fa71193941a74e14a38eee99f30f53d9.r2.dev +NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com diff --git a/app/.env.docker b/app/.env.docker index 4e3f04c60..839c644a9 100644 --- a/app/.env.docker +++ b/app/.env.docker @@ -1,2 +1,2 @@ NEXT_PUBLIC_API_URL=http://localhost:8000 -NEXT_PUBLIC_S3_BUCKET_URL=https://pub-fa71193941a74e14a38eee99f30f53d9.r2.dev +NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com diff --git a/app/.env.production b/app/.env.production index 0e3b58e8a..6fa9498d4 100644 --- a/app/.env.production +++ b/app/.env.production @@ -1,2 +1,2 @@ NEXT_PUBLIC_API_URL=https://districtr-v2-api.fly.dev -NEXT_PUBLIC_S3_BUCKET_URL=https://pub-fa71193941a74e14a38eee99f30f53d9.r2.dev +NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com From 401341f7f0286d73e702eb944097c2f29ff7f13d Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 21 Oct 2024 06:45:03 -0500 Subject: [PATCH 09/50] Persist user maps (#127) --- .../sidebar/GerryDBViewSelector.tsx | 53 +- .../components/sidebar/RecentMapsModal.tsx | 157 ++++++ app/src/app/store/mapEditSubs.ts | 19 +- app/src/app/store/mapStore.ts | 484 ++++++++++-------- app/src/app/store/persistConfig.ts | 12 + app/src/app/utils/api/queries.ts | 62 ++- app/src/app/utils/api/queryParamsListener.ts | 7 +- 7 files changed, 538 insertions(+), 256 deletions(-) create mode 100644 app/src/app/components/sidebar/RecentMapsModal.tsx create mode 100644 app/src/app/store/persistConfig.ts diff --git a/app/src/app/components/sidebar/GerryDBViewSelector.tsx b/app/src/app/components/sidebar/GerryDBViewSelector.tsx index 0cfc2520e..ab304fb51 100644 --- a/app/src/app/components/sidebar/GerryDBViewSelector.tsx +++ b/app/src/app/components/sidebar/GerryDBViewSelector.tsx @@ -1,22 +1,18 @@ import { useState } from "react"; -import { Select } from "@radix-ui/themes"; -import { getAvailableDistrictrMaps } from "../../utils/api/apiHandlers"; +import { Flex, Select } from "@radix-ui/themes"; import { useMapStore } from "../../store/mapStore"; -import { useQuery } from "@tanstack/react-query"; import { document } from "@/app/utils/api/mutations"; +import { RecentMapsModal } from "./RecentMapsModal"; export function GerryDBViewSelector() { const [limit, setLimit] = useState(30); const [offset, setOffset] = useState(0); const mapDocument = useMapStore((state) => state.mapDocument); + const mapViews = useMapStore((state) => state.mapViews); + const { isPending, isError, data, error } = mapViews || {}; - const { isPending, isError, data, error } = useQuery({ - queryKey: ["views", limit, offset], - queryFn: () => getAvailableDistrictrMaps(limit, offset), - }); - const selectedView = data?.find( - (view) => view.gerrydb_table_name === mapDocument?.gerrydb_table, + (view) => view.gerrydb_table_name === mapDocument?.gerrydb_table ); const handleValueChange = (value: string) => { @@ -36,21 +32,32 @@ export function GerryDBViewSelector() { if (isPending) return
Loading geographies... 🌎
; - if (isError) return
Error loading geographies: {error.message}
; + if (isError) return
Error loading geographies: {error?.message}
; return ( - - - - - Districtr map options - {data.map((view, index) => ( - - {view.name} - - ))} - - - + + + + + + Districtr map options + {data?.map((view, index) => ( + + {view.name} + + ))} + + + + + ); } diff --git a/app/src/app/components/sidebar/RecentMapsModal.tsx b/app/src/app/components/sidebar/RecentMapsModal.tsx new file mode 100644 index 000000000..6f6d1d0c7 --- /dev/null +++ b/app/src/app/components/sidebar/RecentMapsModal.tsx @@ -0,0 +1,157 @@ +import { useMapStore } from "@/app/store/mapStore"; +import React from "react"; +import { Cross2Icon } from "@radix-ui/react-icons"; +import { + Button, + Flex, + Text, + Table, + Dialog, + Box, + TextField, + IconButton, +} from "@radix-ui/themes"; +import { usePathname, useSearchParams, useRouter } from "next/navigation"; +import { DocumentObject } from "../../utils/api/apiHandlers"; +type NamedDocumentObject = DocumentObject & { name?: string }; +export const RecentMapsModal = () => { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + const mapDocument = useMapStore((store) => store.mapDocument); + const userMaps = useMapStore((store) => store.userMaps); + const upcertUserMap = useMapStore((store) => store.upcertUserMap); + const setMapDocument = useMapStore((store) => store.setMapDocument); + const [dialogOpen, setDialogOpen] = React.useState(false); + + const handleMapDocument = (data: NamedDocumentObject) => { + setMapDocument(data); + const urlParams = new URLSearchParams(searchParams.toString()); + urlParams.set("document_id", data.document_id); + router.push(pathname + "?" + urlParams.toString()); + // close dialog + setDialogOpen(false); + }; + + if (!userMaps?.length) { + return null; + } + + return ( + + + + + + + + Recent Maps + + + + + + + + + + + Map Name + + Last Updated + {/* load */} + {/* delete */} + + + + + {userMaps.map((userMap, i) => ( + + upcertUserMap({ + userMapData, + userMapDocumentId: userMap.document_id, + }) + } + data={userMap} + onSelect={handleMapDocument} + /> + ))} + + + + + ); +}; + +const RecentMapsRow: React.FC<{ + data: NamedDocumentObject; + onSelect: (data: NamedDocumentObject) => void; + active: boolean; + onChange?: (data?: NamedDocumentObject) => void; +}> = ({ data, onSelect, active, onChange }) => { + const updatedDate = new Date(data.updated_at as string); + const formattedData = updatedDate.toLocaleDateString(); + const name = data?.name || data.gerrydb_table; + + const handleChangeName = (name?: string) => { + name?.length && onChange?.({ ...data, name }); + }; + + return ( + + + {!!(active && onChange) ? ( + + handleChangeName(e.target.value)} + > + + ) : ( + {name} + )} + + + {formattedData} + + + {!active && ( + + )} + + + {!active && ( + onChange?.()} + variant="ghost" + color="ruby" + className="size-full" + > + + + )} + + + ); +}; diff --git a/app/src/app/store/mapEditSubs.ts b/app/src/app/store/mapEditSubs.ts index cd3741735..1967b34a4 100644 --- a/app/src/app/store/mapEditSubs.ts +++ b/app/src/app/store/mapEditSubs.ts @@ -25,12 +25,21 @@ const zoneUpdates = ({ }; const debouncedZoneUpdate = debounce(zoneUpdates, 25); +type zoneSubState = [ + MapStore['getMapRef'], + MapStore['zoneAssignments'], + MapStore['appLoadingState'], + MapStore['mapRenderingState'] +] export const getMapEditSubs = (useMapStore: typeof _useMapStore) => { - const sendZonesOnMapRefSub = useMapStore.subscribe( - (state) => [state.getMapRef, state.zoneAssignments], - () => { - const { getMapRef, zoneAssignments, appLoadingState } = - useMapStore.getState(); + const sendZonesOnMapRefSub = useMapStore.subscribe( + (state) => [state.getMapRef, state.zoneAssignments, state.appLoadingState, state.mapRenderingState], + ([getMapRef, zoneAssignments, appLoadingState, mapRenderingState], [ _prevMapRef, _prevZoneAssignments, prevAppLoadingState, prevMapRenderingState]) => { + const previousNotLoaded = [appLoadingState, mapRenderingState, prevAppLoadingState, prevMapRenderingState].some(state => state !== 'loaded') + if (!getMapRef() || previousNotLoaded) { + return + } + console.log("!!!SENDING UPDATES", appLoadingState, mapRenderingState, prevAppLoadingState, prevMapRenderingState) debouncedZoneUpdate({ getMapRef, zoneAssignments, appLoadingState }); }, { equalityFn: shallowCompareArray} diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 8f66c9df9..5bfd82129 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -1,6 +1,7 @@ +"use client" import type { MapGeoJSONFeature, MapOptions } from "maplibre-gl"; import { create } from "zustand"; -import { devtools, subscribeWithSelector } from "zustand/middleware"; +import { devtools, subscribeWithSelector, persist } from "zustand/middleware"; import type { ActiveTool, MapFeatureInfo, @@ -10,12 +11,13 @@ import type { import { Zone, GDBPath } from "../constants/types"; import { Assignment, + DistrictrMap, DocumentObject, ZonePopulation, } from "../utils/api/apiHandlers"; import maplibregl from "maplibre-gl"; import type { MutableRefObject } from "react"; -import { UseQueryResult } from "@tanstack/react-query"; +import { QueryObserverResult, UseQueryResult } from "@tanstack/react-query"; import { ContextMenuState, LayerVisibility, @@ -28,6 +30,8 @@ import { patchShatter } from "../utils/api/mutations"; import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; import { getMapMetricsSubs } from "./metricsSubs"; import { getMapEditSubs } from "./mapEditSubs"; +import { getMapViewsSubs } from "../utils/api/queries"; +import { persistOptions } from "./persistConfig"; const prodWrapper: typeof devtools = (store: any) => store @@ -42,6 +46,8 @@ export interface MapStore { setMapRef: (map: MutableRefObject) => void; mapLock: boolean; setMapLock: (lock: boolean) => void; + mapViews: Partial>; + setMapViews: (maps: MapStore["mapViews"]) => void; mapDocument: DocumentObject | null; setMapDocument: (mapDocument: DocumentObject) => void; shatterIds: { @@ -75,7 +81,7 @@ export interface MapStore { zonePopulations: Map; setZonePopulations: (zone: Zone, population: number) => void; accumulatedGeoids: Set; - setAccumulatedGeoids: (geoids: MapStore['accumulatedGeoids']) => void; + setAccumulatedGeoids: (geoids: MapStore["accumulatedGeoids"]) => void; brushSize: number; setBrushSize: (size: number) => void; isPainting: boolean; @@ -95,231 +101,287 @@ export interface MapStore { updateVisibleLayerIds: (layerIds: LayerVisibility[]) => void; contextMenu: ContextMenuState | null; setContextMenu: (menu: ContextMenuState | null) => void; + userMaps: Array; + setUserMaps: (userMaps: MapStore["userMaps"]) => void; + upcertUserMap: (props: { + documentId?: string; + mapDocument?: MapStore["mapDocument"]; + userMapDocumentId?: string; + userMapData?: MapStore["userMaps"][number]; + }) => void; } -const initialLoadingState = typeof window !== 'undefined' && new URLSearchParams(window.location.search).has( - "document_id" -) - ? "loading" - : "initializing"; +const initialLoadingState = + typeof window !== "undefined" && + new URLSearchParams(window.location.search).has("document_id") + ? "loading" + : "initializing"; -export const useMapStore = create(devtools( - subscribeWithSelector((set, get) => ({ - appLoadingState: initialLoadingState, - setAppLoadingState: (appLoadingState) => set({ appLoadingState }), - mapRenderingState: "initializing", - setMapRenderingState: (mapRenderingState) => set({ mapRenderingState }), - getMapRef: () => null, - setMapRef: (mapRef) => { - set({ - getMapRef: () => mapRef.current, - appLoadingState: - initialLoadingState === "initializing" - ? "loaded" - : get().appLoadingState, - }) - }, - mapLock: false, - setMapLock: (mapLock) => set({ mapLock }), - mapDocument: null, - setMapDocument: (mapDocument) => { - const currentMapDocument = get().mapDocument - if (currentMapDocument?.document_id === mapDocument.document_id) { - return - } - get().setFreshMap(true); - get().resetZoneAssignments(); - - set({ - mapDocument: mapDocument, - shatterIds: { parents: new Set(), children: new Set() }, - }); - }, - shatterIds: { - parents: new Set(), - children: new Set(), - }, - handleShatter: async (document_id, geoids) => { - set({ mapLock: true }); - const shatterResult = await patchShatter.mutate({ - document_id, - geoids, - }); +export const useMapStore = create( + persist( + devwrapper( + subscribeWithSelector((set, get) => ({ + appLoadingState: initialLoadingState, + setAppLoadingState: (appLoadingState) => set({ appLoadingState }), + mapRenderingState: "initializing", + setMapRenderingState: (mapRenderingState) => set({ mapRenderingState }), + getMapRef: () => null, + setMapRef: (mapRef) => { + set({ + getMapRef: () => mapRef.current, + appLoadingState: + initialLoadingState === "initializing" + ? "loaded" + : get().appLoadingState, + }); + }, + mapLock: false, + setMapLock: (mapLock) => set({ mapLock }), + mapViews: { isPending: true }, + setMapViews: (mapViews) => set({ mapViews }), + mapDocument: null, + setMapDocument: (mapDocument) => { + const currentMapDocument = get().mapDocument; + if (currentMapDocument?.document_id === mapDocument.document_id) { + return; + } + get().setFreshMap(true); + get().resetZoneAssignments(); + get().upcertUserMap({ + mapDocument, + }) + set({ + mapDocument: mapDocument, + shatterIds: { parents: new Set(), children: new Set() }, + }); + }, + upcertUserMap: ({ mapDocument, userMapData, userMapDocumentId }) => { + let userMaps = [ ...get().userMaps ]; + const mapViews = get().mapViews.data + if (mapDocument?.document_id && mapViews) { + const documentIndex = userMaps.findIndex( + (f) => f.document_id === mapDocument?.document_id + ); + const documentInfo = mapViews.find( + (view) => view.gerrydb_table_name === mapDocument.gerrydb_table + ); + if (documentIndex !== -1) { + userMaps[documentIndex] = { + ...documentInfo, + ...userMaps[documentIndex], + ...mapDocument, + }; + } else { + userMaps = [{ ...mapDocument, ...documentInfo }, ...userMaps]; + } + } else if (userMapDocumentId) { + const i = userMaps.findIndex(map => map.document_id === userMapDocumentId) + if (userMapData) { + userMaps.splice(i, 1, userMapData); // Replace the map at index i with the new data + } else { + const urlParams = new URL(window.location.href).searchParams; + urlParams.delete("document_id"); // Remove the document_id parameter + window.history.pushState({}, '', window.location.pathname + '?' + urlParams.toString()); // Update the URL without document_id + userMaps.splice(i, 1); + } + } + set({ + userMaps, + }); + }, + shatterIds: { + parents: new Set(), + children: new Set(), + }, + handleShatter: async (document_id, geoids) => { + set({ mapLock: true }); + const shatterResult = await patchShatter.mutate({ + document_id, + geoids, + }); - const zoneAssignments = new Map(get().zoneAssignments); - const shatterIds = get().shatterIds; + const zoneAssignments = new Map(get().zoneAssignments); + const shatterIds = get().shatterIds; - let existingParents = new Set(shatterIds.parents); - let existingChildren = new Set(shatterIds.children); + let existingParents = new Set(shatterIds.parents); + let existingChildren = new Set(shatterIds.children); - const newParent = shatterResult.parents.geoids; - const newChildren = new Set( - shatterResult.children.map((child) => child.geo_id) - ); + const newParent = shatterResult.parents.geoids; + const newChildren = new Set( + shatterResult.children.map((child) => child.geo_id) + ); - const multipleShattered = shatterResult.parents.geoids.length > 1; - if (!multipleShattered) { - setZones(zoneAssignments, newParent[0], newChildren); - } else { - // todo handle multiple shattered case - } - newParent.forEach((parent) => existingParents.add(parent)); - // there may be a faster way to do this - [newChildren].forEach( - (children) => existingChildren = new Set([...existingChildren, ...children]) - ) + const multipleShattered = shatterResult.parents.geoids.length > 1; + if (!multipleShattered) { + setZones(zoneAssignments, newParent[0], newChildren); + } else { + // todo handle multiple shattered case + } + newParent.forEach((parent) => existingParents.add(parent)); + // there may be a faster way to do this + [newChildren].forEach( + (children) => + (existingChildren = new Set([...existingChildren, ...children])) + ); - set({ - shatterIds: { - parents: existingParents, - children: existingChildren, + set({ + shatterIds: { + parents: existingParents, + children: existingChildren, + }, + zoneAssignments, + }); }, - zoneAssignments, - }); - }, - setShatterIds: ( - existingParents, - existingChildren, - newParent, - newChildren, - multipleShattered - ) => { - const zoneAssignments = new Map(get().zoneAssignments); + setShatterIds: ( + existingParents, + existingChildren, + newParent, + newChildren, + multipleShattered + ) => { + const zoneAssignments = new Map(get().zoneAssignments); - if (!multipleShattered) { - setZones(zoneAssignments, newParent[0], newChildren[0]); - } else { - // todo handle multiple shattered case - } - newParent.forEach((parent) => existingParents.add(parent)); - // there may be a faster way to do this - newChildren.forEach( - (children) => existingChildren = new Set([...existingChildren, ...children]) - ); + if (!multipleShattered) { + setZones(zoneAssignments, newParent[0], newChildren[0]); + } else { + // todo handle multiple shattered case + } + newParent.forEach((parent) => existingParents.add(parent)); + // there may be a faster way to do this + newChildren.forEach( + (children) => + (existingChildren = new Set([...existingChildren, ...children])) + ); - set({ - shatterIds: { - parents: existingParents, - children: existingChildren, + set({ + shatterIds: { + parents: existingParents, + children: existingChildren, + }, + zoneAssignments, + }); }, - zoneAssignments, - }); - }, - hoverFeatures: [], - setHoverFeatures: (_features) => { - const hoverFeatures = _features - ? _features.map((f) => ({ - source: f.source, - sourceLayer: f.sourceLayer, - id: f.id, - })) - : []; + hoverFeatures: [], + setHoverFeatures: (_features) => { + const hoverFeatures = _features + ? _features.map((f) => ({ + source: f.source, + sourceLayer: f.sourceLayer, + id: f.id, + })) + : []; - set({ hoverFeatures }); - }, - mapOptions: { - center: [-98.5795, 39.8283], - zoom: 3, - pitch: 0, - bearing: 0, - container: "", - }, - setMapOptions: (options) => set({ mapOptions: options }), - activeTool: "pan", - setActiveTool: (tool) => set({ activeTool: tool }), - spatialUnit: "tract", - setSpatialUnit: (unit) => set({ spatialUnit: unit }), - selectedZone: 1, - setSelectedZone: (zone) => set({ selectedZone: zone }), - zoneAssignments: new Map(), - accumulatedGeoids: new Set(), - setAccumulatedGeoids: (accumulatedGeoids) => set({accumulatedGeoids}), - setZoneAssignments: (zone, geoids) => { - const zoneAssignments = get().zoneAssignments; - const newZoneAssignments = new Map(zoneAssignments); - geoids.forEach((geoid) => { - newZoneAssignments.set(geoid, zone); - }); - set({ - zoneAssignments: newZoneAssignments, - accumulatedGeoids: new Set(), - }); - }, - loadZoneAssignments: (assignments) => { - const zoneAssignments = new Map(); - const shatterIds = { - parents: new Set(), - children: new Set(), - }; - assignments.forEach((assignment) => { - zoneAssignments.set(assignment.geo_id, assignment.zone); - if (assignment.parent_path) { - shatterIds.parents.add(assignment.parent_path); - shatterIds.children.add(assignment.geo_id); - } - }); - set({ zoneAssignments, shatterIds, appLoadingState: "loaded" }); - }, - accumulatedBlockPopulations: new Map(), - resetAccumulatedBlockPopulations: () => - set({ accumulatedBlockPopulations: new Map() }), - zonePopulations: new Map(), - setZonePopulations: (zone, population) => - set((state) => { - const newZonePopulations = new Map(state.zonePopulations); - newZonePopulations.set(zone, population); - return { - zonePopulations: newZonePopulations, - }; - }), - resetZoneAssignments: () => set({ zoneAssignments: new Map() }), - brushSize: 50, - setBrushSize: (size) => set({ brushSize: size }), - isPainting: false, - setIsPainting: (isPainting) => set({ isPainting }), - paintFunction: getFeaturesInBbox, - setPaintFunction: (paintFunction) => set({ paintFunction }), - clearMapEdits: () => - set({ + set({ hoverFeatures }); + }, + mapOptions: { + center: [-98.5795, 39.8283], + zoom: 3, + pitch: 0, + bearing: 0, + container: "", + }, + setMapOptions: (options) => set({ mapOptions: options }), + activeTool: "pan", + setActiveTool: (tool) => set({ activeTool: tool }), + spatialUnit: "tract", + setSpatialUnit: (unit) => set({ spatialUnit: unit }), + selectedZone: 1, + setSelectedZone: (zone) => set({ selectedZone: zone }), zoneAssignments: new Map(), accumulatedGeoids: new Set(), - selectedZone: 1, - }), - freshMap: false, - setFreshMap: (resetMap) => set({ freshMap: resetMap }), - mapMetrics: null, - setMapMetrics: (metrics) => set({ mapMetrics: metrics }), - visibleLayerIds: ["counties_boundary", "counties_labels"], - setVisibleLayerIds: (layerIds) => set({ visibleLayerIds: layerIds }), - addVisibleLayerIds: (layerIds: string[]) => { - set((state) => { - const newVisibleLayerIds = new Set(state.visibleLayerIds); - layerIds.forEach((layerId) => { - newVisibleLayerIds.add(layerId); - }); - return { visibleLayerIds: Array.from(newVisibleLayerIds) }; - }); - }, - updateVisibleLayerIds: (layerVisibilities: LayerVisibility[]) => { - set((state) => { - const newVisibleLayerIds = new Set(state.visibleLayerIds); - layerVisibilities.forEach((layerVisibility) => { - if (layerVisibility.visibility === "visible") { - newVisibleLayerIds.add(layerVisibility.layerId); - } else { - newVisibleLayerIds.delete(layerVisibility.layerId); - } - }); - return { visibleLayerIds: Array.from(newVisibleLayerIds) }; - }); - }, - contextMenu: null, - setContextMenu: (contextMenu) => set({ contextMenu }), - })) -)); + setAccumulatedGeoids: (accumulatedGeoids) => set({ accumulatedGeoids }), + setZoneAssignments: (zone, geoids) => { + const zoneAssignments = get().zoneAssignments; + const newZoneAssignments = new Map(zoneAssignments); + geoids.forEach((geoid) => { + newZoneAssignments.set(geoid, zone); + }); + set({ + zoneAssignments: newZoneAssignments, + accumulatedGeoids: new Set(), + }); + }, + loadZoneAssignments: (assignments) => { + const zoneAssignments = new Map(); + const shatterIds = { + parents: new Set(), + children: new Set(), + }; + assignments.forEach((assignment) => { + zoneAssignments.set(assignment.geo_id, assignment.zone); + if (assignment.parent_path) { + shatterIds.parents.add(assignment.parent_path); + shatterIds.children.add(assignment.geo_id); + } + }); + set({ zoneAssignments, shatterIds, appLoadingState: "loaded" }); + }, + accumulatedBlockPopulations: new Map(), + resetAccumulatedBlockPopulations: () => + set({ accumulatedBlockPopulations: new Map() }), + zonePopulations: new Map(), + setZonePopulations: (zone, population) => + set((state) => { + const newZonePopulations = new Map(state.zonePopulations); + newZonePopulations.set(zone, population); + return { + zonePopulations: newZonePopulations, + }; + }), + resetZoneAssignments: () => set({ zoneAssignments: new Map() }), + brushSize: 50, + setBrushSize: (size) => set({ brushSize: size }), + isPainting: false, + setIsPainting: (isPainting) => set({ isPainting }), + paintFunction: getFeaturesInBbox, + setPaintFunction: (paintFunction) => set({ paintFunction }), + clearMapEdits: () => + set({ + zoneAssignments: new Map(), + accumulatedGeoids: new Set(), + selectedZone: 1, + }), + freshMap: false, + setFreshMap: (resetMap) => set({ freshMap: resetMap }), + mapMetrics: null, + setMapMetrics: (metrics) => set({ mapMetrics: metrics }), + visibleLayerIds: ["counties_boundary", "counties_labels"], + setVisibleLayerIds: (layerIds) => set({ visibleLayerIds: layerIds }), + addVisibleLayerIds: (layerIds: string[]) => { + set((state) => { + const newVisibleLayerIds = new Set(state.visibleLayerIds); + layerIds.forEach((layerId) => { + newVisibleLayerIds.add(layerId); + }); + return { visibleLayerIds: Array.from(newVisibleLayerIds) }; + }); + }, + updateVisibleLayerIds: (layerVisibilities: LayerVisibility[]) => { + set((state) => { + const newVisibleLayerIds = new Set(state.visibleLayerIds); + layerVisibilities.forEach((layerVisibility) => { + if (layerVisibility.visibility === "visible") { + newVisibleLayerIds.add(layerVisibility.layerId); + } else { + newVisibleLayerIds.delete(layerVisibility.layerId); + } + }); + return { visibleLayerIds: Array.from(newVisibleLayerIds) }; + }); + }, + contextMenu: null, + setContextMenu: (contextMenu) => set({ contextMenu }), + userMaps: [], + setUserMaps: (userMaps) => set({ userMaps }), + })) + ), + persistOptions + ) +); // these need to initialize after the map store getRenderSubscriptions(useMapStore); getMapMetricsSubs(useMapStore); +getMapViewsSubs(useMapStore); getMapEditSubs(useMapStore); -getSearchParamsObersver(); \ No newline at end of file +getSearchParamsObersver(); diff --git a/app/src/app/store/persistConfig.ts b/app/src/app/store/persistConfig.ts new file mode 100644 index 000000000..c7f10baf7 --- /dev/null +++ b/app/src/app/store/persistConfig.ts @@ -0,0 +1,12 @@ +import { PersistOptions } from "zustand/middleware" +import { MapStore } from "./mapStore" + + +export const persistOptions: PersistOptions> = { + name: 'districtr-persistrictr', + version: 0, + partialize: (state) => ({ + userMaps: state.userMaps + }), + +} diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index b5d04db1e..b4f3cd552 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -1,6 +1,8 @@ import { QueryObserver, skipToken } from "@tanstack/react-query"; import { queryClient } from "./queryClient"; import { + DistrictrMap, + getAvailableDistrictrMaps, Assignment, DocumentObject, getAssignments, @@ -10,6 +12,9 @@ import { } from "./apiHandlers"; import { MapStore, useMapStore } from "@/app/store/mapStore"; +const INITIAL_VIEW_LIMIT = 30 +const INITIAL_VIEW_OFFSET = 0 + export const mapMetrics = new QueryObserver(queryClient, { queryKey: ["_zonePopulations"], queryFn: skipToken, @@ -22,29 +27,58 @@ export const updateMapMetrics = (mapDocument: DocumentObject) => { }); }; + mapMetrics.subscribe((result) => { useMapStore.getState().setMapMetrics(result); }); +export const mapViewsQuery = new QueryObserver(queryClient, { + queryKey: ["views", INITIAL_VIEW_LIMIT, INITIAL_VIEW_OFFSET], + queryFn: () => getAvailableDistrictrMaps(INITIAL_VIEW_LIMIT, INITIAL_VIEW_OFFSET), +}); + +export const updateMapViews = (limit: number, offset: number) => { + mapViewsQuery.setOptions({ + queryKey: ["views", limit, offset], + queryFn: () => getAvailableDistrictrMaps(limit, offset), + }); +}; + +export const getMapViewsSubs = (_useMapStore: typeof useMapStore) => { + mapViewsQuery.subscribe((result) => { + if (result) { + _useMapStore.getState().setMapViews(result) + } + }) +} + +const getDocumentFunction = (documentId?: string) => { + return async () => { + const currentId = useMapStore.getState().mapDocument?.document_id; + if (documentId && documentId !== currentId) { + useMapStore.getState().setAppLoadingState('loading'); + return await getDocument(documentId); + } else { + return null; + } +} +} + export const updateDocumentFromId = new QueryObserver( queryClient, { - queryKey: ["mapDocument"], - queryFn: async () => { - const document_id = new URL(window.location.href).searchParams.get( - "document_id" - ); - const mapDocument = useMapStore.getState().mapDocument; - if (document_id && mapDocument?.document_id !== document_id) { - useMapStore.getState().setAppLoadingState('loading'); - return await getDocument(document_id); - } else { - return null; - } - }, - } + queryKey: ["mapDocument", undefined], + queryFn: getDocumentFunction() + }, ); +export const updateGetDocumentFromId = (documentId:string) => { + updateDocumentFromId.setOptions({ + queryKey: ["mapDocument", documentId], + queryFn: getDocumentFunction(documentId) + }); +} + updateDocumentFromId.subscribe((mapDocument) => { if (mapDocument.data) { useMapStore.getState().setMapDocument(mapDocument.data); diff --git a/app/src/app/utils/api/queryParamsListener.ts b/app/src/app/utils/api/queryParamsListener.ts index 42e947dce..955ed7521 100644 --- a/app/src/app/utils/api/queryParamsListener.ts +++ b/app/src/app/utils/api/queryParamsListener.ts @@ -1,4 +1,5 @@ -import { updateDocumentFromId } from "./queries"; +import { updateDocumentFromId, updateGetDocumentFromId } from "./queries"; +export let previousDocumentID = '' export const getSearchParamsObersver = () => { // next ssr safety @@ -21,8 +22,8 @@ export const getSearchParamsObersver = () => { "document_id" ); if (documentId && documentId !== previousDocumentID) { - previousDocumentID = documentId; - updateDocumentFromId.refetch(); + previousDocumentID = documentId + updateGetDocumentFromId(documentId) } }); const config = { subtree: true, childList: true }; From 535da8587dab14a8a4ec8cfe1040fdb53cecff01 Mon Sep 17 00:00:00 2001 From: mariogiampieri Date: Mon, 21 Oct 2024 08:44:51 -0400 Subject: [PATCH 10/50] add action to create PR-specific apps (#116) Co-authored-by: Raphael Paul Laude --- .github/workflows/fly-deploy-pr.yml | 173 +++++++++++++++++++++++++++ README.md | 4 + app/Dockerfile | 9 ++ app/fly.toml | 2 +- app/src/app/components/Map.tsx | 4 +- app/src/app/utils/api/apiHandlers.ts | 26 ++-- backend/app/main.py | 6 +- 7 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/fly-deploy-pr.yml diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml new file mode 100644 index 000000000..edca64b63 --- /dev/null +++ b/.github/workflows/fly-deploy-pr.yml @@ -0,0 +1,173 @@ +name: Fly Deploy Disctictr V2 PR +on: + pull_request: + types: [opened, reopened, synchronize, closed] +env: + FLY_API_TOKEN: ${{ secrets.FLY_ORG_TOTKEN }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + FLY_REGION: "iad" + FLY_ORG: "mggg" +jobs: + pr_review_app: + runs-on: ubuntu-latest + + concurrency: + group: pr-${{ github.event.number }} + + environment: + name: pr-${{ github.event.number }} + + steps: + - uses: actions/checkout@v3 + - uses: superfly/flyctl-actions/setup-flyctl@master + + # Set up common variables + - name: Set Variables + run: | + echo "db_name=pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" >> $GITHUB_ENV + echo "api_app_name=pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" >> $GITHUB_ENV + echo "frontend_app_name=pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" >> $GITHUB_ENV + + - name: Destroy Resources + if: github.event.action == 'closed' + run: | + app_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" + frontend_app_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" + db_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" + + echo "Destroying app $app_name" + flyctl apps destroy "$app_name" + + echo "Destroying frontend app $frontend_app_name" + flyctl apps destroy "$frontend_app_name" + + echo "Destroying database $db_name" + flyctl postgres destroy "$db_name" + + echo "Resources for PR #${{ github.event.number }} have been destroyed." + env: + FLY_API_TOKEN: ${{ secrets.FLY_ORG_TOKEN }} + + # fork new db from existing db if it doesn't already exist + - name: Fork From DB + id: fork-db + run: | + if flyctl postgres list | grep -q pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db; then + echo "DB already exists" + else + flyctl postgres create \ + --name pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db \ + --region ewr \ + --initial-cluster-size 1 \ + --vm-size shared-cpu-2x \ + -p ${{ secrets.FLY_PR_PG_PASSWORD }} \ + --org mggg \ + --fork-from districtr-v2-db + + if [ $? -eq 0 ]; then + echo "Database created successfully." + else + echo "Failed to create database." + exit 1 + fi + fi + echo "::set-output name=name::pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" + + # manually launch and deploy the api app + - name: launch api app + run: | + app="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" + db_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" + config="fly.toml" + + # Check if the app exists + if flyctl apps list | grep -q "$app"; then + echo "App $app already exists. Skipping launch." + else + flyctl launch \ + --no-deploy --copy-config --name "$app" + echo "App $app launched successfully." + fi + # Output app name for use in the deploy step + echo "api_app_name=$app" >> $GITHUB_ENV + working-directory: backend + + - name: deploy api app + run: | + flyctl deploy \ + --config fly.toml --app "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" \ + --strategy immediate '--ha=false' --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 + + flyctl secrets set \ + -a pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api \ + POSTGRES_SCHEME="postgresql+psycopg" \ + POSTGRES_SERVER="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db.flycast" \ + POSTGRES_USER="postgres" \ + POSTGRES_PASSWORD=${{ secrets.FLY_PR_PG_PASSWORD }} \ + POSTGRES_DB="districtr_v2_api" \ + BACKEND_CORS_ORIGINS="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app.fly.dev,https://districtr-v2-frontend.fly.dev" \ + DATABASE_URL="postgresql://postgres:${{ secrets.FLY_PR_PG_PASSWORD }}@${{ steps.fork-db.outputs.name }}.flycast:5432/districtr_v2_api?sslmode=disable&options=-csearch_path%3Dpublic" + + echo "set $app secrets" + working-directory: backend + + - name: Check and Launch Frontend App + id: launch + run: | + app="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" + api_app="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" + config="fly.toml" + + # Check if the app exists + if flyctl apps list | grep -q "$app"; then + echo "App $app already exists. Skipping launch." + else + echo "Launching app $app." + # Run the flyctl launch command + flyctl launch \ + --no-deploy --copy-config --name "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" \ + --build-arg NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ + --build-arg NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} \ + --build-secret NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ + --build-secret NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} + + echo "App $app launched successfully." + fi + + # Output app name for use in the deploy step + echo "frontend_app_name=$app" >> $GITHUB_ENV + working-directory: app + + - name: Deploy Frontend App + run: | + app_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" + config="fly.toml" + + # Deploy the app + flyctl deploy --config "$config" --app "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" \ + --build-arg NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ + --build-arg NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} \ + --build-secret NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ + --build-secret NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} \ + --strategy immediate '--ha=false' \ + --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 + working-directory: app + + # set secrets for f/e app + - name: Set App Secrets + run: | + flyctl secrets set \ + -a "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" \ + NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ + NEXT_PUBLIC_S3_BUCKET_URL=${{secrets.NEXT_PUBLIC_S3_BUCKET_URL}} + + + - name: run database migrations + run: | + flyctl ssh console \ + --app "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" \ + --command "alembic upgrade head" + working-directory: backend + + # provision appropriately, do not over-resource + # make volume like two gb diff --git a/README.md b/README.md index a81108fb6..7deb04109 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,10 @@ When reviewing a PR, use the "HIPPO" method to provide feedback: | **PP** - Personal preference | Possible changes requested. Something the reviewer would do but is non-blocking. | | **O** - Opinion | Comment for discussion. Non-blocking. Could be a bigger idea that's relevant to the PR. | +Open PRs will spin up a set of test apps for review, following the convention `pr--districtr-districtr-v2-`, and would be available for testing at e.g. `https://pr-116-districtr-districtr-v2-app.fly.dev/map`. This behavior can be tweaks via `.github/workflows/fly-deploy-pr.yml` + +Updates to PRs will trigger updates to staging apps, including re-running of migrations on the testing db. + ### CI/CD Deployments are managed with GitHub Actions. diff --git a/app/Dockerfile b/app/Dockerfile index 2992ceb35..b79f23dbb 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -26,6 +26,15 @@ RUN npm ci --include=dev # Copy application code COPY --link . . +# add args for build-time variables from github actions secrets +ARG NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} +ARG NEXT_PUBLIC_S3_BUCKET_URL=${NEXT_PUBLIC_S3_BUCKET_URL} + +ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} +ENV NEXT_PUBLIC_S3_BUCKET_URL=${NEXT_PUBLIC_S3_BUCKET_URL} + + + # Build application RUN npm run build diff --git a/app/fly.toml b/app/fly.toml index 8c7267753..9d9b9870b 100644 --- a/app/fly.toml +++ b/app/fly.toml @@ -11,7 +11,7 @@ primary_region = 'ewr' [http_service] internal_port = 3000 force_https = true - auto_stop_machines = 'stop' + auto_stop_machines = true auto_start_machines = true min_machines_running = 1 processes = ['app'] diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index cb522fe04..9c530c95e 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -8,9 +8,7 @@ import { Protocol } from "pmtiles"; import type { MutableRefObject } from "react"; import React, { useEffect, useRef } from "react"; import { MAP_OPTIONS } from "../constants/configuration"; -import { - mapEvents, -} from "../utils/events/mapEvents"; +import { mapEvents } from "../utils/events/mapEvents"; import { INTERACTIVE_LAYERS } from "../constants/layers"; import { useMapStore } from "../store/mapStore"; diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 93484c36c..02c67fd22 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -4,7 +4,7 @@ import { useMapStore } from "@/app/store/mapStore"; export const FormatAssignments = () => { const assignments = Array.from( - useMapStore.getState().zoneAssignments.entries(), + useMapStore.getState().zoneAssignments.entries() ).map( // @ts-ignore ([geo_id, zone]: [string, number]): { @@ -16,7 +16,7 @@ export const FormatAssignments = () => { useMapStore.getState().mapDocument?.document_id.toString() ?? "", geo_id, zone, - }), + }) ); return assignments; }; @@ -76,7 +76,7 @@ export interface DocumentCreate { } export const createMapDocument: ( - document: DocumentCreate, + document: DocumentCreate ) => Promise = async (document: DocumentCreate) => { return await axios .post(`${process.env.NEXT_PUBLIC_API_URL}/api/create_document`, { @@ -93,7 +93,7 @@ export const createMapDocument: ( * @returns Promise */ export const getDocument: ( - document_id: string, + document_id: string ) => Promise = async (document_id: string) => { if (document_id) { return await axios @@ -107,12 +107,12 @@ export const getDocument: ( }; export const getAssignments: ( - mapDocument: DocumentObject, + mapDocument: DocumentObject ) => Promise = async (mapDocument) => { if (mapDocument) { return await axios .get( - `${process.env.NEXT_PUBLIC_API_URL}/api/get_assignments/${mapDocument.document_id}`, + `${process.env.NEXT_PUBLIC_API_URL}/api/get_assignments/${mapDocument.document_id}` ) .then((res) => { return res.data; @@ -140,12 +140,12 @@ export interface ZonePopulation { * @returns Promise */ export const getZonePopulations: ( - mapDocument: DocumentObject, + mapDocument: DocumentObject ) => Promise = async (mapDocument) => { if (mapDocument) { return await axios .get( - `${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/total_pop`, + `${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/total_pop` ) .then((res) => { return res.data; @@ -163,11 +163,11 @@ export const getZonePopulations: ( */ export const getAvailableDistrictrMaps: ( limit?: number, - offset?: number, + offset?: number ) => Promise = async (limit = 10, offset = 0) => { return await axios .get( - `${process.env.NEXT_PUBLIC_API_URL}/api/gerrydb/views?limit=${limit}&offset=${offset}`, + `${process.env.NEXT_PUBLIC_API_URL}/api/gerrydb/views?limit=${limit}&offset=${offset}` ) .then((res) => { return res.data; @@ -185,7 +185,7 @@ export interface Assignment { document_id: string; geo_id: string; zone: number; - parent_path?: string + parent_path?: string; } /** @@ -203,7 +203,7 @@ export interface AssignmentsCreate { * @returns server object containing the updated assignments per geoid */ export const patchUpdateAssignments: ( - assignments: Assignment[], + assignments: Assignment[] ) => Promise = async (assignments: Assignment[]) => { return await axios .patch(`${process.env.NEXT_PUBLIC_API_URL}/api/update_assignments`, { @@ -241,7 +241,7 @@ export const patchShatterParents: (params: { `${process.env.NEXT_PUBLIC_API_URL}/api/update_assignments/${document_id}/shatter_parents`, { geoids: geoids, - }, + } ) .then((res) => { return res.data; diff --git a/backend/app/main.py b/backend/app/main.py index aceeada77..209095c34 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -149,8 +149,10 @@ async def update_assignments( async def shatter_parent( document_id: str, data: GEOIDS, session: Session = Depends(get_session) ): - stmt = text("""SELECT * - FROM shatter_parent(:input_document_id, :parent_geoids)""").bindparams( + stmt = text( + """SELECT * + FROM shatter_parent(:input_document_id, :parent_geoids)""" + ).bindparams( bindparam(key="input_document_id", type_=UUIDType), bindparam(key="parent_geoids", type_=ARRAY(String)), ) From 4d4048829df707aacf73c17851bc199b742c7de6 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 21 Oct 2024 08:58:56 -0400 Subject: [PATCH 11/50] update token name (#136) --- .github/workflows/fly-deploy-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index edca64b63..7400c69b4 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -46,7 +46,7 @@ jobs: echo "Resources for PR #${{ github.event.number }} have been destroyed." env: - FLY_API_TOKEN: ${{ secrets.FLY_ORG_TOKEN }} + FLY_API_TOKEN: ${{ secrets.FLY_ORG_TOTKEN }} # fork new db from existing db if it doesn't already exist - name: Fork From DB From 194a9628f79767eb31be747734147501d5c9ac4d Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 21 Oct 2024 20:32:42 -0400 Subject: [PATCH 12/50] Paint doest highlight in middle of geo (#137) --- .github/workflows/fly-deploy-pr.yml | 6 +++--- app/src/app/utils/events/mapEvents.ts | 11 ++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index 7400c69b4..a2ef25808 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -36,13 +36,13 @@ jobs: db_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" echo "Destroying app $app_name" - flyctl apps destroy "$app_name" + flyctl apps destroy "$app_name" -y echo "Destroying frontend app $frontend_app_name" - flyctl apps destroy "$frontend_app_name" + flyctl apps destroy "$frontend_app_name" -y echo "Destroying database $db_name" - flyctl postgres destroy "$db_name" + flyctl postgres destroy "$db_name" -y echo "Resources for PR #${{ github.event.number }} have been destroyed." env: diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index 4851baaf3..9fe7afac3 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -13,8 +13,7 @@ import { ResetMapSelectState } from "@utils/events/handlers"; import { INTERACTIVE_LAYERS, BLOCK_HOVER_LAYER_ID, - BLOCK_LAYER_ID, - BLOCK_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD, } from "@constants/layers"; /* @@ -26,8 +25,8 @@ MapEvent handling; these functions are called by the event listeners in the MapC */ function getLayerIdsToPaint(child_layer: string | undefined | null) { return child_layer - ? [BLOCK_LAYER_ID, BLOCK_LAYER_ID_CHILD] - : [BLOCK_LAYER_ID]; + ? [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CHILD] + : [BLOCK_HOVER_LAYER_ID]; } /** @@ -164,9 +163,7 @@ export const handleMapZoomEnd = ( map: MapLibreMap | null, ) => {}; -export const handleResetMapSelectState = ( - map: MapLibreMap | null, -) => { +export const handleResetMapSelectState = (map: MapLibreMap | null) => { const mapStore = useMapStore.getState(); const sourceLayer = mapStore.mapDocument?.parent_layer; if (sourceLayer) { From 9bb080d2bf814fff453f715634041aeefe4020d9 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 21 Oct 2024 22:38:26 -0400 Subject: [PATCH 13/50] =?UTF-8?q?Don't=20use=20docker=20env=20vars=20?= =?UTF-8?q?=E2=80=93=20breaking=20prod=20I=20think=20(#138)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/fly-deploy-pr.yml | 91 ++++++++++++----------------- app/Dockerfile | 15 ++--- backend/fly.toml | 3 + 3 files changed, 45 insertions(+), 64 deletions(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index a2ef25808..29c4e54fc 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -1,11 +1,11 @@ -name: Fly Deploy Disctictr V2 PR +name: Fly Deploy Disctictr V2 Pull Request app on: pull_request: types: [opened, reopened, synchronize, closed] env: FLY_API_TOKEN: ${{ secrets.FLY_ORG_TOTKEN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - FLY_REGION: "iad" + FLY_REGION: "ewr" FLY_ORG: "mggg" jobs: pr_review_app: @@ -24,16 +24,16 @@ jobs: # Set up common variables - name: Set Variables run: | - echo "db_name=pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" >> $GITHUB_ENV - echo "api_app_name=pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" >> $GITHUB_ENV - echo "frontend_app_name=pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" >> $GITHUB_ENV + echo "db_name=${{ github.event.repository.name }}-${{ github.event.number }}-db" >> $GITHUB_ENV + echo "api_app_name=${{ github.event.repository.name }}-${{ github.event.number }}-api" >> $GITHUB_ENV + echo "frontend_app_name=${{ github.event.repository.name }}-${{ github.event.number }}-app" >> $GITHUB_ENV - name: Destroy Resources if: github.event.action == 'closed' run: | - app_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" - frontend_app_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" - db_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" + app_name="${{ github.event.repository.name }}-${{ github.event.number }}-api" + frontend_app_name="${{ github.event.repository.name }}-${{ github.event.number }}-app" + db_name="${{ github.event.repository.name }}-${{ github.event.number }}-db" echo "Destroying app $app_name" flyctl apps destroy "$app_name" -y @@ -42,7 +42,7 @@ jobs: flyctl apps destroy "$frontend_app_name" -y echo "Destroying database $db_name" - flyctl postgres destroy "$db_name" -y + flyctl apps destroy "$db_name" -y echo "Resources for PR #${{ github.event.number }} have been destroyed." env: @@ -52,11 +52,11 @@ jobs: - name: Fork From DB id: fork-db run: | - if flyctl postgres list | grep -q pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db; then + if flyctl postgres list | grep -q ${{ github.event.repository.name }}-${{ github.event.number }}-db; then echo "DB already exists" else flyctl postgres create \ - --name pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db \ + --name ${{ github.event.repository.name }}-${{ github.event.number }}-db \ --region ewr \ --initial-cluster-size 1 \ --vm-size shared-cpu-2x \ @@ -71,13 +71,13 @@ jobs: exit 1 fi fi - echo "::set-output name=name::pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" + echo "::set-output name=name::${{ github.event.repository.name }}-${{ github.event.number }}-db" # manually launch and deploy the api app - name: launch api app run: | - app="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" - db_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db" + app="${{ github.event.repository.name }}-${{ github.event.number }}-api" + db_name="${{ github.event.repository.name }}-${{ github.event.number }}-db" config="fly.toml" # Check if the app exists @@ -94,28 +94,26 @@ jobs: - name: deploy api app run: | - flyctl deploy \ - --config fly.toml --app "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" \ - --strategy immediate '--ha=false' --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 - flyctl secrets set \ - -a pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api \ + -a ${{ github.event.repository.name }}-${{ github.event.number }}-api \ POSTGRES_SCHEME="postgresql+psycopg" \ - POSTGRES_SERVER="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-db.flycast" \ + POSTGRES_SERVER="${{ github.event.repository.name }}-${{ github.event.number }}-db.flycast" \ POSTGRES_USER="postgres" \ POSTGRES_PASSWORD=${{ secrets.FLY_PR_PG_PASSWORD }} \ POSTGRES_DB="districtr_v2_api" \ - BACKEND_CORS_ORIGINS="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app.fly.dev,https://districtr-v2-frontend.fly.dev" \ + BACKEND_CORS_ORIGINS="https://${{ github.event.repository.name }}-${{ github.event.number }}-app.fly.dev,https://districtr-v2-frontend.fly.dev" \ DATABASE_URL="postgresql://postgres:${{ secrets.FLY_PR_PG_PASSWORD }}@${{ steps.fork-db.outputs.name }}.flycast:5432/districtr_v2_api?sslmode=disable&options=-csearch_path%3Dpublic" - - echo "set $app secrets" + + flyctl deploy \ + --config fly.toml --app "${{ github.event.repository.name }}-${{ github.event.number }}-api" \ + --strategy immediate '--ha=false' --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 working-directory: backend - name: Check and Launch Frontend App id: launch run: | - app="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" - api_app="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" + app="${{ github.event.repository.name }}-${{ github.event.number }}-app" + api_app="${{ github.event.repository.name }}-${{ github.event.number }}-api" config="fly.toml" # Check if the app exists @@ -125,11 +123,9 @@ jobs: echo "Launching app $app." # Run the flyctl launch command flyctl launch \ - --no-deploy --copy-config --name "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" \ - --build-arg NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ - --build-arg NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} \ - --build-secret NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ - --build-secret NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} + --no-deploy --copy-config --name "${{ github.event.repository.name }}-${{ github.event.number }}-app" \ + --build-arg NEXT_PUBLIC_API_URL="https://${{ github.event.repository.name }}-${{ github.event.number }}-api.fly.dev" \ + --build-arg NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com echo "App $app launched successfully." fi @@ -140,34 +136,19 @@ jobs: - name: Deploy Frontend App run: | - app_name="pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" + app_name="${{ github.event.repository.name }}-${{ github.event.number }}-app" config="fly.toml" + + flyctl secrets set \ + -a "${{ github.event.repository.name }}-${{ github.event.number }}-app" \ + NEXT_PUBLIC_API_URL="https://${{ github.event.repository.name }}-${{ github.event.number }}-api.fly.dev" \ + NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com # Deploy the app - flyctl deploy --config "$config" --app "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" \ - --build-arg NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ - --build-arg NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} \ - --build-secret NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ - --build-secret NEXT_PUBLIC_S3_BUCKET_URL=${{ secrets.NEXT_PUBLIC_S3_BUCKET_URL }} \ + flyctl deploy --config "$config" --app "${{ github.event.repository.name }}-${{ github.event.number }}-app" \ + --build-arg NEXT_PUBLIC_API_URL="https://${{ github.event.repository.name }}-${{ github.event.number }}-api.fly.dev" \ + --build-arg NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com \ --strategy immediate '--ha=false' \ --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 + working-directory: app - - # set secrets for f/e app - - name: Set App Secrets - run: | - flyctl secrets set \ - -a "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-app" \ - NEXT_PUBLIC_API_URL="https://pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api.fly.dev" \ - NEXT_PUBLIC_S3_BUCKET_URL=${{secrets.NEXT_PUBLIC_S3_BUCKET_URL}} - - - - name: run database migrations - run: | - flyctl ssh console \ - --app "pr-${{ github.event.number }}-${{ github.repository_owner }}-${{ github.event.repository.name }}-api" \ - --command "alembic upgrade head" - working-directory: backend - - # provision appropriately, do not over-resource - # make volume like two gb diff --git a/app/Dockerfile b/app/Dockerfile index b79f23dbb..8e9b60ac8 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -14,6 +14,8 @@ ENV NODE_ENV="production" # Throw-away build stage to reduce size of final image FROM base as build +ARG NEXT_PUBLIC_API_URL +ARG NEXT_PUBLIC_S3_BUCKET_URL # Install packages needed to build node modules RUN apt-get update -qq && \ @@ -23,18 +25,13 @@ RUN apt-get update -qq && \ COPY --link package-lock.json package.json ./ RUN npm ci --include=dev +RUN echo NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL && \ + echo "NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL" > .env.production && \ + echo "NEXT_PUBLIC_S3_BUCKET_URL=$NEXT_PUBLIC_S3_BUCKET_URL" >> .env.production + # Copy application code COPY --link . . -# add args for build-time variables from github actions secrets -ARG NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} -ARG NEXT_PUBLIC_S3_BUCKET_URL=${NEXT_PUBLIC_S3_BUCKET_URL} - -ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} -ENV NEXT_PUBLIC_S3_BUCKET_URL=${NEXT_PUBLIC_S3_BUCKET_URL} - - - # Build application RUN npm run build diff --git a/backend/fly.toml b/backend/fly.toml index af99eda3b..2a2825d23 100644 --- a/backend/fly.toml +++ b/backend/fly.toml @@ -32,3 +32,6 @@ primary_region = 'ewr' memory = '1gb' cpu_kind = 'shared' cpus = 1 + +[deploy] + release_command = "alembic upgrade head" From 760df7a5839090e77789d272d7f9070c780d61c1 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 21 Oct 2024 22:56:21 -0400 Subject: [PATCH 14/50] update workflow (#139) --- .github/workflows/fly-deploy-pr.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index 29c4e54fc..fab84efbd 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -21,8 +21,7 @@ jobs: - uses: actions/checkout@v3 - uses: superfly/flyctl-actions/setup-flyctl@master - # Set up common variables - - name: Set Variables + - name: Set shared environment variables run: | echo "db_name=${{ github.event.repository.name }}-${{ github.event.number }}-db" >> $GITHUB_ENV echo "api_app_name=${{ github.event.repository.name }}-${{ github.event.number }}-api" >> $GITHUB_ENV @@ -48,9 +47,11 @@ jobs: env: FLY_API_TOKEN: ${{ secrets.FLY_ORG_TOTKEN }} - # fork new db from existing db if it doesn't already exist + # fork new db from existing production db if it doesn't already exist + # eventually we may want to maintain a stage and only fork that - name: Fork From DB id: fork-db + if: github.event.action != 'closed' run: | if flyctl postgres list | grep -q ${{ github.event.repository.name }}-${{ github.event.number }}-db; then echo "DB already exists" @@ -75,6 +76,7 @@ jobs: # manually launch and deploy the api app - name: launch api app + if: github.event.action != 'closed' run: | app="${{ github.event.repository.name }}-${{ github.event.number }}-api" db_name="${{ github.event.repository.name }}-${{ github.event.number }}-db" @@ -111,6 +113,7 @@ jobs: - name: Check and Launch Frontend App id: launch + if: github.event.action != 'closed' run: | app="${{ github.event.repository.name }}-${{ github.event.number }}-app" api_app="${{ github.event.repository.name }}-${{ github.event.number }}-api" @@ -135,6 +138,7 @@ jobs: working-directory: app - name: Deploy Frontend App + if: github.event.action != 'closed' run: | app_name="${{ github.event.repository.name }}-${{ github.event.number }}-app" config="fly.toml" @@ -150,5 +154,4 @@ jobs: --build-arg NEXT_PUBLIC_S3_BUCKET_URL=https://districtr-v2-dev.s3.amazonaws.com \ --strategy immediate '--ha=false' \ --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 - working-directory: app From 358a37e676df53e4aa8fb93cc3c192b0301e777f Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 21 Oct 2024 22:59:21 -0400 Subject: [PATCH 15/50] Don't recreate apps on close after destroy (#140) --- .github/workflows/fly-deploy-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index fab84efbd..c2dd40650 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -95,6 +95,7 @@ jobs: working-directory: backend - name: deploy api app + if: github.event.action != 'closed' run: | flyctl secrets set \ -a ${{ github.event.repository.name }}-${{ github.event.number }}-api \ From 7a0b4aa7ea22476019cc683d39590d6e1605c51e Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 24 Oct 2024 08:09:16 -0400 Subject: [PATCH 16/50] autostop apps (#144) --- .github/workflows/fly-deploy-pr.yml | 6 +++--- app/fly.toml | 4 ++-- backend/fly.toml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index c2dd40650..39a115ffc 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -75,7 +75,7 @@ jobs: echo "::set-output name=name::${{ github.event.repository.name }}-${{ github.event.number }}-db" # manually launch and deploy the api app - - name: launch api app + - name: Launch API if: github.event.action != 'closed' run: | app="${{ github.event.repository.name }}-${{ github.event.number }}-api" @@ -94,7 +94,7 @@ jobs: echo "api_app_name=$app" >> $GITHUB_ENV working-directory: backend - - name: deploy api app + - name: Deploy API if: github.event.action != 'closed' run: | flyctl secrets set \ @@ -138,7 +138,7 @@ jobs: echo "frontend_app_name=$app" >> $GITHUB_ENV working-directory: app - - name: Deploy Frontend App + - name: Deploy frontend if: github.event.action != 'closed' run: | app_name="${{ github.event.repository.name }}-${{ github.event.number }}-app" diff --git a/app/fly.toml b/app/fly.toml index 9d9b9870b..da2cfbfb1 100644 --- a/app/fly.toml +++ b/app/fly.toml @@ -11,9 +11,9 @@ primary_region = 'ewr' [http_service] internal_port = 3000 force_https = true - auto_stop_machines = true + auto_stop_machines = "stop" auto_start_machines = true - min_machines_running = 1 + min_machines_running = 0 processes = ['app'] [[vm]] diff --git a/backend/fly.toml b/backend/fly.toml index 2a2825d23..c0850e7a3 100644 --- a/backend/fly.toml +++ b/backend/fly.toml @@ -23,9 +23,9 @@ primary_region = 'ewr' [http_service] internal_port = 8080 force_https = true - auto_stop_machines = true + auto_stop_machines = "stop" auto_start_machines = true - min_machines_running = 1 + min_machines_running = 0 processes = ['app'] [[vm]] From 2c0a75d573bec40706b6ea9c3bfffc89f83aa739 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Fri, 25 Oct 2024 00:43:50 -0400 Subject: [PATCH 17/50] Shatter tool (#142) --- .../sidebar/GerryDBViewSelector.tsx | 2 - .../components/sidebar/MapModeSelector.jsx | 21 ++++++--- .../components/sidebar/RecentMapsModal.tsx | 15 +++---- app/src/app/constants/types.ts | 4 +- app/src/app/store/mapRenderSubs.ts | 44 ++++++++++++++++--- app/src/app/utils/events/mapEvents.ts | 38 +++++++++++++--- app/src/app/utils/helpers.ts | 44 ++++++++++++------- 7 files changed, 125 insertions(+), 43 deletions(-) diff --git a/app/src/app/components/sidebar/GerryDBViewSelector.tsx b/app/src/app/components/sidebar/GerryDBViewSelector.tsx index ab304fb51..bc255f65c 100644 --- a/app/src/app/components/sidebar/GerryDBViewSelector.tsx +++ b/app/src/app/components/sidebar/GerryDBViewSelector.tsx @@ -2,7 +2,6 @@ import { useState } from "react"; import { Flex, Select } from "@radix-ui/themes"; import { useMapStore } from "../../store/mapStore"; import { document } from "@/app/utils/api/mutations"; -import { RecentMapsModal } from "./RecentMapsModal"; export function GerryDBViewSelector() { const [limit, setLimit] = useState(30); @@ -57,7 +56,6 @@ export function GerryDBViewSelector() { - ); } diff --git a/app/src/app/components/sidebar/MapModeSelector.jsx b/app/src/app/components/sidebar/MapModeSelector.jsx index 5d6ff1130..2938f8b75 100644 --- a/app/src/app/components/sidebar/MapModeSelector.jsx +++ b/app/src/app/components/sidebar/MapModeSelector.jsx @@ -1,21 +1,31 @@ import React from "react"; import * as RadioGroup from "@radix-ui/react-radio-group"; import { styled } from "@stitches/react"; -import { blackA } from "@radix-ui/colors"; -import { useMapStore } from "../../store/mapStore"; +import { useMapStore } from "@store/mapStore"; import { RadioCards, Box } from "@radix-ui/themes"; -import { EraserIcon, Pencil2Icon, HandIcon } from "@radix-ui/react-icons"; +import { + EraserIcon, + Pencil2Icon, + HandIcon, + BorderSplitIcon, +} from "@radix-ui/react-icons"; +import { RecentMapsModal } from "@components/sidebar/RecentMapsModal"; export function MapModeSelector() { - const mapStore = useMapStore.getState(); const activeTool = useMapStore((state) => state.activeTool); const setActiveTool = useMapStore((state) => state.setActiveTool); - + if (!activeTool) return null; const activeTools = [ { mode: "pan", disabled: false, label: "Pan", icon: }, { mode: "brush", disabled: false, label: "Brush", icon: }, { mode: "eraser", disabled: false, label: "Erase", icon: }, + { + mode: "shatter", + disabled: false, + label: "Shatter", + icon: , + }, ]; const handleRadioChange = (value) => { @@ -42,6 +52,7 @@ export function MapModeSelector() { ))} + ); diff --git a/app/src/app/components/sidebar/RecentMapsModal.tsx b/app/src/app/components/sidebar/RecentMapsModal.tsx index 6f6d1d0c7..746f53e49 100644 --- a/app/src/app/components/sidebar/RecentMapsModal.tsx +++ b/app/src/app/components/sidebar/RecentMapsModal.tsx @@ -1,6 +1,6 @@ import { useMapStore } from "@/app/store/mapStore"; import React from "react"; -import { Cross2Icon } from "@radix-ui/react-icons"; +import { Cross2Icon, CounterClockwiseClockIcon } from "@radix-ui/react-icons"; import { Button, Flex, @@ -10,6 +10,7 @@ import { Box, TextField, IconButton, + RadioCards, } from "@radix-ui/themes"; import { usePathname, useSearchParams, useRouter } from "next/navigation"; import { DocumentObject } from "../../utils/api/apiHandlers"; @@ -40,14 +41,10 @@ export const RecentMapsModal = () => { return ( - + + + Recents + diff --git a/app/src/app/constants/types.ts b/app/src/app/constants/types.ts index 1073d5130..cf357da0e 100644 --- a/app/src/app/constants/types.ts +++ b/app/src/app/constants/types.ts @@ -1,7 +1,7 @@ import type { MapOptions, MapLibreEvent } from "maplibre-gl"; export type Zone = number; -export type NullableZone = Zone | null +export type NullableZone = Zone | null; export type GEOID = string; @@ -9,7 +9,7 @@ export type GDBPath = string; export type ZoneDict = Map; -export type ActiveTool = "pan" | "brush" | "eraser"; // others? +export type ActiveTool = "pan" | "brush" | "eraser" | "shatter"; // others? export type SpatialUnit = | "county" diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 8762c85f1..75eb6897b 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -5,13 +5,14 @@ import { PARENT_LAYERS, CHILD_LAYERS, getLayerFilter, -} from "../constants/layers"; +} from "@constants/layers"; import { ColorZoneAssignmentsState, colorZoneAssignments, shallowCompareArray, } from "../utils/helpers"; -import { useMapStore as _useMapStore, MapStore } from "./mapStore"; +import { useMapStore as _useMapStore, MapStore } from "@store/mapStore"; +import { getFeatureUnderCursor } from "@utils/helpers"; export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { const addLayerSubMapDocument = useMapStore.subscribe< @@ -20,7 +21,7 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { (state) => [state.mapDocument, state.getMapRef], ([mapDocument, getMapRef]) => { const mapStore = useMapStore.getState(); - const mapRef = getMapRef() + const mapRef = getMapRef(); if (mapRef && mapDocument) { addBlockLayers(mapRef, mapDocument); mapStore.addVisibleLayerIds([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]); @@ -30,12 +31,16 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ); const _shatterMapSideEffectRender = useMapStore.subscribe< - [MapStore["shatterIds"], MapStore["getMapRef"], MapStore["mapRenderingState"]] + [ + MapStore["shatterIds"], + MapStore["getMapRef"], + MapStore["mapRenderingState"], + ] >( (state) => [state.shatterIds, state.getMapRef, state.mapRenderingState], ([shatterIds, getMapRef, mapRenderingState]) => { const state = useMapStore.getState(); - const mapRef = getMapRef() + const mapRef = getMapRef(); const setMapLock = state.setMapLock; if (!mapRef || mapRenderingState !== "loaded") { @@ -91,10 +96,39 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { { equalityFn: shallowCompareArray }, ); + const _updateMapCursor = useMapStore.subscribe( + (state) => state.activeTool, + (activeTool) => { + const mapRef = useMapStore.getState().getMapRef(); + if (!mapRef) return; + + let cursor; + switch (activeTool) { + case "pan": + cursor = ""; + break; + case "brush": + cursor = "pointer"; + break; + case "eraser": + cursor = "pointer"; + break; + case "shatter": + cursor = "crosshair"; + useMapStore.getState().setPaintFunction(getFeatureUnderCursor); + break; + default: + cursor = ""; + } + mapRef.getCanvas().style.cursor = cursor; + }, + ); + return [ addLayerSubMapDocument, _shatterMapSideEffectRender, _hoverMapSideEffectRender, _zoneAssignmentMapSideEffectRender, + _updateMapCursor, ]; }; diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index 9fe7afac3..aa6ad6187 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -15,15 +15,30 @@ import { BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CHILD, } from "@constants/layers"; +import { ActiveTool } from "@/app/constants/types"; /* MapEvent handling; these functions are called by the event listeners in the MapComponent */ /** + * Get the layer IDs to paint based on whether we have + * a shatterable map (based on whether a child layer is + * present) and the active tool. If the active tool is + * shatter, we only want to paint the shatterable layer. + * + * @param child_layer - string | undefined | null, the child layer + * @param activeTool - ActiveTool, the active tool + * @returns string[], the layer IDs to paint + */ +function getLayerIdsToPaint( + child_layer: string | undefined | null, + activeTool: ActiveTool, +) { + if (activeTool === "shatter") { + return [BLOCK_HOVER_LAYER_ID]; + } -*/ -function getLayerIdsToPaint(child_layer: string | undefined | null) { return child_layer ? [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CHILD] : [BLOCK_HOVER_LAYER_ID]; @@ -41,9 +56,13 @@ export const handleMapClick = ( const mapStore = useMapStore.getState(); const activeTool = mapStore.activeTool; const sourceLayer = mapStore.mapDocument?.parent_layer; + const handleShatter = mapStore.handleShatter; if (activeTool === "brush" || activeTool === "eraser") { - const paintLayers = getLayerIdsToPaint(mapStore.mapDocument?.child_layer); + const paintLayers = getLayerIdsToPaint( + mapStore.mapDocument?.child_layer, + activeTool, + ); const selectedFeatures = mapStore.paintFunction( map, e, @@ -57,6 +76,12 @@ export const handleMapClick = ( SelectZoneAssignmentFeatures(mapStore); }); } + } else if (activeTool === "shatter") { + const documentId = mapStore.mapDocument?.document_id; + const featureId = e.features?.[0].id?.toString(); + if (documentId && featureId) { + handleShatter(documentId, [featureId]); + } } else { // tbd, for pan mode - is there an info mode on click? } @@ -129,7 +154,10 @@ export const handleMapMouseMove = ( const setHoverFeatures = mapStore.setHoverFeatures; const isPainting = mapStore.isPainting; const sourceLayer = mapStore.mapDocument?.parent_layer; - const paintLayers = getLayerIdsToPaint(mapStore.mapDocument?.child_layer); + const paintLayers = getLayerIdsToPaint( + mapStore.mapDocument?.child_layer, + activeTool, + ); const selectedFeatures = mapStore.paintFunction( map, e, @@ -137,7 +165,7 @@ export const handleMapMouseMove = ( paintLayers, ); const isBrushingTool = - sourceLayer && ["brush", "eraser"].includes(activeTool); + sourceLayer && ["brush", "eraser", "shatter"].includes(activeTool); if (isBrushingTool) { setHoverFeatures(selectedFeatures); } diff --git a/app/src/app/utils/helpers.ts b/app/src/app/utils/helpers.ts index b415028a1..18db33049 100644 --- a/app/src/app/utils/helpers.ts +++ b/app/src/app/utils/helpers.ts @@ -8,12 +8,7 @@ import { LngLatLike, } from "maplibre-gl"; import { Point } from "maplibre-gl"; -import { - BLOCK_HOVER_LAYER_ID, - BLOCK_LAYER_ID, - BLOCK_LAYER_ID_CHILD, - BLOCK_SOURCE_ID, -} from "@/app/constants/layers"; +import { BLOCK_HOVER_LAYER_ID, BLOCK_SOURCE_ID } from "@/app/constants/layers"; import { polygon, multiPolygon } from "@turf/helpers"; import { booleanWithin } from "@turf/boolean-within"; import { pointOnFeature } from "@turf/point-on-feature"; @@ -79,13 +74,30 @@ export const getFeaturesInBbox = ( map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, - layers: string[] = [BLOCK_LAYER_ID], + layers: string[] = [BLOCK_HOVER_LAYER_ID], ): MapGeoJSONFeature[] | undefined => { const bbox = boxAroundPoint(e, brushSize); return map?.queryRenderedFeatures(bbox, { layers }); }; +/** + * getFeatureUnderCursor + * Get the feature under the cursor on the map. + * @param map - Map | null, the maplibre map instance + * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object + * @param brushSize - number, the size of the brush + * @returns MapGeoJSONFeature | undefined - A map feature or undefined + */ +export const getFeatureUnderCursor = ( + map: Map | null, + e: MapLayerMouseEvent | MapLayerTouchEvent, + brushSize: number, + layers: string[] = [BLOCK_HOVER_LAYER_ID], +): MapGeoJSONFeature[] | undefined => { + return map?.queryRenderedFeatures(e.point, { layers }); +}; + /** * getFeaturesIntersectingCounties * Get the features intersecting counties on the map. @@ -98,7 +110,7 @@ export const getFeaturesIntersectingCounties = ( map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, - layers: string[] = [BLOCK_LAYER_ID], + layers: string[] = [BLOCK_HOVER_LAYER_ID], ): MapGeoJSONFeature[] | undefined => { if (!map) return; @@ -208,7 +220,6 @@ export function toggleLayerVisibility( mapRef: maplibregl.Map, layerIds: string[], ): LayerVisibility[] { - const activeLayerIds = getVisibleLayers(mapRef)?.map((layer) => layer.id); if (!activeLayerIds) return []; @@ -247,7 +258,8 @@ export type ColorZoneAssignmentsState = [ export const getMap = (_getMapRef?: MapStore["getMapRef"]) => { const mapRef = _getMapRef?.() || useMapStore.getState().getMapRef(); if ( - mapRef?.getStyle() + mapRef + ?.getStyle() .layers.findIndex((layer) => layer.id === BLOCK_HOVER_LAYER_ID) !== -1 ) { return null; @@ -288,8 +300,8 @@ export const colorZoneAssignments = ( mapRenderingState, ] = state; const previousZoneAssignments = previousState?.[0] || null; - const mapRef = getMapRef() - const shatterIds = useMapStore.getState().shatterIds + const mapRef = getMapRef(); + const shatterIds = useMapStore.getState().shatterIds; if ( !mapRef || !mapDocument || @@ -303,13 +315,15 @@ export const colorZoneAssignments = ( zoneAssignments.forEach((zone, id) => { if ( - (id && !isInitialRender && - previousZoneAssignments?.get(id) === zoneAssignments.get(id)) || (!id) + (id && + !isInitialRender && + previousZoneAssignments?.get(id) === zoneAssignments.get(id)) || + !id ) { return; } - const isChild = shatterIds.children.has(id) + const isChild = shatterIds.children.has(id); const sourceLayer = isChild ? mapDocument.child_layer : mapDocument.parent_layer; From 4524de7b5964880ae804876009e1f09d5c951341 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sat, 26 Oct 2024 21:07:52 -0400 Subject: [PATCH 18/50] should disable shatter button if not shatterable map (#146) --- app/src/app/components/sidebar/MapModeSelector.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/src/app/components/sidebar/MapModeSelector.jsx b/app/src/app/components/sidebar/MapModeSelector.jsx index 2938f8b75..d2dcd2b47 100644 --- a/app/src/app/components/sidebar/MapModeSelector.jsx +++ b/app/src/app/components/sidebar/MapModeSelector.jsx @@ -14,6 +14,7 @@ import { RecentMapsModal } from "@components/sidebar/RecentMapsModal"; export function MapModeSelector() { const activeTool = useMapStore((state) => state.activeTool); const setActiveTool = useMapStore((state) => state.setActiveTool); + const mapDocument = useMapStore((state) => state.mapDocument); if (!activeTool) return null; const activeTools = [ @@ -22,7 +23,7 @@ export function MapModeSelector() { { mode: "eraser", disabled: false, label: "Erase", icon: }, { mode: "shatter", - disabled: false, + disabled: !mapDocument?.child_layer, label: "Shatter", icon: , }, From 56fbb46298a536aeb3c00756b38e063c236369ca Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sun, 27 Oct 2024 22:57:48 -0400 Subject: [PATCH 19/50] Rename all geometry columns to geometry (#148) --- ...041d95_rename_geometry_cols_to_geometry.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 backend/app/alembic/versions/167892041d95_rename_geometry_cols_to_geometry.py diff --git a/backend/app/alembic/versions/167892041d95_rename_geometry_cols_to_geometry.py b/backend/app/alembic/versions/167892041d95_rename_geometry_cols_to_geometry.py new file mode 100644 index 000000000..7aa5af342 --- /dev/null +++ b/backend/app/alembic/versions/167892041d95_rename_geometry_cols_to_geometry.py @@ -0,0 +1,69 @@ +"""rename geometry cols to geometry + +Revision ID: 167892041d95 +Revises: 5ab466c5650a +Create Date: 2024-10-27 18:38:13.798056 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "167892041d95" +down_revision: Union[str, None] = "5ab466c5650a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + sa.text(""" + DO + $$ + DECLARE + rec RECORD; + sql TEXT; + BEGIN + FOR rec IN + WITH all_geom_cols AS ( + SELECT f_table_schema, f_table_name, ARRAY_AGG(f_geometry_column) f_geometry_column_arr, type, srid + FROM public.geometry_columns GROUP BY f_table_schema, f_table_name,f_geometry_column, type, srid + ) + SELECT f_table_schema, f_table_name, f_geometry_column, type, srid + FROM all_geom_cols, + UNNEST(f_geometry_column_arr) f_geometry_column + WHERE 'geometry' != ANY(f_geometry_column_arr) + LOOP + sql := format(' + ALTER TABLE %I.%I DROP COLUMN IF EXISTS geometry; + SELECT AddGeometryColumn(%L, %L, ''geometry'', %L, %L, 2); + + UPDATE %I.%I + SET geometry = %I; + + ALTER TABLE %I.%I + DROP COLUMN %I; + ', + rec.f_table_schema, rec.f_table_name, + rec.f_table_schema, rec.f_table_name, rec.srid, rec.type, + rec.f_table_schema, rec.f_table_name, + rec.f_geometry_column, + rec.f_table_schema, rec.f_table_name, + rec.f_geometry_column + ); + EXECUTE sql; + END LOOP; + END + $$; + """) + ) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass # This is a one-way migration + # ### end Alembic commands ### From ffaf247a76761c1a0dbc7fe0c9032b6b05f2e1b6 Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 28 Oct 2024 09:54:18 -0500 Subject: [PATCH 20/50] Add airbnb prettier config (#156) --- app/.prettierrc.js | 3 +++ app/package-lock.json | 21 +++++++++++++++++++++ app/package.json | 1 + 3 files changed, 25 insertions(+) create mode 100644 app/.prettierrc.js diff --git a/app/.prettierrc.js b/app/.prettierrc.js new file mode 100644 index 000000000..5dc5d8a8e --- /dev/null +++ b/app/.prettierrc.js @@ -0,0 +1,3 @@ +module.exports = { + ...require('prettier-airbnb-config'), +}; diff --git a/app/package-lock.json b/app/package-lock.json index 6a5317d58..4f4543c08 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -25,6 +25,7 @@ "maplibre-gl": "^4.4.1", "next": "^14.2.7", "pmtiles": "^3.0.7", + "prettier-airbnb-config": "^1.0.0", "react": "^18", "react-dom": "^18", "react-resizable": "^3.0.5", @@ -10938,6 +10939,26 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", + "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", + "peer": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/prettier-airbnb-config": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-airbnb-config/-/prettier-airbnb-config-1.0.0.tgz", + "integrity": "sha512-FsEe38fJftYi19AskzuOWbw8zBnXDcZwv2Uw8BI65ROyK4hE6+iojXVFu+FiYNfrRBL+UZJbl2XGk56Ur2QGsA==", + "peerDependencies": { + "prettier": "^1.18.2" + } + }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", diff --git a/app/package.json b/app/package.json index 49b94314c..44793d00b 100644 --- a/app/package.json +++ b/app/package.json @@ -46,6 +46,7 @@ "jest": "^29.7.0", "nock": "^13.5.4", "postcss": "^8", + "prettier-airbnb-config": "^1.0.0", "tailwindcss": "^3.4.1", "typescript": "^5" } From a4e15cc59c23888313ea7f71d5b9fda0763c0344 Mon Sep 17 00:00:00 2001 From: mariogiampieri Date: Wed, 30 Oct 2024 21:29:27 -0400 Subject: [PATCH 21/50] Feat/zoom to view (#147) --- .github/workflows/fly-deploy-pr.yml | 3 +- app/src/app/components/Map.tsx | 14 ++- app/src/app/constants/configuration.ts | 4 +- app/src/app/constants/layers.ts | 47 +++++----- app/src/app/utils/api/apiHandlers.ts | 1 + app/src/app/utils/api/mutations.ts | 11 +-- .../dc391733e10a_add_extent_to_gerrydb.py | 87 +++++++++++++++++++ backend/app/main.py | 3 + backend/app/models.py | 4 + backend/app/sql/create_districtr_map_udf.sql | 15 +++- backend/app/utils.py | 7 +- backend/cli.py | 13 ++- 12 files changed, 166 insertions(+), 43 deletions(-) create mode 100644 backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index 39a115ffc..b32a78fe0 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -109,7 +109,8 @@ jobs: flyctl deploy \ --config fly.toml --app "${{ github.event.repository.name }}-${{ github.event.number }}-api" \ - --strategy immediate '--ha=false' --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 + --strategy immediate '--ha=false' --vm-cpu-kind shared --vm-cpus 1 --vm-memory 256 \ + working-directory: backend - name: Check and Launch Frontend App diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index 9c530c95e..f33be4c3e 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -16,8 +16,8 @@ export const MapComponent: React.FC = () => { const map: MutableRefObject = useRef(null); const mapContainer: MutableRefObject = useRef(null); const mapLock = useMapStore((state) => state.mapLock); - const setMapRef = useMapStore((state) => state.setMapRef); + const mapOptions = useMapStore((state) => state.mapOptions); useEffect(() => { let protocol = new Protocol(); @@ -28,8 +28,18 @@ export const MapComponent: React.FC = () => { }, []); useEffect(() => { - if (map.current || !mapContainer.current) return; + if (map.current && mapOptions.bounds) { + if (mapOptions.bounds) { + map.current.fitBounds(mapOptions.bounds, { + padding: 20, + }); + } + } + }, [mapOptions]); + useEffect(() => { + if (map.current || !mapContainer.current) return; + map.current = new maplibregl.Map({ container: mapContainer.current, style: MAP_OPTIONS.style, diff --git a/app/src/app/constants/configuration.ts b/app/src/app/constants/configuration.ts index 9889caf89..b74f5a1f5 100644 --- a/app/src/app/constants/configuration.ts +++ b/app/src/app/constants/configuration.ts @@ -2,7 +2,7 @@ import { LngLatLike } from "maplibre-gl"; import type { MapOptions, StyleSpecification } from "maplibre-gl"; import { BASEMAP_LAYERS } from "./basemapLayers"; -export const MAP_CENTER: LngLatLike = [-105.358887, 39.113014]; // colorado +export const MAP_CENTER: LngLatLike = [-98.5556199, 39.8097343]; // kansas export const BASEMAP_STYLE: StyleSpecification = { version: 8, @@ -25,7 +25,7 @@ export const BASEMAP_STYLE: StyleSpecification = { export const MAP_OPTIONS: MapOptions = { style: BASEMAP_STYLE, - zoom: 6.75, + zoom: 3.75, center: MAP_CENTER, maxZoom: 22, minZoom: 3, diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 9cf524a5d..8924620b7 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -18,17 +18,11 @@ export const BLOCK_HOVER_LAYER_ID_CHILD = `${BLOCK_LAYER_ID_CHILD}-hover`; export const INTERACTIVE_LAYERS = [ BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CHILD, -] +]; -export const PARENT_LAYERS = [ - BLOCK_LAYER_ID, - BLOCK_HOVER_LAYER_ID -] +export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; -export const CHILD_LAYERS = [ - BLOCK_LAYER_ID_CHILD, - BLOCK_HOVER_LAYER_ID_CHILD -] +export const CHILD_LAYERS = [BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD]; export const DEFAULT_PAINT_STYLE: ExpressionSpecification = [ "case", @@ -83,7 +77,7 @@ export function getLayerFilter( export function getBlocksLayerSpecification( sourceLayer: string, - layerId: string, + layerId: string ): LayerSpecification { const shatterIds = useMapStore.getState().shatterIds; return { @@ -109,7 +103,7 @@ export function getBlocksLayerSpecification( export function getBlocksHoverLayerSpecification( sourceLayer: string, - layerId: string, + layerId: string ): LayerSpecification { return { id: layerId, @@ -163,10 +157,7 @@ export function getBlocksHoverLayerSpecification( }; } -const addBlockLayers = ( - map: Map | null, - mapDocument: DocumentObject, -) => { +const addBlockLayers = (map: Map | null, mapDocument: DocumentObject) => { if (!map || !mapDocument.tiles_s3_path) { console.log("map or mapDocument not ready", mapDocument); return; @@ -176,39 +167,45 @@ const addBlockLayers = ( map?.addSource(BLOCK_SOURCE_ID, blockSource); map?.addLayer( getBlocksLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); map?.addLayer( getBlocksHoverLayerSpecification( mapDocument.parent_layer, - BLOCK_HOVER_LAYER_ID, + BLOCK_HOVER_LAYER_ID ), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); if (mapDocument.child_layer) { map?.addLayer( getBlocksLayerSpecification( mapDocument.child_layer, - BLOCK_LAYER_ID_CHILD, + BLOCK_LAYER_ID_CHILD ), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); map?.addLayer( getBlocksHoverLayerSpecification( mapDocument.child_layer, - BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD ), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); } - useMapStore.getState().setMapRenderingState("loaded") + useMapStore.getState().setMapRenderingState("loaded"); + + // update map bounds based on document extent + useMapStore.getState().setMapOptions({ + bounds: mapDocument.extent as [number, number, number, number], + container: useMapStore.getState().mapOptions.container, + }); }; export function removeBlockLayers(map: Map | null) { if (!map) { - return + return; } - useMapStore.getState().setMapRenderingState("loading") + useMapStore.getState().setMapRenderingState("loading"); if (map.getLayer(BLOCK_LAYER_ID)) { map.removeLayer(BLOCK_LAYER_ID); } diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 02c67fd22..27c899f88 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -63,6 +63,7 @@ export interface DocumentObject { num_districts: number | null; created_at: string; updated_at: string | null; + extent: [number, number, number, number]; // [minx, miny, maxx, maxy] } /** diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts index ed9ac5425..be1e7df90 100644 --- a/app/src/app/utils/api/mutations.ts +++ b/app/src/app/utils/api/mutations.ts @@ -8,6 +8,7 @@ import { } from "@/app/utils/api/apiHandlers"; import { useMapStore } from "@/app/store/mapStore"; import { mapMetrics } from "./queries"; +import { use } from "react"; export const patchShatter = new MutationObserver(queryClient, { mutationFn: patchShatterParents, @@ -50,17 +51,17 @@ export const document = new MutationObserver(queryClient, { mutationFn: createMapDocument, onMutate: () => { console.log("Creating document"); - useMapStore.getState().setAppLoadingState('loading') - useMapStore.getState().resetZoneAssignments() + useMapStore.getState().setAppLoadingState("loading"); + useMapStore.getState().resetZoneAssignments(); }, onError: (error) => { console.error("Error creating map document: ", error); }, onSuccess: (data) => { useMapStore.getState().setMapDocument(data); - useMapStore.getState().setAppLoadingState('loaded') - const documentUrl = new URL(window.location.toString()) + useMapStore.getState().setAppLoadingState("loaded"); + const documentUrl = new URL(window.location.toString()); documentUrl.searchParams.set("document_id", data.document_id); history.pushState({}, "", documentUrl.toString()); }, -}); \ No newline at end of file +}); diff --git a/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py b/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py new file mode 100644 index 000000000..e7c06d8d0 --- /dev/null +++ b/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py @@ -0,0 +1,87 @@ +"""add extent to gerrydb + +Revision ID: dc391733e10a +Revises: 5ab466c5650a +Create Date: 2024-10-27 19:38:13.798056 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from app.models import DistrictrMap + +# revision identifiers, used by Alembic. +revision: str = "dc391733e10a" +down_revision: Union[str, None] = "5ab466c5650a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on = ("167892041d95",) # geometry col renaming +down_revision = "167892041d95" + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Make column nullable to avoid errors during initial insert + op.add_column( + "districtrmap", sa.Column("extent", sa.ARRAY(sa.Float()), nullable=True) + ) + + # Use a session to interact with the database + bind = op.get_bind() + Session = sa.orm.sessionmaker(bind=bind) + session = Session() + + # retreive table names from the districtrmap parent_layer field + table_names = session.execute(sa.select(DistrictrMap.parent_layer)).scalars().all() + + if not table_names: + raise ValueError("No matching table names found in the index table") + + cases = [] + for table_name in table_names: + case = f""" + WHEN districtrmap."parent_layer" = '{table_name}' THEN ( + SELECT + ARRAY[ + ST_XMin(ST_Extent(ST_Transform(geometry, 4326))), + ST_YMin(ST_Extent(ST_Transform(geometry, 4326))), + ST_XMax(ST_Extent(ST_Transform(geometry, 4326))), + ST_YMax(ST_Extent(ST_Transform(geometry, 4326))) + ] + FROM gerrydb."{table_name}", public.districtrmap + + ) + """ + + cases.append(case) + # Combine all cases into a single SQL statement + case_statement = " ".join(cases) + # Execute a single UPDATE statement + update_query = sa.text( + f""" + UPDATE districtrmap + SET extent = CASE + {case_statement} + ELSE ARRAY[-102.0517,36.99301,-94.5883,40.0031] + -- if this fails, there is no + -- matching table from parent_layer + -- and that's a problem + END; + """ + ) + + bind.execute(update_query) + + # Make the `extent` column non-nullable + op.alter_column( + "districtrmap", "extent", existing_type=sa.ARRAY(sa.Float()), nullable=False + ) + + session.commit() + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("districtrmap", "extent") + # ### end Alembic commands ### diff --git a/backend/app/main.py b/backend/app/main.py index 209095c34..2b4657a5b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -99,6 +99,7 @@ async def create_document( DistrictrMap.child_layer.label("child_layer"), # pyright: ignore DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore + DistrictrMap.extent.label("extent"), # pyright: ignore ) .where(Document.document_id == document_id) .join( @@ -210,6 +211,7 @@ async def get_document(document_id: str, session: Session = Depends(get_session) DistrictrMap.child_layer.label("child_layer"), # pyright: ignore DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore + DistrictrMap.extent.label("extent"), # pyright: ignore ) # pyright: ignore .where(Document.document_id == document_id) .join( @@ -220,6 +222,7 @@ async def get_document(document_id: str, session: Session = Depends(get_session) .limit(1) ) result = session.exec(stmt) + return result.one() diff --git a/backend/app/models.py b/backend/app/models.py index faf842126..13103718d 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -13,6 +13,8 @@ MetaData, String, ) +from sqlalchemy.types import ARRAY +from sqlalchemy import Float from app.constants import DOCUMENT_SCHEMA @@ -62,6 +64,7 @@ class DistrictrMap(TimeStampMixin, SQLModel, table=True): String, ForeignKey("gerrydbtable.name"), default=None, nullable=True ) ) + extent: list[float] = Field(sa_column=Column(ARRAY(Float), nullable=False)) # schema? will need to contrain the schema # where does this go? # when you create the view, pull the columns that you need @@ -126,6 +129,7 @@ class DocumentPublic(BaseModel): num_districts: int | None = None created_at: datetime updated_at: datetime + extent: list[float] class AssignmentsBase(SQLModel): diff --git a/backend/app/sql/create_districtr_map_udf.sql b/backend/app/sql/create_districtr_map_udf.sql index 08806ea82..4fa89aca1 100644 --- a/backend/app/sql/create_districtr_map_udf.sql +++ b/backend/app/sql/create_districtr_map_udf.sql @@ -18,10 +18,21 @@ BEGIN num_districts, tiles_s3_path, parent_layer, - child_layer + child_layer, + -- calculate extent based on parent_layer extent + select( + ARRAY[ + ST_XMin(ST_Extent(ST_Transform(geometry, 4326))), + ST_YMin(ST_Extent(ST_Transform(geometry, 4326))), + ST_XMax(ST_Extent(ST_Transform(geometry, 4326))), + ST_YMax(ST_Extent(ST_Transform(geometry, 4326))) + ] + FROM gerrydb.$5, public.districtrmap + WHERE districtrmap.parent_layer = $5 + ) ) VALUES (now(), gen_random_uuid(), $1, $2, $3, $4, $5, $6) RETURNING uuid INTO inserted_districtr_uuid; RETURN inserted_districtr_uuid; END; -$$ LANGUAGE plpgsql; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/backend/app/utils.py b/backend/app/utils.py index 0ac723b76..06e007ce2 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -2,6 +2,7 @@ from sqlalchemy import bindparam, Integer, String, Text from sqlmodel import Session + from app.models import UUIDType @@ -29,7 +30,8 @@ def create_districtr_map( Returns: The UUID of the inserted map. """ - stmt = text(""" + stmt = text( + """ SELECT * FROM create_districtr_map( :map_name, @@ -38,7 +40,8 @@ def create_districtr_map( :tiles_s3_path, :parent_layer_name, :child_layer_name - )""").bindparams( + )""" + ).bindparams( bindparam(key="map_name", type_=String), bindparam(key="gerrydb_table_name", type_=String), bindparam(key="num_districts", type_=Integer), diff --git a/backend/cli.py b/backend/cli.py index 9f51d91c7..e99e149b7 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -12,6 +12,7 @@ create_districtr_map as _create_districtr_map, create_shatterable_gerrydb_view as _create_shatterable_gerrydb_view, create_parent_child_edges as _create_parent_child_edges, + transform_bounding_box, ) logger = logging.getLogger(__name__) @@ -111,13 +112,15 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): _session = get_session() session = next(_session) - upsert_query = text(""" + upsert_query = text( + """ INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) ON CONFLICT (name) DO UPDATE SET updated_at = now() - """) + """ + ) try: session.execute( @@ -163,10 +166,12 @@ def delete_parent_child_edges(districtr_map: str): session = next(get_session()) - delete_query = text(""" + delete_query = text( + """ DELETE FROM parentchildedges WHERE districtr_map = :districtr_map - """) + """ + ) session.execute( delete_query, { From 848dda4e67f7eaedf07f6753854460a5cecc0bd8 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 30 Oct 2024 22:00:06 -0400 Subject: [PATCH 22/50] Apply prettier config to app (#158) --- app/package-lock.json | 4 +- app/src/app/api/sentry-example-api/route.ts | 8 +- app/src/app/components/ContextMenu.tsx | 19 +-- .../components/sidebar/BrushSizeSelector.tsx | 17 +- .../sidebar/GerryDBViewSelector.tsx | 44 ++--- app/src/app/components/sidebar/Layers.tsx | 33 ++-- .../app/components/sidebar/PaintByCounty.tsx | 25 ++- .../components/sidebar/RecentMapsModal.tsx | 44 +++-- .../app/components/sidebar/ResetMapButton.tsx | 6 +- app/src/app/components/sidebar/Sidebar.tsx | 35 ++-- .../sidebar/charts/HorizontalBarChart.tsx | 41 ++--- app/src/app/constants/colors.ts | 80 +++++----- app/src/app/constants/layers.ts | 151 +++++++----------- app/src/app/constants/sources.ts | 10 +- app/src/app/global-error.tsx | 12 +- app/src/app/map/page.tsx | 15 +- app/src/app/page.tsx | 11 +- app/src/app/sentry-example-page/page.tsx | 66 ++++---- app/src/app/store/mapEditSubs.ts | 67 ++++---- app/src/app/store/mapRenderSubs.ts | 95 ++++++----- app/src/app/store/metricsSubs.ts | 15 +- app/src/app/store/persistConfig.ts | 14 +- app/src/app/utils/api/apiHandlers.ts | 61 +++---- app/src/app/utils/api/mutations.ts | 50 +++--- app/src/app/utils/api/queries.ts | 84 +++++----- app/src/app/utils/api/queryClient.ts | 4 +- app/src/app/utils/api/queryParamsListener.ts | 20 ++- app/src/app/utils/arrays.ts | 2 +- app/src/app/utils/events/handlers.ts | 42 +++-- app/src/app/utils/helpers.ts | 128 ++++++--------- 30 files changed, 532 insertions(+), 671 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 4f4543c08..ce236f06c 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -25,7 +25,6 @@ "maplibre-gl": "^4.4.1", "next": "^14.2.7", "pmtiles": "^3.0.7", - "prettier-airbnb-config": "^1.0.0", "react": "^18", "react-dom": "^18", "react-resizable": "^3.0.5", @@ -46,6 +45,7 @@ "jest": "^29.7.0", "nock": "^13.5.4", "postcss": "^8", + "prettier-airbnb-config": "^1.0.0", "tailwindcss": "^3.4.1", "typescript": "^5" } @@ -10943,6 +10943,7 @@ "version": "1.19.1", "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", + "dev": true, "peer": true, "bin": { "prettier": "bin-prettier.js" @@ -10955,6 +10956,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/prettier-airbnb-config/-/prettier-airbnb-config-1.0.0.tgz", "integrity": "sha512-FsEe38fJftYi19AskzuOWbw8zBnXDcZwv2Uw8BI65ROyK4hE6+iojXVFu+FiYNfrRBL+UZJbl2XGk56Ur2QGsA==", + "dev": true, "peerDependencies": { "prettier": "^1.18.2" } diff --git a/app/src/app/api/sentry-example-api/route.ts b/app/src/app/api/sentry-example-api/route.ts index f486f3d1d..aad6597e8 100644 --- a/app/src/app/api/sentry-example-api/route.ts +++ b/app/src/app/api/sentry-example-api/route.ts @@ -1,9 +1,9 @@ -import { NextResponse } from "next/server"; +import {NextResponse} from 'next/server'; -export const dynamic = "force-dynamic"; +export const dynamic = 'force-dynamic'; // A faulty API route to test Sentry's error monitoring export function GET() { - throw new Error("Sentry Example API Route Error"); - return NextResponse.json({ data: "Testing Sentry Error..." }); + throw new Error('Sentry Example API Route Error'); + return NextResponse.json({data: 'Testing Sentry Error...'}); } diff --git a/app/src/app/components/ContextMenu.tsx b/app/src/app/components/ContextMenu.tsx index ed1a04f88..ecaeea7d9 100644 --- a/app/src/app/components/ContextMenu.tsx +++ b/app/src/app/components/ContextMenu.tsx @@ -1,11 +1,11 @@ -import React from "react"; -import { ContextMenu, Text } from "@radix-ui/themes"; -import { useMapStore } from "@/app/store/mapStore"; +import React from 'react'; +import {ContextMenu, Text} from '@radix-ui/themes'; +import {useMapStore} from '@/app/store/mapStore'; export const MapContextMenu: React.FC = () => { - const mapDocument = useMapStore((state) => state.mapDocument); - const contextMenu = useMapStore((state) => state.contextMenu); - const handleShatter = useMapStore((state) => state.handleShatter); + const mapDocument = useMapStore(state => state.mapDocument); + const contextMenu = useMapStore(state => state.contextMenu); + const handleShatter = useMapStore(state => state.handleShatter); if (!contextMenu) return null; const handleSelect = () => { @@ -27,7 +27,7 @@ export const MapContextMenu: React.FC = () => { // also, if in the future we need the context menu outside of the map, // this sets us up to do that style={{ - position: "fixed", + position: 'fixed', top: contextMenu.y, left: contextMenu.x, }} @@ -39,10 +39,7 @@ export const MapContextMenu: React.FC = () => { )} - + Shatter diff --git a/app/src/app/components/sidebar/BrushSizeSelector.tsx b/app/src/app/components/sidebar/BrushSizeSelector.tsx index e6efa0ebf..d962a0a2f 100644 --- a/app/src/app/components/sidebar/BrushSizeSelector.tsx +++ b/app/src/app/components/sidebar/BrushSizeSelector.tsx @@ -1,5 +1,5 @@ -import { Slider, Flex, Heading, Text } from "@radix-ui/themes"; -import { useMapStore } from "../../store/mapStore"; +import {Slider, Flex, Heading, Text} from '@radix-ui/themes'; +import {useMapStore} from '../../store/mapStore'; /** * BrushSizeSelector @@ -11,22 +11,17 @@ import { useMapStore } from "../../store/mapStore"; * @returns {JSX.Element} The component */ export function BrushSizeSelector() { - const brushSize = useMapStore((state) => state.brushSize); - const setBrushSize = useMapStore((state) => state.setBrushSize); + const brushSize = useMapStore(state => state.brushSize); + const setBrushSize = useMapStore(state => state.setBrushSize); const handleChangeEnd = (value: Array) => { - console.log("the final value size is", value); + console.log('the final value size is', value); setBrushSize(value.length ? value[0] : 0); }; return ( - + Brush Size (30); const [offset, setOffset] = useState(0); - const mapDocument = useMapStore((state) => state.mapDocument); - const mapViews = useMapStore((state) => state.mapViews); - const { isPending, isError, data, error } = mapViews || {}; + const mapDocument = useMapStore(state => state.mapDocument); + const mapViews = useMapStore(state => state.mapViews); + const {isPending, isError, data, error} = mapViews || {}; - const selectedView = data?.find( - (view) => view.gerrydb_table_name === mapDocument?.gerrydb_table - ); + const selectedView = data?.find(view => view.gerrydb_table_name === mapDocument?.gerrydb_table); const handleValueChange = (value: string) => { - console.log("Value changed: ", value); - const selectedDistrictrMap = data?.find((view) => view.name === value); - console.log("Selected view: ", selectedDistrictrMap); + console.log('Value changed: ', value); + const selectedDistrictrMap = data?.find(view => view.name === value); + console.log('Selected view: ', selectedDistrictrMap); if ( !selectedDistrictrMap || selectedDistrictrMap.gerrydb_table_name === mapDocument?.gerrydb_table ) { - console.log("No document or same document"); + console.log('No document or same document'); return; } - console.log("mutating to create new document"); - document.mutate({ gerrydb_table: selectedDistrictrMap.gerrydb_table_name }); + console.log('mutating to create new document'); + document.mutate({gerrydb_table: selectedDistrictrMap.gerrydb_table_name}); }; if (isPending) return
Loading geographies... 🌎
; @@ -34,17 +32,9 @@ export function GerryDBViewSelector() { if (isError) return
Error loading geographies: {error?.message}
; return ( - - - + + + Districtr map options diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index 7ba84ee49..86b761710 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -1,11 +1,7 @@ -import { Heading, CheckboxGroup, Flex } from "@radix-ui/themes"; -import { useMapStore } from "@/app/store/mapStore"; -import { - COUNTY_LAYER_IDS, - BLOCK_LAYER_ID, - BLOCK_HOVER_LAYER_ID, -} from "../../constants/layers"; -import { toggleLayerVisibility } from "../../utils/helpers"; +import {Heading, CheckboxGroup, Flex} from '@radix-ui/themes'; +import {useMapStore} from '@/app/store/mapStore'; +import {COUNTY_LAYER_IDS, BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID} from '../../constants/layers'; +import {toggleLayerVisibility} from '../../utils/helpers'; /** Layers * This component is responsible for rendering the layers that can be toggled @@ -16,10 +12,10 @@ import { toggleLayerVisibility } from "../../utils/helpers"; * - Support tribes and communities */ export default function Layers() { - const mapRef = useMapStore((state) => state.getMapRef()); - const mapDocument = useMapStore((state) => state.mapDocument); - const visibleLayerIds = useMapStore((state) => state.visibleLayerIds); - const updateVisibleLayerIds = useMapStore((state) => state.updateVisibleLayerIds); + const mapRef = useMapStore(state => state.getMapRef()); + const mapDocument = useMapStore(state => state.mapDocument); + const visibleLayerIds = useMapStore(state => state.visibleLayerIds); + const updateVisibleLayerIds = useMapStore(state => state.updateVisibleLayerIds); const toggleLayers = (layerIds: string[]) => { if (!mapRef) return; @@ -35,7 +31,7 @@ export default function Layers() { visibleLayerIds.includes(layerId)) - ? ["1"] - : [] - } + value={COUNTY_LAYER_IDS.every(layerId => visibleLayerIds.includes(layerId)) ? ['1'] : []} > - toggleLayers(COUNTY_LAYER_IDS)} - > + toggleLayers(COUNTY_LAYER_IDS)}> Show county boundaries diff --git a/app/src/app/components/sidebar/PaintByCounty.tsx b/app/src/app/components/sidebar/PaintByCounty.tsx index 4dee72031..88e72afa4 100644 --- a/app/src/app/components/sidebar/PaintByCounty.tsx +++ b/app/src/app/components/sidebar/PaintByCounty.tsx @@ -1,24 +1,21 @@ -import { Box, Text, Checkbox, Flex } from "@radix-ui/themes"; -import { useMapStore } from "@/app/store/mapStore"; -import { COUNTY_LAYER_IDS } from "../../constants/layers"; -import { useState, useEffect } from "react"; -import { - getFeaturesInBbox, - getFeaturesIntersectingCounties, -} from "../../utils/helpers"; +import {Box, Text, Checkbox, Flex} from '@radix-ui/themes'; +import {useMapStore} from '@/app/store/mapStore'; +import {COUNTY_LAYER_IDS} from '../../constants/layers'; +import {useState, useEffect} from 'react'; +import {getFeaturesInBbox, getFeaturesIntersectingCounties} from '../../utils/helpers'; export default function PaintByCounty() { - const mapRef = useMapStore((state) => state.getMapRef()); - const addVisibleLayerIds = useMapStore((state) => state.addVisibleLayerIds); - const setPaintFunction = useMapStore((state) => state.setPaintFunction); + const mapRef = useMapStore(state => state.getMapRef()); + const addVisibleLayerIds = useMapStore(state => state.addVisibleLayerIds); + const setPaintFunction = useMapStore(state => state.setPaintFunction); const [checked, setChecked] = useState(false); useEffect(() => { if (!mapRef) return; if (checked) { - COUNTY_LAYER_IDS.forEach((layerId) => { - mapRef.setLayoutProperty(layerId, "visibility", "visible"); + COUNTY_LAYER_IDS.forEach(layerId => { + mapRef.setLayoutProperty(layerId, 'visibility', 'visible'); }); addVisibleLayerIds(COUNTY_LAYER_IDS); setPaintFunction(getFeaturesIntersectingCounties); @@ -34,7 +31,7 @@ export default function PaintByCounty() { setChecked((prevIsChecked) => !prevIsChecked)} + onClick={() => setChecked(prevIsChecked => !prevIsChecked)} /> Paint by County diff --git a/app/src/app/components/sidebar/RecentMapsModal.tsx b/app/src/app/components/sidebar/RecentMapsModal.tsx index 746f53e49..3226e2b0b 100644 --- a/app/src/app/components/sidebar/RecentMapsModal.tsx +++ b/app/src/app/components/sidebar/RecentMapsModal.tsx @@ -1,6 +1,6 @@ -import { useMapStore } from "@/app/store/mapStore"; -import React from "react"; -import { Cross2Icon, CounterClockwiseClockIcon } from "@radix-ui/react-icons"; +import {useMapStore} from '@/app/store/mapStore'; +import React from 'react'; +import {Cross2Icon, CounterClockwiseClockIcon} from '@radix-ui/react-icons'; import { Button, Flex, @@ -11,25 +11,25 @@ import { TextField, IconButton, RadioCards, -} from "@radix-ui/themes"; -import { usePathname, useSearchParams, useRouter } from "next/navigation"; -import { DocumentObject } from "../../utils/api/apiHandlers"; -type NamedDocumentObject = DocumentObject & { name?: string }; +} from '@radix-ui/themes'; +import {usePathname, useSearchParams, useRouter} from 'next/navigation'; +import {DocumentObject} from '../../utils/api/apiHandlers'; +type NamedDocumentObject = DocumentObject & {name?: string}; export const RecentMapsModal = () => { const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - const mapDocument = useMapStore((store) => store.mapDocument); - const userMaps = useMapStore((store) => store.userMaps); - const upcertUserMap = useMapStore((store) => store.upcertUserMap); - const setMapDocument = useMapStore((store) => store.setMapDocument); + const mapDocument = useMapStore(store => store.mapDocument); + const userMaps = useMapStore(store => store.userMaps); + const upcertUserMap = useMapStore(store => store.upcertUserMap); + const setMapDocument = useMapStore(store => store.setMapDocument); const [dialogOpen, setDialogOpen] = React.useState(false); const handleMapDocument = (data: NamedDocumentObject) => { setMapDocument(data); const urlParams = new URLSearchParams(searchParams.toString()); - urlParams.set("document_id", data.document_id); - router.push(pathname + "?" + urlParams.toString()); + urlParams.set('document_id', data.document_id); + router.push(pathname + '?' + urlParams.toString()); // close dialog setDialogOpen(false); }; @@ -48,9 +48,7 @@ export const RecentMapsModal = () => { - - Recent Maps - + Recent Maps { - - Map Name - + Map Name Last Updated {/* load */} {/* delete */} @@ -76,7 +72,7 @@ export const RecentMapsModal = () => { + onChange={userMapData => upcertUserMap({ userMapData, userMapDocumentId: userMap.document_id, @@ -98,17 +94,17 @@ const RecentMapsRow: React.FC<{ onSelect: (data: NamedDocumentObject) => void; active: boolean; onChange?: (data?: NamedDocumentObject) => void; -}> = ({ data, onSelect, active, onChange }) => { +}> = ({data, onSelect, active, onChange}) => { const updatedDate = new Date(data.updated_at as string); const formattedData = updatedDate.toLocaleDateString(); const name = data?.name || data.gerrydb_table; const handleChangeName = (name?: string) => { - name?.length && onChange?.({ ...data, name }); + name?.length && onChange?.({...data, name}); }; return ( - + {!!(active && onChange) ? ( @@ -116,7 +112,7 @@ const RecentMapsRow: React.FC<{ placeholder={name} size="3" value={name} - onChange={(e) => handleChangeName(e.target.value)} + onChange={e => handleChangeName(e.target.value)} > ) : ( diff --git a/app/src/app/components/sidebar/ResetMapButton.tsx b/app/src/app/components/sidebar/ResetMapButton.tsx index 4e289b57f..06f8af11b 100644 --- a/app/src/app/components/sidebar/ResetMapButton.tsx +++ b/app/src/app/components/sidebar/ResetMapButton.tsx @@ -1,5 +1,5 @@ -import { useMapStore } from "@/app/store/mapStore"; -import { Button } from "@radix-ui/themes"; +import {useMapStore} from '@/app/store/mapStore'; +import {Button} from '@radix-ui/themes'; export function ResetMapButton() { const mapStore = useMapStore.getState(); @@ -11,7 +11,7 @@ export function ResetMapButton() { }; return ( - ); diff --git a/app/src/app/components/sidebar/Sidebar.tsx b/app/src/app/components/sidebar/Sidebar.tsx index 1c1aed307..4b0fc833a 100644 --- a/app/src/app/components/sidebar/Sidebar.tsx +++ b/app/src/app/components/sidebar/Sidebar.tsx @@ -1,36 +1,33 @@ -import { Box, Flex, Heading } from "@radix-ui/themes"; -import { MapModeSelector } from "./MapModeSelector"; -import { ColorPicker } from "./ColorPicker"; -import { ResetMapButton } from "./ResetMapButton"; -import { GerryDBViewSelector } from "./GerryDBViewSelector"; -import { HorizontalBar } from "./charts/HorizontalBarChart"; -import { useMapStore } from "@/app/store/mapStore"; -import { Tabs, Text } from "@radix-ui/themes"; -import Layers from "./Layers"; -import PaintByCounty from "./PaintByCounty"; -import { BrushSizeSelector } from "./BrushSizeSelector"; +import {Box, Flex, Heading} from '@radix-ui/themes'; +import {MapModeSelector} from './MapModeSelector'; +import {ColorPicker} from './ColorPicker'; +import {ResetMapButton} from './ResetMapButton'; +import {GerryDBViewSelector} from './GerryDBViewSelector'; +import {HorizontalBar} from './charts/HorizontalBarChart'; +import {useMapStore} from '@/app/store/mapStore'; +import {Tabs, Text} from '@radix-ui/themes'; +import Layers from './Layers'; +import PaintByCounty from './PaintByCounty'; +import {BrushSizeSelector} from './BrushSizeSelector'; export default function SidebarComponent() { - const activeTool = useMapStore((state) => state.activeTool); + const activeTool = useMapStore(state => state.activeTool); return ( - + Districtr - {activeTool === "brush" || activeTool === "eraser" ? ( + {activeTool === 'brush' || activeTool === 'eraser' ? (
- {" "} + {' '}
) : null} - {activeTool === "brush" ? ( + {activeTool === 'brush' ? (
diff --git a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx index 8c4d609ba..74ed8b4da 100644 --- a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx +++ b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx @@ -1,24 +1,16 @@ -import { useMapStore } from "@/app/store/mapStore"; -import { Card, Flex, Heading, Text } from "@radix-ui/themes"; -import { - BarChart, - Bar, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, - Cell, -} from "recharts"; -import { colorScheme } from "@/app/constants/colors"; +import {useMapStore} from '@/app/store/mapStore'; +import {Card, Flex, Heading, Text} from '@radix-ui/themes'; +import {BarChart, Bar, ResponsiveContainer, Tooltip, XAxis, YAxis, Cell} from 'recharts'; +import {colorScheme} from '@/app/constants/colors'; type TooltipInput = { active?: boolean; - payload?: [{ payload: { total_pop: number; zone: number } }]; + payload?: [{payload: {total_pop: number; zone: number}}]; }; -const numberFormat = new Intl.NumberFormat("en-US"); +const numberFormat = new Intl.NumberFormat('en-US'); -const CustomTooltip = ({ active, payload: items }: TooltipInput) => { +const CustomTooltip = ({active, payload: items}: TooltipInput) => { if (active && items && items.length) { const payload = items[0].payload; return ( @@ -31,7 +23,7 @@ const CustomTooltip = ({ active, payload: items }: TooltipInput) => { }; export const HorizontalBar = () => { - const mapMetrics = useMapStore((state) => state.mapMetrics); + const mapMetrics = useMapStore(state => state.mapMetrics); if (mapMetrics?.isPending) { return
Loading...
; @@ -60,18 +52,12 @@ export const HorizontalBar = () => { height={colorScheme.length * 18} minHeight="200px" > - + numberFormat.format(value)} + domain={[0, 'maxData']} + tickFormatter={value => numberFormat.format(value)} /> } /> @@ -79,10 +65,7 @@ export const HorizontalBar = () => { {mapMetrics.data .sort((a, b) => a.zone - b.zone) .map((entry, index) => ( - + ))} diff --git a/app/src/app/constants/colors.ts b/app/src/app/constants/colors.ts index d4249155b..754cc9e44 100644 --- a/app/src/app/constants/colors.ts +++ b/app/src/app/constants/colors.ts @@ -25,49 +25,49 @@ import { mint, lime, sky, -} from "@radix-ui/colors"; +} from '@radix-ui/colors'; export const colorScheme = [ - "#0099cd", - "#ffca5d", - "#00cd99", - "#99cd00", - "#cd0099", - "#aa44ef", // lighter, req from San Diego + '#0099cd', + '#ffca5d', + '#00cd99', + '#99cd00', + '#cd0099', + '#aa44ef', // lighter, req from San Diego // Color brewer: - "#8dd3c7", - "#bebada", - "#fb8072", - "#80b1d3", - "#fdb462", - "#b3de69", - "#fccde5", - "#bc80bd", - "#ccebc5", - "#ffed6f", - "#ffffb3", + '#8dd3c7', + '#bebada', + '#fb8072', + '#80b1d3', + '#fdb462', + '#b3de69', + '#fccde5', + '#bc80bd', + '#ccebc5', + '#ffed6f', + '#ffffb3', // other color brewer scheme: - "#a6cee3", - "#1f78b4", - "#b2df8a", - "#33a02c", - "#fb9a99", - "#e31a1c", - "#fdbf6f", - "#ff7f00", - "#cab2d6", - "#6a3d9a", - "#b15928", + '#a6cee3', + '#1f78b4', + '#b2df8a', + '#33a02c', + '#fb9a99', + '#e31a1c', + '#fdbf6f', + '#ff7f00', + '#cab2d6', + '#6a3d9a', + '#b15928', // random material design colors: - "#64ffda", - "#00B8D4", - "#A1887F", - "#76FF03", - "#DCE775", - "#B388FF", - "#FF80AB", - "#D81B60", - "#26A69A", - "#FFEA00", - "#6200EA", + '#64ffda', + '#00B8D4', + '#A1887F', + '#76FF03', + '#DCE775', + '#B388FF', + '#FF80AB', + '#D81B60', + '#26A69A', + '#FFEA00', + '#6200EA', ]; diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 8924620b7..79b3404a2 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -1,77 +1,56 @@ -import { - ExpressionSpecification, - FilterSpecification, - LayerSpecification, -} from "maplibre-gl"; -import { Map } from "maplibre-gl"; -import { getBlocksSource } from "./sources"; -import { DocumentObject } from "../utils/api/apiHandlers"; -import { MapStore, useMapStore } from "../store/mapStore"; -import { colorScheme } from "./colors"; - -export const BLOCK_SOURCE_ID = "blocks"; -export const BLOCK_LAYER_ID = "blocks"; -export const BLOCK_LAYER_ID_CHILD = "blocks-child"; +import {ExpressionSpecification, FilterSpecification, LayerSpecification} from 'maplibre-gl'; +import {Map} from 'maplibre-gl'; +import {getBlocksSource} from './sources'; +import {DocumentObject} from '../utils/api/apiHandlers'; +import {MapStore, useMapStore} from '../store/mapStore'; +import {colorScheme} from './colors'; + +export const BLOCK_SOURCE_ID = 'blocks'; +export const BLOCK_LAYER_ID = 'blocks'; +export const BLOCK_LAYER_ID_CHILD = 'blocks-child'; export const BLOCK_HOVER_LAYER_ID = `${BLOCK_LAYER_ID}-hover`; export const BLOCK_HOVER_LAYER_ID_CHILD = `${BLOCK_LAYER_ID_CHILD}-hover`; -export const INTERACTIVE_LAYERS = [ - BLOCK_HOVER_LAYER_ID, - BLOCK_HOVER_LAYER_ID_CHILD, -]; +export const INTERACTIVE_LAYERS = [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CHILD]; export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; export const CHILD_LAYERS = [BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD]; export const DEFAULT_PAINT_STYLE: ExpressionSpecification = [ - "case", - ["boolean", ["feature-state", "hover"], false], - "#FF0000", - "#000000", + 'case', + ['boolean', ['feature-state', 'hover'], false], + '#FF0000', + '#000000', ]; -export const COUNTY_LAYER_IDS: string[] = [ - "counties_boundary", - "counties_labels", -]; +export const COUNTY_LAYER_IDS: string[] = ['counties_boundary', 'counties_labels']; -export const LABELS_BREAK_LAYER_ID = "places_subplace"; +export const LABELS_BREAK_LAYER_ID = 'places_subplace'; -const colorStyleBaseline: any[] = ["case"]; +const colorStyleBaseline: any[] = ['case']; -export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = colorScheme.reduce( - (val, color, i) => { - val.push(["==", ["feature-state", "zone"], i + 1], color); // 1-indexed per mapStore.ts - return val; - }, - colorStyleBaseline -); -ZONE_ASSIGNMENT_STYLE_DYNAMIC.push("#cecece"); +export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = colorScheme.reduce((val, color, i) => { + val.push(['==', ['feature-state', 'zone'], i + 1], color); // 1-indexed per mapStore.ts + return val; +}, colorStyleBaseline); +ZONE_ASSIGNMENT_STYLE_DYNAMIC.push('#cecece'); // cast the above as an ExpressionSpecification // @ts-ignore -export const ZONE_ASSIGNMENT_STYLE: ExpressionSpecification = - ZONE_ASSIGNMENT_STYLE_DYNAMIC; +export const ZONE_ASSIGNMENT_STYLE: ExpressionSpecification = ZONE_ASSIGNMENT_STYLE_DYNAMIC; -export function getLayerFilter( - layerId: string, - _shatterIds?: MapStore["shatterIds"] -) { +export function getLayerFilter(layerId: string, _shatterIds?: MapStore['shatterIds']) { const shatterIds = _shatterIds || useMapStore.getState().shatterIds; const isChildLayer = CHILD_LAYERS.includes(layerId); const ids = isChildLayer ? shatterIds.children : shatterIds.parents; const cleanIds = Boolean(ids) ? Array.from(ids) : []; - const filterBase: FilterSpecification = [ - "in", - ["get", "path"], - ["literal", cleanIds], - ]; + const filterBase: FilterSpecification = ['in', ['get', 'path'], ['literal', cleanIds]]; if (isChildLayer) { return filterBase; } - const parentFilter: FilterSpecification = ["!", filterBase]; + const parentFilter: FilterSpecification = ['!', filterBase]; return parentFilter; } @@ -83,20 +62,15 @@ export function getBlocksLayerSpecification( return { id: layerId, source: BLOCK_SOURCE_ID, - "source-layer": sourceLayer, - type: "line", + 'source-layer': sourceLayer, + type: 'line', layout: { - visibility: "visible", + visibility: 'visible', }, filter: getLayerFilter(layerId), paint: { - "line-opacity": [ - "case", - ["boolean", ["feature-state", "hover"], false], - 1, - 0.8, - ], - "line-color": "#cecece", + 'line-opacity': ['case', ['boolean', ['feature-state', 'hover'], false], 1, 0.8], + 'line-color': '#cecece', }, }; } @@ -108,58 +82,58 @@ export function getBlocksHoverLayerSpecification( return { id: layerId, source: BLOCK_SOURCE_ID, - "source-layer": sourceLayer, - type: "fill", + 'source-layer': sourceLayer, + type: 'fill', layout: { - visibility: "visible", + visibility: 'visible', }, filter: getLayerFilter(layerId), paint: { - "fill-opacity": [ - "case", + 'fill-opacity': [ + 'case', // zone is selected and hover is true and hover is not null [ - "all", + 'all', // @ts-ignore - ["!", ["==", ["feature-state", "zone"], null]], //< desired behavior but typerror + ['!', ['==', ['feature-state', 'zone'], null]], //< desired behavior but typerror [ - "all", + 'all', // @ts-ignore - ["!", ["==", ["feature-state", "hover"], null]], //< desired behavior but typerror - ["boolean", ["feature-state", "hover"], true], + ['!', ['==', ['feature-state', 'hover'], null]], //< desired behavior but typerror + ['boolean', ['feature-state', 'hover'], true], ], ], 0.9, // zone is selected and hover is false, and hover is not null [ - "all", + 'all', // @ts-ignore - ["!", ["==", ["feature-state", "zone"], null]], //< desired behavior but typerror + ['!', ['==', ['feature-state', 'zone'], null]], //< desired behavior but typerror [ - "all", + 'all', // @ts-ignore - ["!", ["==", ["feature-state", "hover"], null]], //< desired behavior but typerror - ["boolean", ["feature-state", "hover"], false], + ['!', ['==', ['feature-state', 'hover'], null]], //< desired behavior but typerror + ['boolean', ['feature-state', 'hover'], false], ], ], 0.7, // zone is selected, fallback, regardless of hover state // @ts-ignore - ["!", ["==", ["feature-state", "zone"], null]], //< desired behavior but typerror + ['!', ['==', ['feature-state', 'zone'], null]], //< desired behavior but typerror 0.7, // hover is true, fallback, regardless of zone state - ["boolean", ["feature-state", "hover"], false], + ['boolean', ['feature-state', 'hover'], false], 0.6, 0.2, ], - "fill-color": ZONE_ASSIGNMENT_STYLE || "#000000", + 'fill-color': ZONE_ASSIGNMENT_STYLE || '#000000', }, }; } const addBlockLayers = (map: Map | null, mapDocument: DocumentObject) => { if (!map || !mapDocument.tiles_s3_path) { - console.log("map or mapDocument not ready", mapDocument); + console.log('map or mapDocument not ready', mapDocument); return; } const blockSource = getBlocksSource(mapDocument.tiles_s3_path); @@ -170,30 +144,21 @@ const addBlockLayers = (map: Map | null, mapDocument: DocumentObject) => { LABELS_BREAK_LAYER_ID ); map?.addLayer( - getBlocksHoverLayerSpecification( - mapDocument.parent_layer, - BLOCK_HOVER_LAYER_ID - ), + getBlocksHoverLayerSpecification(mapDocument.parent_layer, BLOCK_HOVER_LAYER_ID), LABELS_BREAK_LAYER_ID ); if (mapDocument.child_layer) { map?.addLayer( - getBlocksLayerSpecification( - mapDocument.child_layer, - BLOCK_LAYER_ID_CHILD - ), + getBlocksLayerSpecification(mapDocument.child_layer, BLOCK_LAYER_ID_CHILD), LABELS_BREAK_LAYER_ID ); map?.addLayer( - getBlocksHoverLayerSpecification( - mapDocument.child_layer, - BLOCK_HOVER_LAYER_ID_CHILD - ), + getBlocksHoverLayerSpecification(mapDocument.child_layer, BLOCK_HOVER_LAYER_ID_CHILD), LABELS_BREAK_LAYER_ID ); } - useMapStore.getState().setMapRenderingState("loaded"); - + useMapStore.getState().setMapRenderingState('loaded'); + // update map bounds based on document extent useMapStore.getState().setMapOptions({ bounds: mapDocument.extent as [number, number, number, number], @@ -205,7 +170,7 @@ export function removeBlockLayers(map: Map | null) { if (!map) { return; } - useMapStore.getState().setMapRenderingState("loading"); + useMapStore.getState().setMapRenderingState('loading'); if (map.getLayer(BLOCK_LAYER_ID)) { map.removeLayer(BLOCK_LAYER_ID); } @@ -223,4 +188,4 @@ export function removeBlockLayers(map: Map | null) { } } -export { addBlockLayers }; +export {addBlockLayers}; diff --git a/app/src/app/constants/sources.ts b/app/src/app/constants/sources.ts index e4ef23318..d814b37e4 100644 --- a/app/src/app/constants/sources.ts +++ b/app/src/app/constants/sources.ts @@ -1,11 +1,9 @@ -import { VectorSourceSpecification } from "maplibre-gl"; +import {VectorSourceSpecification} from 'maplibre-gl'; -export function getBlocksSource( - layer_subpath: string -): VectorSourceSpecification { +export function getBlocksSource(layer_subpath: string): VectorSourceSpecification { return { - type: "vector", + type: 'vector', url: `pmtiles://${process.env.NEXT_PUBLIC_S3_BUCKET_URL}/${layer_subpath}`, - promoteId: "path", + promoteId: 'path', }; } diff --git a/app/src/app/global-error.tsx b/app/src/app/global-error.tsx index 9bda5feef..58469cc07 100644 --- a/app/src/app/global-error.tsx +++ b/app/src/app/global-error.tsx @@ -1,10 +1,10 @@ -"use client"; +'use client'; -import * as Sentry from "@sentry/nextjs"; -import NextError from "next/error"; -import { useEffect } from "react"; +import * as Sentry from '@sentry/nextjs'; +import NextError from 'next/error'; +import {useEffect} from 'react'; -export default function GlobalError({ error }: { error: Error & { digest?: string } }) { +export default function GlobalError({error}: {error: Error & {digest?: string}}) { useEffect(() => { Sentry.captureException(error); }, [error]); @@ -20,4 +20,4 @@ export default function GlobalError({ error }: { error: Error & { digest?: strin ); -} \ No newline at end of file +} diff --git a/app/src/app/map/page.tsx b/app/src/app/map/page.tsx index 916ac5d76..a2d3592c5 100644 --- a/app/src/app/map/page.tsx +++ b/app/src/app/map/page.tsx @@ -1,13 +1,12 @@ -"use client"; -import React from "react"; -import { MapContextMenu } from "../components/ContextMenu"; -import { MapComponent } from "../components/Map"; -import SidebarComponent from "../components/sidebar/Sidebar"; -import { QueryClientProvider } from "@tanstack/react-query"; -import { queryClient } from "../utils/api/queryClient"; +'use client'; +import React from 'react'; +import {MapContextMenu} from '../components/ContextMenu'; +import {MapComponent} from '../components/Map'; +import SidebarComponent from '../components/sidebar/Sidebar'; +import {QueryClientProvider} from '@tanstack/react-query'; +import {queryClient} from '../utils/api/queryClient'; export default function Map() { - if (queryClient) { return ( diff --git a/app/src/app/page.tsx b/app/src/app/page.tsx index 1f1f92158..c6b60f8bb 100644 --- a/app/src/app/page.tsx +++ b/app/src/app/page.tsx @@ -1,9 +1,8 @@ -"use client"; -import { MapComponent } from "./components/Map"; -import SidebarComponent from "./components/sidebar/Sidebar"; -import { QueryClientProvider } from "@tanstack/react-query"; -import { queryClient } from "./utils/api/queryClient"; - +'use client'; +import {MapComponent} from './components/Map'; +import SidebarComponent from './components/sidebar/Sidebar'; +import {QueryClientProvider} from '@tanstack/react-query'; +import {queryClient} from './utils/api/queryClient'; export default function Home() { return ( diff --git a/app/src/app/sentry-example-page/page.tsx b/app/src/app/sentry-example-page/page.tsx index 88914e7d1..7f2d2bfcd 100644 --- a/app/src/app/sentry-example-page/page.tsx +++ b/app/src/app/sentry-example-page/page.tsx @@ -1,7 +1,7 @@ -"use client"; +'use client'; -import Head from "next/head"; -import * as Sentry from "@sentry/nextjs"; +import Head from 'next/head'; +import * as Sentry from '@sentry/nextjs'; export default function Page() { return ( @@ -13,17 +13,17 @@ export default function Page() {
-

+

{ - await Sentry.startSpan({ - name: 'Example Frontend Span', - op: 'test' - }, async () => { - const res = await fetch("/api/sentry-example-api"); - if (!res.ok) { - throw new Error("Sentry Example Frontend Error"); + await Sentry.startSpan( + { + name: 'Example Frontend Span', + op: 'test', + }, + async () => { + const res = await fetch('/api/sentry-example-api'); + if (!res.ok) { + throw new Error('Sentry Example Frontend Error'); + } } - }); + ); }} > Throw error!

- Next, look for the error on the{" "} - Issues Page. + Next, look for the error on the{' '} + + Issues Page + + .

-

- For more information, see{" "} +

+ For more information, see{' '} https://docs.sentry.io/platforms/javascript/guides/nextjs/ diff --git a/app/src/app/store/mapEditSubs.ts b/app/src/app/store/mapEditSubs.ts index 1967b34a4..0599aee99 100644 --- a/app/src/app/store/mapEditSubs.ts +++ b/app/src/app/store/mapEditSubs.ts @@ -1,24 +1,12 @@ -import { debounce } from "lodash"; -import { - Assignment, - FormatAssignments, - getAssignments, -} from "../utils/api/apiHandlers"; -import { patchUpdates } from "../utils/api/mutations"; -import { useMapStore as _useMapStore, MapStore } from "./mapStore"; -import { shallowCompareArray } from "../utils/helpers"; -import { updateAssignments } from "../utils/api/queries"; +import {debounce} from 'lodash'; +import {Assignment, FormatAssignments, getAssignments} from '../utils/api/apiHandlers'; +import {patchUpdates} from '../utils/api/mutations'; +import {useMapStore as _useMapStore, MapStore} from './mapStore'; +import {shallowCompareArray} from '../utils/helpers'; +import {updateAssignments} from '../utils/api/queries'; -const zoneUpdates = ({ - getMapRef, - zoneAssignments, - appLoadingState, -}: Partial) => { - if ( - getMapRef?.() && - (zoneAssignments?.size) && - appLoadingState === "loaded" - ) { +const zoneUpdates = ({getMapRef, zoneAssignments, appLoadingState}: Partial) => { + if (getMapRef?.() && zoneAssignments?.size && appLoadingState === 'loaded') { const assignments = FormatAssignments(); patchUpdates.mutate(assignments); } @@ -30,24 +18,43 @@ type zoneSubState = [ MapStore['zoneAssignments'], MapStore['appLoadingState'], MapStore['mapRenderingState'] -] +]; export const getMapEditSubs = (useMapStore: typeof _useMapStore) => { const sendZonesOnMapRefSub = useMapStore.subscribe( - (state) => [state.getMapRef, state.zoneAssignments, state.appLoadingState, state.mapRenderingState], - ([getMapRef, zoneAssignments, appLoadingState, mapRenderingState], [ _prevMapRef, _prevZoneAssignments, prevAppLoadingState, prevMapRenderingState]) => { - const previousNotLoaded = [appLoadingState, mapRenderingState, prevAppLoadingState, prevMapRenderingState].some(state => state !== 'loaded') + state => [ + state.getMapRef, + state.zoneAssignments, + state.appLoadingState, + state.mapRenderingState, + ], + ( + [getMapRef, zoneAssignments, appLoadingState, mapRenderingState], + [_prevMapRef, _prevZoneAssignments, prevAppLoadingState, prevMapRenderingState] + ) => { + const previousNotLoaded = [ + appLoadingState, + mapRenderingState, + prevAppLoadingState, + prevMapRenderingState, + ].some(state => state !== 'loaded'); if (!getMapRef() || previousNotLoaded) { - return + return; } - console.log("!!!SENDING UPDATES", appLoadingState, mapRenderingState, prevAppLoadingState, prevMapRenderingState) - debouncedZoneUpdate({ getMapRef, zoneAssignments, appLoadingState }); + console.log( + '!!!SENDING UPDATES', + appLoadingState, + mapRenderingState, + prevAppLoadingState, + prevMapRenderingState + ); + debouncedZoneUpdate({getMapRef, zoneAssignments, appLoadingState}); }, - { equalityFn: shallowCompareArray} + {equalityFn: shallowCompareArray} ); const fetchAssignmentsSub = useMapStore.subscribe( - (state) => state.mapDocument, - (mapDocument) => mapDocument && updateAssignments(mapDocument) + state => state.mapDocument, + mapDocument => mapDocument && updateAssignments(mapDocument) ); return [sendZonesOnMapRefSub, fetchAssignmentsSub]; diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 75eb6897b..9b906c349 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -5,20 +5,20 @@ import { PARENT_LAYERS, CHILD_LAYERS, getLayerFilter, -} from "@constants/layers"; +} from '@constants/layers'; import { ColorZoneAssignmentsState, colorZoneAssignments, shallowCompareArray, -} from "../utils/helpers"; -import { useMapStore as _useMapStore, MapStore } from "@store/mapStore"; -import { getFeatureUnderCursor } from "@utils/helpers"; +} from '../utils/helpers'; +import {useMapStore as _useMapStore, MapStore} from '@store/mapStore'; +import {getFeatureUnderCursor} from '@utils/helpers'; export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { const addLayerSubMapDocument = useMapStore.subscribe< - [MapStore["mapDocument"], MapStore["getMapRef"]] + [MapStore['mapDocument'], MapStore['getMapRef']] >( - (state) => [state.mapDocument, state.getMapRef], + state => [state.mapDocument, state.getMapRef], ([mapDocument, getMapRef]) => { const mapStore = useMapStore.getState(); const mapRef = getMapRef(); @@ -27,23 +27,19 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { mapStore.addVisibleLayerIds([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]); } }, - { equalityFn: shallowCompareArray }, + {equalityFn: shallowCompareArray} ); const _shatterMapSideEffectRender = useMapStore.subscribe< - [ - MapStore["shatterIds"], - MapStore["getMapRef"], - MapStore["mapRenderingState"], - ] + [MapStore['shatterIds'], MapStore['getMapRef'], MapStore['mapRenderingState']] >( - (state) => [state.shatterIds, state.getMapRef, state.mapRenderingState], + state => [state.shatterIds, state.getMapRef, state.mapRenderingState], ([shatterIds, getMapRef, mapRenderingState]) => { const state = useMapStore.getState(); const mapRef = getMapRef(); const setMapLock = state.setMapLock; - if (!mapRef || mapRenderingState !== "loaded") { + if (!mapRef || mapRenderingState !== 'loaded') { return; } @@ -51,20 +47,20 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { if (state.mapDocument?.child_layer) layersToFilter.push(...CHILD_LAYERS); - layersToFilter.forEach((layerId) => - mapRef.setFilter(layerId, getLayerFilter(layerId, shatterIds)), + layersToFilter.forEach(layerId => + mapRef.setFilter(layerId, getLayerFilter(layerId, shatterIds)) ); - mapRef.once("render", () => { + mapRef.once('render', () => { setMapLock(false); console.log(`Unlocked at`, performance.now()); }); }, - { equalityFn: shallowCompareArray }, + {equalityFn: shallowCompareArray} ); const _hoverMapSideEffectRender = useMapStore.subscribe( - (state) => state.hoverFeatures, + state => state.hoverFeatures, (hoverFeatures, previousHoverFeatures) => { const mapRef = useMapStore.getState().getMapRef(); @@ -72,56 +68,55 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { return; } - previousHoverFeatures.forEach((feature) => { - mapRef.setFeatureState(feature, { hover: false }); + previousHoverFeatures.forEach(feature => { + mapRef.setFeatureState(feature, {hover: false}); }); - hoverFeatures.forEach((feature) => { - mapRef.setFeatureState(feature, { hover: true }); + hoverFeatures.forEach(feature => { + mapRef.setFeatureState(feature, {hover: true}); }); - }, + } ); - const _zoneAssignmentMapSideEffectRender = - useMapStore.subscribe( - (state) => [ - state.zoneAssignments, - state.mapDocument, - state.getMapRef, - state.shatterIds, - state.appLoadingState, - state.mapRenderingState, - ], - (curr, prev) => colorZoneAssignments(curr, prev), - { equalityFn: shallowCompareArray }, - ); + const _zoneAssignmentMapSideEffectRender = useMapStore.subscribe( + state => [ + state.zoneAssignments, + state.mapDocument, + state.getMapRef, + state.shatterIds, + state.appLoadingState, + state.mapRenderingState, + ], + (curr, prev) => colorZoneAssignments(curr, prev), + {equalityFn: shallowCompareArray} + ); - const _updateMapCursor = useMapStore.subscribe( - (state) => state.activeTool, - (activeTool) => { + const _updateMapCursor = useMapStore.subscribe( + state => state.activeTool, + activeTool => { const mapRef = useMapStore.getState().getMapRef(); if (!mapRef) return; let cursor; switch (activeTool) { - case "pan": - cursor = ""; + case 'pan': + cursor = ''; break; - case "brush": - cursor = "pointer"; + case 'brush': + cursor = 'pointer'; break; - case "eraser": - cursor = "pointer"; + case 'eraser': + cursor = 'pointer'; break; - case "shatter": - cursor = "crosshair"; + case 'shatter': + cursor = 'crosshair'; useMapStore.getState().setPaintFunction(getFeatureUnderCursor); break; default: - cursor = ""; + cursor = ''; } mapRef.getCanvas().style.cursor = cursor; - }, + } ); return [ diff --git a/app/src/app/store/metricsSubs.ts b/app/src/app/store/metricsSubs.ts index 9bf573106..25a8c7713 100644 --- a/app/src/app/store/metricsSubs.ts +++ b/app/src/app/store/metricsSubs.ts @@ -1,17 +1,14 @@ -import { updateMapMetrics } from "../utils/api/queries"; -import { useMapStore as _useMapStore } from "./mapStore"; +import {updateMapMetrics} from '../utils/api/queries'; +import {useMapStore as _useMapStore} from './mapStore'; export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { - const mapMetricsSub = useMapStore.subscribe( - (state) => state.mapDocument, - (mapDocument) => { + state => state.mapDocument, + mapDocument => { if (mapDocument) { updateMapMetrics(mapDocument); } } ); - return [ - mapMetricsSub - ] -} + return [mapMetricsSub]; +}; diff --git a/app/src/app/store/persistConfig.ts b/app/src/app/store/persistConfig.ts index c7f10baf7..163e3cac2 100644 --- a/app/src/app/store/persistConfig.ts +++ b/app/src/app/store/persistConfig.ts @@ -1,12 +1,10 @@ -import { PersistOptions } from "zustand/middleware" -import { MapStore } from "./mapStore" - +import {PersistOptions} from 'zustand/middleware'; +import {MapStore} from './mapStore'; export const persistOptions: PersistOptions> = { - name: 'districtr-persistrictr', + name: 'districtr-persistrictr', version: 0, - partialize: (state) => ({ - userMaps: state.userMaps + partialize: state => ({ + userMaps: state.userMaps, }), - -} +}; diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 27c899f88..377251532 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -1,19 +1,16 @@ -import axios from "axios"; -import "maplibre-gl"; -import { useMapStore } from "@/app/store/mapStore"; +import axios from 'axios'; +import 'maplibre-gl'; +import {useMapStore} from '@/app/store/mapStore'; export const FormatAssignments = () => { - const assignments = Array.from( - useMapStore.getState().zoneAssignments.entries() - ).map( + const assignments = Array.from(useMapStore.getState().zoneAssignments.entries()).map( // @ts-ignore ([geo_id, zone]: [string, number]): { document_id: string; geo_id: string; zone: number; } => ({ - document_id: - useMapStore.getState().mapDocument?.document_id.toString() ?? "", + document_id: useMapStore.getState().mapDocument?.document_id.toString() ?? '', geo_id, zone, }) @@ -76,14 +73,14 @@ export interface DocumentCreate { gerrydb_table: string; } -export const createMapDocument: ( +export const createMapDocument: (document: DocumentCreate) => Promise = async ( document: DocumentCreate -) => Promise = async (document: DocumentCreate) => { +) => { return await axios .post(`${process.env.NEXT_PUBLIC_API_URL}/api/create_document`, { gerrydb_table: document.gerrydb_table, }) - .then((res) => { + .then(res => { return res.data; }); }; @@ -93,33 +90,31 @@ export const createMapDocument: ( * @param document_id - string, the document id * @returns Promise */ -export const getDocument: ( +export const getDocument: (document_id: string) => Promise = async ( document_id: string -) => Promise = async (document_id: string) => { +) => { if (document_id) { return await axios .get(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${document_id}`) - .then((res) => { + .then(res => { return res.data; }); } else { - throw new Error("No document id found"); + throw new Error('No document id found'); } }; export const getAssignments: ( mapDocument: DocumentObject -) => Promise = async (mapDocument) => { +) => Promise = async mapDocument => { if (mapDocument) { return await axios - .get( - `${process.env.NEXT_PUBLIC_API_URL}/api/get_assignments/${mapDocument.document_id}` - ) - .then((res) => { + .get(`${process.env.NEXT_PUBLIC_API_URL}/api/get_assignments/${mapDocument.document_id}`) + .then(res => { return res.data; }); } else { - throw new Error("No document provided"); + throw new Error('No document provided'); } }; @@ -142,17 +137,15 @@ export interface ZonePopulation { */ export const getZonePopulations: ( mapDocument: DocumentObject -) => Promise = async (mapDocument) => { +) => Promise = async mapDocument => { if (mapDocument) { return await axios - .get( - `${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/total_pop` - ) - .then((res) => { + .get(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/total_pop`) + .then(res => { return res.data; }); } else { - throw new Error("No document provided"); + throw new Error('No document provided'); } }; @@ -167,10 +160,8 @@ export const getAvailableDistrictrMaps: ( offset?: number ) => Promise = async (limit = 10, offset = 0) => { return await axios - .get( - `${process.env.NEXT_PUBLIC_API_URL}/api/gerrydb/views?limit=${limit}&offset=${offset}` - ) - .then((res) => { + .get(`${process.env.NEXT_PUBLIC_API_URL}/api/gerrydb/views?limit=${limit}&offset=${offset}`) + .then(res => { return res.data; }); }; @@ -210,7 +201,7 @@ export const patchUpdateAssignments: ( .patch(`${process.env.NEXT_PUBLIC_API_URL}/api/update_assignments`, { assignments: assignments, }) - .then((res) => { + .then(res => { return res.data; }); }; @@ -222,7 +213,7 @@ export const patchUpdateAssignments: ( * @property {Assignment[]} children - The children. */ export interface ShatterResult { - parents: { geoids: string[] }; + parents: {geoids: string[]}; children: Assignment[]; } @@ -236,7 +227,7 @@ export interface ShatterResult { export const patchShatterParents: (params: { document_id: string; geoids: string[]; -}) => Promise = async ({ document_id, geoids }) => { +}) => Promise = async ({document_id, geoids}) => { return await axios .patch( `${process.env.NEXT_PUBLIC_API_URL}/api/update_assignments/${document_id}/shatter_parents`, @@ -244,7 +235,7 @@ export const patchShatterParents: (params: { geoids: geoids, } ) - .then((res) => { + .then(res => { return res.data; }); }; diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts index be1e7df90..6bdc1d580 100644 --- a/app/src/app/utils/api/mutations.ts +++ b/app/src/app/utils/api/mutations.ts @@ -1,18 +1,18 @@ -import { MutationObserver } from "@tanstack/query-core"; -import { queryClient } from "./queryClient"; +import {MutationObserver} from '@tanstack/query-core'; +import {queryClient} from './queryClient'; import { AssignmentsCreate, createMapDocument, patchShatterParents, patchUpdateAssignments, -} from "@/app/utils/api/apiHandlers"; -import { useMapStore } from "@/app/store/mapStore"; -import { mapMetrics } from "./queries"; -import { use } from "react"; +} from '@/app/utils/api/apiHandlers'; +import {useMapStore} from '@/app/store/mapStore'; +import {mapMetrics} from './queries'; +import {use} from 'react'; export const patchShatter = new MutationObserver(queryClient, { mutationFn: patchShatterParents, - onMutate: ({ document_id, geoids }) => { + onMutate: ({document_id, geoids}) => { useMapStore.getState().setMapLock(true); console.log( `Shattering parents for ${geoids} in document ${document_id}...`, @@ -20,13 +20,11 @@ export const patchShatter = new MutationObserver(queryClient, { performance.now() ); }, - onError: (error) => { - console.log("Error updating assignments: ", error); + onError: error => { + console.log('Error updating assignments: ', error); }, - onSuccess: (data) => { - console.log( - `Successfully shattered parents into ${data.children.length} children` - ); + onSuccess: data => { + console.log(`Successfully shattered parents into ${data.children.length} children`); return data; }, }); @@ -34,15 +32,13 @@ export const patchShatter = new MutationObserver(queryClient, { export const patchUpdates = new MutationObserver(queryClient, { mutationFn: patchUpdateAssignments, onMutate: () => { - console.log("Updating assignments"); + console.log('Updating assignments'); }, - onError: (error) => { - console.log("Error updating assignments: ", error); + onError: error => { + console.log('Error updating assignments: ', error); }, onSuccess: (data: AssignmentsCreate) => { - console.log( - `Successfully upserted ${data.assignments_upserted} assignments` - ); + console.log(`Successfully upserted ${data.assignments_upserted} assignments`); mapMetrics.refetch(); }, }); @@ -50,18 +46,18 @@ export const patchUpdates = new MutationObserver(queryClient, { export const document = new MutationObserver(queryClient, { mutationFn: createMapDocument, onMutate: () => { - console.log("Creating document"); - useMapStore.getState().setAppLoadingState("loading"); + console.log('Creating document'); + useMapStore.getState().setAppLoadingState('loading'); useMapStore.getState().resetZoneAssignments(); }, - onError: (error) => { - console.error("Error creating map document: ", error); + onError: error => { + console.error('Error creating map document: ', error); }, - onSuccess: (data) => { + onSuccess: data => { useMapStore.getState().setMapDocument(data); - useMapStore.getState().setAppLoadingState("loaded"); + useMapStore.getState().setAppLoadingState('loaded'); const documentUrl = new URL(window.location.toString()); - documentUrl.searchParams.set("document_id", data.document_id); - history.pushState({}, "", documentUrl.toString()); + documentUrl.searchParams.set('document_id', data.document_id); + history.pushState({}, '', documentUrl.toString()); }, }); diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index b4f3cd552..e50f7996c 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -1,5 +1,5 @@ -import { QueryObserver, skipToken } from "@tanstack/react-query"; -import { queryClient } from "./queryClient"; +import {QueryObserver, skipToken} from '@tanstack/react-query'; +import {queryClient} from './queryClient'; import { DistrictrMap, getAvailableDistrictrMaps, @@ -9,48 +9,47 @@ import { getDocument, getZonePopulations, ZonePopulation, -} from "./apiHandlers"; -import { MapStore, useMapStore } from "@/app/store/mapStore"; +} from './apiHandlers'; +import {MapStore, useMapStore} from '@/app/store/mapStore'; -const INITIAL_VIEW_LIMIT = 30 -const INITIAL_VIEW_OFFSET = 0 +const INITIAL_VIEW_LIMIT = 30; +const INITIAL_VIEW_OFFSET = 0; export const mapMetrics = new QueryObserver(queryClient, { - queryKey: ["_zonePopulations"], + queryKey: ['_zonePopulations'], queryFn: skipToken, }); export const updateMapMetrics = (mapDocument: DocumentObject) => { mapMetrics.setOptions({ - queryKey: ["zonePopulations", mapDocument.document_id], + queryKey: ['zonePopulations', mapDocument.document_id], queryFn: mapDocument ? () => getZonePopulations(mapDocument) : skipToken, }); }; - -mapMetrics.subscribe((result) => { +mapMetrics.subscribe(result => { useMapStore.getState().setMapMetrics(result); }); export const mapViewsQuery = new QueryObserver(queryClient, { - queryKey: ["views", INITIAL_VIEW_LIMIT, INITIAL_VIEW_OFFSET], + queryKey: ['views', INITIAL_VIEW_LIMIT, INITIAL_VIEW_OFFSET], queryFn: () => getAvailableDistrictrMaps(INITIAL_VIEW_LIMIT, INITIAL_VIEW_OFFSET), }); export const updateMapViews = (limit: number, offset: number) => { mapViewsQuery.setOptions({ - queryKey: ["views", limit, offset], + queryKey: ['views', limit, offset], queryFn: () => getAvailableDistrictrMaps(limit, offset), }); }; export const getMapViewsSubs = (_useMapStore: typeof useMapStore) => { - mapViewsQuery.subscribe((result) => { + mapViewsQuery.subscribe(result => { if (result) { - _useMapStore.getState().setMapViews(result) + _useMapStore.getState().setMapViews(result); } - }) -} + }); +}; const getDocumentFunction = (documentId?: string) => { return async () => { @@ -61,53 +60,46 @@ const getDocumentFunction = (documentId?: string) => { } else { return null; } -} -} + }; +}; -export const updateDocumentFromId = new QueryObserver( - queryClient, - { - queryKey: ["mapDocument", undefined], - queryFn: getDocumentFunction() - }, -); +export const updateDocumentFromId = new QueryObserver(queryClient, { + queryKey: ['mapDocument', undefined], + queryFn: getDocumentFunction(), +}); -export const updateGetDocumentFromId = (documentId:string) => { +export const updateGetDocumentFromId = (documentId: string) => { updateDocumentFromId.setOptions({ - queryKey: ["mapDocument", documentId], - queryFn: getDocumentFunction(documentId) + queryKey: ['mapDocument', documentId], + queryFn: getDocumentFunction(documentId), }); -} +}; -updateDocumentFromId.subscribe((mapDocument) => { +updateDocumentFromId.subscribe(mapDocument => { if (mapDocument.data) { useMapStore.getState().setMapDocument(mapDocument.data); } }); const getFetchAssignmentsQuery = (mapDocument?: MapStore['mapDocument']) => { - if (!mapDocument) return () => null - return async () => await getAssignments(mapDocument) -} + if (!mapDocument) return () => null; + return async () => await getAssignments(mapDocument); +}; -export const fetchAssignments = new QueryObserver( - queryClient, - { - queryKey: ["assignments"], - queryFn: getFetchAssignmentsQuery(), - } -) +export const fetchAssignments = new QueryObserver(queryClient, { + queryKey: ['assignments'], + queryFn: getFetchAssignmentsQuery(), +}); export const updateAssignments = (mapDocument: DocumentObject) => { fetchAssignments.setOptions({ queryFn: getFetchAssignmentsQuery(mapDocument), - queryKey: ['assignments', performance.now()] - }) -} - + queryKey: ['assignments', performance.now()], + }); +}; -fetchAssignments.subscribe((assignments) => { +fetchAssignments.subscribe(assignments => { if (assignments.data) { useMapStore.getState().loadZoneAssignments(assignments.data); } -}); \ No newline at end of file +}); diff --git a/app/src/app/utils/api/queryClient.ts b/app/src/app/utils/api/queryClient.ts index 7b3125501..34adc3d71 100644 --- a/app/src/app/utils/api/queryClient.ts +++ b/app/src/app/utils/api/queryClient.ts @@ -1,2 +1,2 @@ -import { QueryClient } from "@tanstack/react-query"; -export const queryClient = new QueryClient(); \ No newline at end of file +import {QueryClient} from '@tanstack/react-query'; +export const queryClient = new QueryClient(); diff --git a/app/src/app/utils/api/queryParamsListener.ts b/app/src/app/utils/api/queryParamsListener.ts index 955ed7521..37fdb7364 100644 --- a/app/src/app/utils/api/queryParamsListener.ts +++ b/app/src/app/utils/api/queryParamsListener.ts @@ -1,10 +1,10 @@ -import { updateDocumentFromId, updateGetDocumentFromId } from "./queries"; -export let previousDocumentID = '' +import {updateDocumentFromId, updateGetDocumentFromId} from './queries'; +export let previousDocumentID = ''; export const getSearchParamsObersver = () => { // next ssr safety - if (typeof window === "undefined") { - return + if (typeof window === 'undefined') { + return; } // listener for tab refocus @@ -16,17 +16,15 @@ export const getSearchParamsObersver = () => { document.addEventListener('visibilitychange', handleVisibilityChange); - let previousDocumentID = ""; + let previousDocumentID = ''; const observer = new MutationObserver(() => { - const documentId = new URLSearchParams(window.location.search).get( - "document_id" - ); + const documentId = new URLSearchParams(window.location.search).get('document_id'); if (documentId && documentId !== previousDocumentID) { - previousDocumentID = documentId - updateGetDocumentFromId(documentId) + previousDocumentID = documentId; + updateGetDocumentFromId(documentId); } }); - const config = { subtree: true, childList: true }; + const config = {subtree: true, childList: true}; // start listening to changes observer.observe(document, config); return observer; diff --git a/app/src/app/utils/arrays.ts b/app/src/app/utils/arrays.ts index e6990774a..bd44fcba2 100644 --- a/app/src/app/utils/arrays.ts +++ b/app/src/app/utils/arrays.ts @@ -1,3 +1,3 @@ export const onlyUnique = (value: unknown, index: number, self: unknown[]) => { - return self.indexOf(value) === index; + return self.indexOf(value) === index; }; diff --git a/app/src/app/utils/events/handlers.ts b/app/src/app/utils/events/handlers.ts index 18c29e4c1..157469ba3 100644 --- a/app/src/app/utils/events/handlers.ts +++ b/app/src/app/utils/events/handlers.ts @@ -1,9 +1,9 @@ -import { BLOCK_SOURCE_ID } from "@/app/constants/layers"; -import { MutableRefObject } from "react"; -import { Map, MapGeoJSONFeature } from "maplibre-gl"; -import { debounce } from "lodash"; -import { NullableZone, Zone } from "@/app/constants/types"; -import { MapStore } from "@/app/store/mapStore"; +import {BLOCK_SOURCE_ID} from '@/app/constants/layers'; +import {MutableRefObject} from 'react'; +import {Map, MapGeoJSONFeature} from 'maplibre-gl'; +import {debounce} from 'lodash'; +import {NullableZone, Zone} from '@/app/constants/types'; +import {MapStore} from '@/app/store/mapStore'; /** * Debounced function to set zone assignments in the store without resetting the state every time the mouse moves (assuming onhover event). @@ -19,11 +19,11 @@ const debouncedSetZoneAssignments = debounce( const population = Array.from(Object.values(accumulatedBlockPopulations)).reduce( (acc, val) => acc + Number(val), - 0, + 0 ); selectedZone && mapStoreRef.setZonePopulations(selectedZone, population); }, - 1, // 1ms debounce + 1 // 1ms debounce ); /** @@ -40,32 +40,30 @@ const debouncedSetZoneAssignments = debounce( export const SelectMapFeatures = ( features: Array | undefined, map: Map | null, - mapStoreRef: MapStore, + mapStoreRef: MapStore ) => { if (map) { - let { accumulatedGeoids, accumulatedBlockPopulations, activeTool } = - mapStoreRef; - const selectedZone = - activeTool === "eraser" ? null : mapStoreRef.selectedZone; + let {accumulatedGeoids, accumulatedBlockPopulations, activeTool} = mapStoreRef; + const selectedZone = activeTool === 'eraser' ? null : mapStoreRef.selectedZone; - features?.forEach((feature) => { + features?.forEach(feature => { map.setFeatureState( { source: BLOCK_SOURCE_ID, id: feature?.id ?? undefined, sourceLayer: feature.sourceLayer, }, - { selected: true, zone: selectedZone } + {selected: true, zone: selectedZone} ); }); if (features?.length) { - features.forEach((feature) => { + features.forEach(feature => { accumulatedGeoids.add(feature.properties?.path); - accumulatedBlockPopulations.set(feature.properties?.path, feature.properties?.total_pop) + accumulatedBlockPopulations.set(feature.properties?.path, feature.properties?.total_pop); }); } } - return new Promise((resolve) => { + return new Promise(resolve => { // Resolve the Promise after the function completes // this is so we can chain the function and call the next one resolve(); @@ -84,8 +82,8 @@ export const SelectZoneAssignmentFeatures = (mapStoreRef: MapStore) => { if (accumulatedGeoids?.size) { debouncedSetZoneAssignments( mapStoreRef, - mapStoreRef.activeTool === "brush" ? mapStoreRef.selectedZone : null, - mapStoreRef.accumulatedGeoids, + mapStoreRef.activeTool === 'brush' ? mapStoreRef.selectedZone : null, + mapStoreRef.accumulatedGeoids ); } }; @@ -99,7 +97,7 @@ export const SelectZoneAssignmentFeatures = (mapStoreRef: MapStore) => { export const ResetMapSelectState = ( map: Map | null, mapStoreRef: MapStore, - sourceLayer: string, + sourceLayer: string ) => { if (map && Object.keys(mapStoreRef.zoneAssignments).length) { map.removeFeatureState({ @@ -107,7 +105,7 @@ export const ResetMapSelectState = ( sourceLayer: sourceLayer, }); - mapStoreRef.setAccumulatedGeoids(new Set()) + mapStoreRef.setAccumulatedGeoids(new Set()); // reset zoneAssignments mapStoreRef.resetZoneAssignments(); // confirm the map has been reset diff --git a/app/src/app/utils/helpers.ts b/app/src/app/utils/helpers.ts index 18db33049..8953242a4 100644 --- a/app/src/app/utils/helpers.ts +++ b/app/src/app/utils/helpers.ts @@ -6,13 +6,13 @@ import { MapGeoJSONFeature, LngLat, LngLatLike, -} from "maplibre-gl"; -import { Point } from "maplibre-gl"; -import { BLOCK_HOVER_LAYER_ID, BLOCK_SOURCE_ID } from "@/app/constants/layers"; -import { polygon, multiPolygon } from "@turf/helpers"; -import { booleanWithin } from "@turf/boolean-within"; -import { pointOnFeature } from "@turf/point-on-feature"; -import { MapStore, useMapStore } from "../store/mapStore"; +} from 'maplibre-gl'; +import {Point} from 'maplibre-gl'; +import {BLOCK_HOVER_LAYER_ID, BLOCK_SOURCE_ID} from '@/app/constants/layers'; +import {polygon, multiPolygon} from '@turf/helpers'; +import {booleanWithin} from '@turf/boolean-within'; +import {pointOnFeature} from '@turf/point-on-feature'; +import {MapStore, useMapStore} from '../store/mapStore'; /** * PaintEventHandler @@ -25,7 +25,7 @@ export type PaintEventHandler = ( map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, - layers?: string[], + layers?: string[] ) => MapGeoJSONFeature[] | undefined; /** @@ -54,7 +54,7 @@ export type ContextMenuState = { */ export const boxAroundPoint = ( e: MapLayerMouseEvent | MapLayerTouchEvent, - radius: number, + radius: number ): [PointLike, PointLike] => { return [ [e.point.x - radius, e.point.y - radius], @@ -74,11 +74,11 @@ export const getFeaturesInBbox = ( map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, - layers: string[] = [BLOCK_HOVER_LAYER_ID], + layers: string[] = [BLOCK_HOVER_LAYER_ID] ): MapGeoJSONFeature[] | undefined => { const bbox = boxAroundPoint(e, brushSize); - return map?.queryRenderedFeatures(bbox, { layers }); + return map?.queryRenderedFeatures(bbox, {layers}); }; /** @@ -93,9 +93,9 @@ export const getFeatureUnderCursor = ( map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, - layers: string[] = [BLOCK_HOVER_LAYER_ID], + layers: string[] = [BLOCK_HOVER_LAYER_ID] ): MapGeoJSONFeature[] | undefined => { - return map?.queryRenderedFeatures(e.point, { layers }); + return map?.queryRenderedFeatures(e.point, {layers}); }; /** @@ -110,12 +110,12 @@ export const getFeaturesIntersectingCounties = ( map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent, brushSize: number, - layers: string[] = [BLOCK_HOVER_LAYER_ID], + layers: string[] = [BLOCK_HOVER_LAYER_ID] ): MapGeoJSONFeature[] | undefined => { if (!map) return; const countyFeatures = map.queryRenderedFeatures(e.point, { - layers: ["counties_fill"], + layers: ['counties_fill'], }); if (!countyFeatures) return; @@ -139,7 +139,7 @@ export const getFeaturesIntersectingCounties = ( countyPoly = multiPolygon(countyFeatures[0].geometry.coordinates); } - return features.filter((p) => { + return features.filter(p => { const point = pointOnFeature(p); return booleanWithin(point, countyPoly); }); @@ -152,7 +152,7 @@ export const getFeaturesIntersectingCounties = ( * @returns [PointLike, PointLike] - An array containing the SW and NE corners of the bounding box */ const getBoundingBoxFromFeatures = ( - features: MapGeoJSONFeature[], + features: MapGeoJSONFeature[] ): [LngLatLike, LngLatLike] | null => { if (!features || features.length === 0) { return null; @@ -161,7 +161,7 @@ const getBoundingBoxFromFeatures = ( const sw = new LngLat(180, 90); const ne = new LngLat(-180, -90); - features.forEach((feature) => { + features.forEach(feature => { // this will always have an even number of coordinates // iterating over the coordinates in pairs yields (lng, lat) // @ts-ignore: Property 'coordinates' does not exist on type 'Geometry'. @@ -186,22 +186,19 @@ const getBoundingBoxFromFeatures = ( * @param e - MapLayerMouseEvent | MapLayerTouchEvent, the event object * @returns Point - The position of the mouse on the map */ -export const mousePos = ( - map: Map | null, - e: MapLayerMouseEvent | MapLayerTouchEvent, -) => { +export const mousePos = (map: Map | null, e: MapLayerMouseEvent | MapLayerTouchEvent) => { const canvas = map?.getCanvasContainer(); if (!canvas) return new Point(0, 0); const rect = canvas.getBoundingClientRect(); return new Point( e.point.x - rect.left - canvas.clientLeft, - e.point.y - rect.top - canvas.clientTop, + e.point.y - rect.top - canvas.clientTop ); }; export interface LayerVisibility { layerId: string; - visibility: "none" | "visible"; + visibility: 'none' | 'visible'; } /** @@ -218,18 +215,18 @@ export interface LayerVisibility { */ export function toggleLayerVisibility( mapRef: maplibregl.Map, - layerIds: string[], + layerIds: string[] ): LayerVisibility[] { - const activeLayerIds = getVisibleLayers(mapRef)?.map((layer) => layer.id); + const activeLayerIds = getVisibleLayers(mapRef)?.map(layer => layer.id); if (!activeLayerIds) return []; - return layerIds.map((layerId) => { + return layerIds.map(layerId => { if (activeLayerIds && activeLayerIds.includes(layerId)) { - mapRef.setLayoutProperty(layerId, "visibility", "none"); - return { layerId: layerId, visibility: "none" }; + mapRef.setLayoutProperty(layerId, 'visibility', 'none'); + return {layerId: layerId, visibility: 'none'}; } else { - mapRef.setLayoutProperty(layerId, "visibility", "visible"); - return { layerId: layerId, visibility: "visible" }; + mapRef.setLayoutProperty(layerId, 'visibility', 'visible'); + return {layerId: layerId, visibility: 'visible'}; } }, {}); } @@ -241,27 +238,23 @@ export function toggleLayerVisibility( * @param {maplibregl.Map} map - The map reference. */ export function getVisibleLayers(map: Map | null) { - return map?.getStyle().layers.filter((layer) => { - return layer.layout?.visibility === "visible"; + return map?.getStyle().layers.filter(layer => { + return layer.layout?.visibility === 'visible'; }); } export type ColorZoneAssignmentsState = [ - MapStore["zoneAssignments"], - MapStore["mapDocument"], - MapStore["getMapRef"], - MapStore["shatterIds"], - MapStore["appLoadingState"], - MapStore["mapRenderingState"], + MapStore['zoneAssignments'], + MapStore['mapDocument'], + MapStore['getMapRef'], + MapStore['shatterIds'], + MapStore['appLoadingState'], + MapStore['mapRenderingState'] ]; -export const getMap = (_getMapRef?: MapStore["getMapRef"]) => { +export const getMap = (_getMapRef?: MapStore['getMapRef']) => { const mapRef = _getMapRef?.() || useMapStore.getState().getMapRef(); - if ( - mapRef - ?.getStyle() - .layers.findIndex((layer) => layer.id === BLOCK_HOVER_LAYER_ID) !== -1 - ) { + if (mapRef?.getStyle().layers.findIndex(layer => layer.id === BLOCK_HOVER_LAYER_ID) !== -1) { return null; } @@ -289,44 +282,27 @@ export const getMap = (_getMapRef?: MapStore["getMapRef"]) => { */ export const colorZoneAssignments = ( state: ColorZoneAssignmentsState, - previousState?: ColorZoneAssignmentsState, + previousState?: ColorZoneAssignmentsState ) => { - const [ - zoneAssignments, - mapDocument, - getMapRef, - _, - appLoadingState, - mapRenderingState, - ] = state; + const [zoneAssignments, mapDocument, getMapRef, _, appLoadingState, mapRenderingState] = state; const previousZoneAssignments = previousState?.[0] || null; const mapRef = getMapRef(); const shatterIds = useMapStore.getState().shatterIds; - if ( - !mapRef || - !mapDocument || - appLoadingState !== "loaded" || - mapRenderingState !== "loaded" - ) { + if (!mapRef || !mapDocument || appLoadingState !== 'loaded' || mapRenderingState !== 'loaded') { return; } - const isInitialRender = - previousState?.[4] !== "loaded" || previousState?.[5] !== "loaded"; + const isInitialRender = previousState?.[4] !== 'loaded' || previousState?.[5] !== 'loaded'; zoneAssignments.forEach((zone, id) => { if ( - (id && - !isInitialRender && - previousZoneAssignments?.get(id) === zoneAssignments.get(id)) || + (id && !isInitialRender && previousZoneAssignments?.get(id) === zoneAssignments.get(id)) || !id ) { return; } const isChild = shatterIds.children.has(id); - const sourceLayer = isChild - ? mapDocument.child_layer - : mapDocument.parent_layer; + const sourceLayer = isChild ? mapDocument.child_layer : mapDocument.parent_layer; if (!sourceLayer) { return; @@ -341,17 +317,17 @@ export const colorZoneAssignments = ( { selected: true, zone, - }, + } ); }); }; // property changes on which to re-color assignments export const colorZoneAssignmentTriggers = [ - "zoneAssignments", - "mapDocument", - "mapRef", - "shatterIds", + 'zoneAssignments', + 'mapDocument', + 'mapRef', + 'shatterIds', ] as Array; /** @@ -368,13 +344,13 @@ export const colorZoneAssignmentTriggers = [ * This is typically used when "shattering" a parent element into its constituent parts. */ export const setZones = ( - zoneAssignments: MapStore["zoneAssignments"], + zoneAssignments: MapStore['zoneAssignments'], parent: string, - children: Set, + children: Set ) => { const zone = zoneAssignments.get(parent); if (zone) { - children.forEach((childId) => { + children.forEach(childId => { zoneAssignments.set(childId, zone); }); zoneAssignments.delete(parent); From 387ab7eee241e3183c29d46f9d3a6bca7fd59fd6 Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 31 Oct 2024 13:38:13 -0500 Subject: [PATCH 23/50] Show / hide child layers on toggle (#159) --- app/src/app/components/sidebar/Layers.tsx | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index 86b761710..f96b4ed4c 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -1,6 +1,12 @@ import {Heading, CheckboxGroup, Flex} from '@radix-ui/themes'; import {useMapStore} from '@/app/store/mapStore'; -import {COUNTY_LAYER_IDS, BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID} from '../../constants/layers'; +import { + COUNTY_LAYER_IDS, + BLOCK_LAYER_ID, + BLOCK_HOVER_LAYER_ID, + BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_LAYER_ID_CHILD, +} from '../../constants/layers'; import {toggleLayerVisibility} from '../../utils/helpers'; /** Layers @@ -35,7 +41,14 @@ export default function Layers() { > toggleLayers([BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID])} + onClick={() => + toggleLayers([ + BLOCK_LAYER_ID, + BLOCK_HOVER_LAYER_ID, + BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_LAYER_ID_CHILD, + ]) + } disabled={mapDocument === null} > Show painted districts From bba7b0642cf3c8285223fdc227cf1e92dc960994 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sun, 3 Nov 2024 13:13:57 -0500 Subject: [PATCH 24/50] Fix backend tests (#161) --- .github/workflows/fly-deploy-pr.yml | 3 +- .../versions/ccb2a6b81a8b_shattering.py | 35 ++++++- .../dc391733e10a_add_extent_to_gerrydb.py | 92 +++++++----------- backend/app/models.py | 4 +- backend/app/sql/create_districtr_map_udf.sql | 39 +++++--- backend/app/utils.py | 97 ++++++++++++++++++- backend/cli.py | 59 ++++++++++- backend/tests/conftest.py | 18 +++- backend/tests/test_utils.py | 33 ++++++- 9 files changed, 300 insertions(+), 80 deletions(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index b32a78fe0..64379460a 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -60,7 +60,8 @@ jobs: --name ${{ github.event.repository.name }}-${{ github.event.number }}-db \ --region ewr \ --initial-cluster-size 1 \ - --vm-size shared-cpu-2x \ + --vm-size shared-cpu-1x \ + --vm-memory 512 \ -p ${{ secrets.FLY_PR_PG_PASSWORD }} \ --org mggg \ --fork-from districtr-v2-db diff --git a/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py b/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py index 1866a3875..15e06713b 100644 --- a/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py +++ b/backend/app/alembic/versions/ccb2a6b81a8b_shattering.py @@ -91,13 +91,46 @@ def upgrade() -> None: for file_name in [ "parent_child_relationships.sql", "create_shatterable_gerrydb_view.sql", - "create_districtr_map_udf.sql", "shatter_parent.sql", ]: with open(SQL_PATH / file_name, "r") as f: sql = f.read() op.execute(sql) + op.execute( + sa.text( + """ + CREATE OR REPLACE FUNCTION create_districtr_map( + map_name VARCHAR, + gerrydb_table_name VARCHAR, + num_districts INTEGER, + tiles_s3_path VARCHAR, + parent_layer_name VARCHAR, + child_layer_name VARCHAR + ) + RETURNS UUID AS $$ + DECLARE + inserted_districtr_uuid UUID; + BEGIN + INSERT INTO districtrmap ( + created_at, + uuid, + name, + gerrydb_table_name, + num_districts, + tiles_s3_path, + parent_layer, + child_layer + ) + VALUES (now(), gen_random_uuid(), $1, $2, $3, $4, $5, $6) + RETURNING uuid INTO inserted_districtr_uuid; + RETURN inserted_districtr_uuid; + END; + $$ LANGUAGE plpgsql; + """ + ) + ) + def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### diff --git a/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py b/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py index e7c06d8d0..61fa9729e 100644 --- a/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py +++ b/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py @@ -10,7 +10,6 @@ from alembic import op import sqlalchemy as sa -from app.models import DistrictrMap # revision identifiers, used by Alembic. revision: str = "dc391733e10a" @@ -21,67 +20,48 @@ def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - # Make column nullable to avoid errors during initial insert op.add_column( - "districtrmap", sa.Column("extent", sa.ARRAY(sa.Float()), nullable=True) + "districtrmap", + sa.Column("extent", sa.ARRAY(sa.Float()), nullable=True, default=None), ) - # Use a session to interact with the database - bind = op.get_bind() - Session = sa.orm.sessionmaker(bind=bind) - session = Session() - - # retreive table names from the districtrmap parent_layer field - table_names = session.execute(sa.select(DistrictrMap.parent_layer)).scalars().all() - - if not table_names: - raise ValueError("No matching table names found in the index table") - - cases = [] - for table_name in table_names: - case = f""" - WHEN districtrmap."parent_layer" = '{table_name}' THEN ( - SELECT - ARRAY[ - ST_XMin(ST_Extent(ST_Transform(geometry, 4326))), - ST_YMin(ST_Extent(ST_Transform(geometry, 4326))), - ST_XMax(ST_Extent(ST_Transform(geometry, 4326))), - ST_YMax(ST_Extent(ST_Transform(geometry, 4326))) - ] - FROM gerrydb."{table_name}", public.districtrmap - - ) + op.execute( + sa.text( + """ + DO $$ + DECLARE + rec RECORD; + layer_extent GEOMETRY; + BEGIN + FOR rec IN + SELECT name, parent_layer + FROM districtrmap + LOOP + BEGIN + EXECUTE format(' + SELECT ST_Extent(ST_Transform(geometry, 4326)) + FROM gerrydb.%I', + rec.parent_layer + ) INTO layer_extent; + + UPDATE districtrmap + SET extent = ARRAY[ + ST_XMin(layer_extent), + ST_YMin(layer_extent), + ST_XMax(layer_extent), + ST_YMax(layer_extent) + ] + WHERE uuid = rec.uuid; + + EXCEPTION WHEN undefined_table THEN + RAISE NOTICE 'Table % does not exist for layer %', rec.parent_layer, rec.name; + END; + END LOOP; + END $$; """ - - cases.append(case) - # Combine all cases into a single SQL statement - case_statement = " ".join(cases) - # Execute a single UPDATE statement - update_query = sa.text( - f""" - UPDATE districtrmap - SET extent = CASE - {case_statement} - ELSE ARRAY[-102.0517,36.99301,-94.5883,40.0031] - -- if this fails, there is no - -- matching table from parent_layer - -- and that's a problem - END; - """ - ) - - bind.execute(update_query) - - # Make the `extent` column non-nullable - op.alter_column( - "districtrmap", "extent", existing_type=sa.ARRAY(sa.Float()), nullable=False + ) ) - session.commit() - def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### op.drop_column("districtrmap", "extent") - # ### end Alembic commands ### diff --git a/backend/app/models.py b/backend/app/models.py index 13103718d..a178d2ec5 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -64,7 +64,7 @@ class DistrictrMap(TimeStampMixin, SQLModel, table=True): String, ForeignKey("gerrydbtable.name"), default=None, nullable=True ) ) - extent: list[float] = Field(sa_column=Column(ARRAY(Float), nullable=False)) + extent: list[float] = Field(sa_column=Column(ARRAY(Float), nullable=True)) # schema? will need to contrain the schema # where does this go? # when you create the view, pull the columns that you need @@ -129,7 +129,7 @@ class DocumentPublic(BaseModel): num_districts: int | None = None created_at: datetime updated_at: datetime - extent: list[float] + extent: list[float] | None = None class AssignmentsBase(SQLModel): diff --git a/backend/app/sql/create_districtr_map_udf.sql b/backend/app/sql/create_districtr_map_udf.sql index 4fa89aca1..5c4d1cb49 100644 --- a/backend/app/sql/create_districtr_map_udf.sql +++ b/backend/app/sql/create_districtr_map_udf.sql @@ -9,7 +9,14 @@ CREATE OR REPLACE FUNCTION create_districtr_map( RETURNS UUID AS $$ DECLARE inserted_districtr_uuid UUID; + extent GEOMETRY; BEGIN + EXECUTE format(' + SELECT ST_Extent(ST_Transform(geometry, 4326)) + FROM gerrydb.%I', + parent_layer_name + ) INTO extent; + INSERT INTO districtrmap ( created_at, uuid, @@ -19,20 +26,26 @@ BEGIN tiles_s3_path, parent_layer, child_layer, - -- calculate extent based on parent_layer extent - select( - ARRAY[ - ST_XMin(ST_Extent(ST_Transform(geometry, 4326))), - ST_YMin(ST_Extent(ST_Transform(geometry, 4326))), - ST_XMax(ST_Extent(ST_Transform(geometry, 4326))), - ST_YMax(ST_Extent(ST_Transform(geometry, 4326))) - ] - FROM gerrydb.$5, public.districtrmap - WHERE districtrmap.parent_layer = $5 - ) + extent + ) + VALUES ( + now(), + gen_random_uuid(), + map_name, + gerrydb_table_name, + num_districts, + tiles_s3_path, + parent_layer_name, + child_layer_name, + ARRAY[ + ST_XMin(extent), + ST_YMin(extent), + ST_XMax(extent), + ST_YMax(extent) + ] ) - VALUES (now(), gen_random_uuid(), $1, $2, $3, $4, $5, $6) RETURNING uuid INTO inserted_districtr_uuid; + RETURN inserted_districtr_uuid; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; diff --git a/backend/app/utils.py b/backend/app/utils.py index 06e007ce2..007e3a3e2 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -1,10 +1,15 @@ from sqlalchemy import text from sqlalchemy import bindparam, Integer, String, Text -from sqlmodel import Session +from sqlalchemy.types import UUID +from sqlmodel import Session, Float +import logging from app.models import UUIDType +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + def create_districtr_map( session: Session, @@ -107,3 +112,93 @@ def create_parent_child_edges( "districtr_map_uuid": districtr_map_uuid, }, ) + + +def add_extent_to_districtrmap( + session: Session, districtr_map_uuid: str | UUID, bounds: list[float] | None = None +) -> None: + """ + Add the extent to the districtr map. + + Args: + session: The database session. + districtr_map_uuid: The UUID of the districtr map. + bounds: The bounds of the map. + """ + logger.info(f"Adding extent for {districtr_map_uuid}...") + + if bounds: + assert all( + b is not None for b in bounds + ), "If setting the extent manually, all values must be set." + assert len(bounds) == 4, "The extent must have 4 values." + assert all(isinstance(b, float) for b in bounds), "All values must be floats." + x_min, y_min, x_max, y_max = bounds + assert ( + x_max > x_min and y_max > y_min + ), "The max values must be greater than the min values." + stmt = text( + "UPDATE districtrmap SET extent = ARRAY[:x_min, :y_min, :x_max, :y_max] WHERE uuid = :districtr_map_uuid RETURNING extent" + ).bindparams( + bindparam(key="districtr_map_uuid", type_=UUIDType), + bindparam(key="x_min", type_=Float), + bindparam(key="y_min", type_=Float), + bindparam(key="x_max", type_=Float), + bindparam(key="y_max", type_=Float), + ) + (result,) = session.execute( + stmt, + { + "districtr_map_uuid": districtr_map_uuid, + "x_min": x_min, + "y_min": y_min, + "x_max": x_max, + "y_max": y_max, + }, + ).one() + logger.info( + f"Updated extent for districtr map {districtr_map_uuid} to {result}" + ) + return + + _select_result = session.execute( + statement=text( + "SELECT uuid FROM districtrmap WHERE uuid = :districtr_map_uuid" + ).bindparams(bindparam(key="districtr_map_uuid", type_=UUIDType)), + params={"districtr_map_uuid": districtr_map_uuid}, + ).one() + if _select_result is None: + raise ValueError( + f"Districtr map with UUID {districtr_map_uuid} does not exist." + ) + stmt = text(f""" + DO $$ + DECLARE + rec RECORD; + layer_extent GEOMETRY; + BEGIN + SELECT uuid, parent_layer + FROM districtrmap + WHERE uuid = '{districtr_map_uuid}'::UUID + INTO rec; + + EXECUTE format(' + SELECT ST_Extent(ST_Transform(geometry, 4326)) + FROM gerrydb.%I', + rec.parent_layer + ) INTO layer_extent; + + UPDATE districtrmap + SET extent = ARRAY[ + ST_XMin(layer_extent), + ST_YMin(layer_extent), + ST_XMax(layer_extent), + ST_YMax(layer_extent) + ] + WHERE uuid = rec.uuid; + + EXCEPTION WHEN undefined_table THEN + RAISE NOTICE 'Table % does not exist for layer %', rec.parent_layer, rec.name; + END $$; + """) + session.execute(stmt) diff --git a/backend/cli.py b/backend/cli.py index e99e149b7..11b592109 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -12,7 +12,7 @@ create_districtr_map as _create_districtr_map, create_shatterable_gerrydb_view as _create_shatterable_gerrydb_view, create_parent_child_edges as _create_parent_child_edges, - transform_bounding_box, + add_extent_to_districtrmap as _add_extent_to_districtrmap, ) logger = logging.getLogger(__name__) @@ -191,6 +191,18 @@ def delete_parent_child_edges(districtr_map: str): @click.option("--gerrydb-table-name", help="Name of the GerryDB table", required=True) @click.option("--num-districts", help="Number of districts", required=False) @click.option("--tiles-s3-path", help="S3 path to the tileset", required=False) +@click.option( + "--no-extent", help="Do not calculate extent", is_flag=True, default=False +) +@click.option( + "--bounds", + "-b", + help="Bounds of the extent as `--bounds x_min y_min x_max y_max`", + required=False, + type=float, + default=None, + nargs=4, +) def create_districtr_map( name: str, parent_layer_name: str, @@ -198,10 +210,12 @@ def create_districtr_map( gerrydb_table_name: str, num_districts: int | None, tiles_s3_path: str | None, + no_extent: bool = False, + bounds: list[float] | None = None, ): logger.info("Creating districtr map...") session = next(get_session()) - districtr_map_uuid = _create_districtr_map( + (districtr_map_uuid,) = _create_districtr_map( session=session, name=name, parent_layer_name=parent_layer_name, @@ -210,6 +224,15 @@ def create_districtr_map( num_districts=num_districts, tiles_s3_path=tiles_s3_path, ) + + if not no_extent: + logger.info( + f"Calculating extent... bounds received: {bounds}. If none will use parent layer extent." + ) + _add_extent_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid, bounds=bounds + ) + session.commit() logger.info(f"Districtr map created successfully {districtr_map_uuid}") @@ -237,5 +260,37 @@ def create_shatterable_gerrydb_view( ) +@cli.command("add-extent-to-districtr-map") +@click.option("--districtr-map", "-d", help="Districtr map name", required=True) +@click.option( + "--bounds", + "-b", + help="Bounds of the extent as `--bounds x_min y_min x_max y_max`", + required=False, + type=float, + default=None, + nargs=4, +) +def add_extent_to_districtr_map(districtr_map: str, bounds: list[float] | None = None): + logger.info(f"User provided bounds: {bounds}") + + session = next(get_session()) + stmt = text( + "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" + ) + (districtr_map_uuid,) = session.execute( + stmt, params={"districtrmap_name": districtr_map} + ).one() + print(f"Found districtmap uuid: {districtr_map_uuid}") + + _add_extent_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid, bounds=bounds + ) + session.commit() + logger.info("Updated extent successfully.") + + session.close() + + if __name__ == "__main__": cli() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index cc651b3ab..dfd53533e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -47,7 +47,23 @@ def engine_fixture(request): except (OperationalError, ProgrammingError): pass - subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) + try: + subprocess.run( + ["alembic", "upgrade", "head"], + check=True, + env=my_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as e: + print("Alembic upgrade failed:") + print("Return code:", e.returncode) + print( + "Standard Output:", e.output or e.stdout + ) # Prints any general output from the command + print("Error Output:", e.stderr) # Prints only the error output + raise e def teardown(): if TEARDOWN_TEST_DB: diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index cebda92a4..d0c0f0f30 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -4,6 +4,7 @@ create_districtr_map, create_shatterable_gerrydb_view, create_parent_child_edges, + add_extent_to_districtrmap, ) from sqlmodel import Session import subprocess @@ -143,9 +144,7 @@ def test_create_districtr_map( session.commit() -def test_create_districtr_map_some_nulls( - session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb -): +def test_create_districtr_map_some_nulls(session: Session, simple_parent_geos_gerrydb): # This is also an example of a districtr map before other set-up operations # are performed, such as creating a tileset and a shatterable view (inserted_districtr_map,) = create_districtr_map( @@ -157,6 +156,34 @@ def test_create_districtr_map_some_nulls( session.commit() +def test_add_extent_to_districtrmap(session: Session, simple_parent_geos_gerrydb): + (inserted_districtr_map,) = create_districtr_map( + session, + name="Simple non-shatterable layer 2", + gerrydb_table_name="simple_parent_geos_some_nulls2", + parent_layer_name="simple_parent_geos", + ) + add_extent_to_districtrmap( + session=session, districtr_map_uuid=inserted_districtr_map + ) + + +def test_add_extent_to_districtrmap_manual_bounds( + session: Session, simple_parent_geos_gerrydb +): + (inserted_districtr_map,) = create_districtr_map( + session, + name="Simple non-shatterable layer 2", + gerrydb_table_name="simple_parent_geos_some_nulls2", + parent_layer_name="simple_parent_geos", + ) + add_extent_to_districtrmap( + session=session, + districtr_map_uuid=inserted_districtr_map, + bounds=[-109.06, 36.99, -102.04, 41.00], + ) + + def test_create_shatterable_gerrydb_view( session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb ): From 7cefb43e7c131ee684cb3f75fa6b028f2c9d3b84 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sun, 3 Nov 2024 19:06:24 -0500 Subject: [PATCH 25/50] Fix typo (#162) --- .github/workflows/fly-deploy-pr.yml | 3 +-- .../app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fly-deploy-pr.yml b/.github/workflows/fly-deploy-pr.yml index 64379460a..b32a78fe0 100644 --- a/.github/workflows/fly-deploy-pr.yml +++ b/.github/workflows/fly-deploy-pr.yml @@ -60,8 +60,7 @@ jobs: --name ${{ github.event.repository.name }}-${{ github.event.number }}-db \ --region ewr \ --initial-cluster-size 1 \ - --vm-size shared-cpu-1x \ - --vm-memory 512 \ + --vm-size shared-cpu-2x \ -p ${{ secrets.FLY_PR_PG_PASSWORD }} \ --org mggg \ --fork-from districtr-v2-db diff --git a/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py b/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py index 61fa9729e..f55a1ca47 100644 --- a/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py +++ b/backend/app/alembic/versions/dc391733e10a_add_extent_to_gerrydb.py @@ -34,7 +34,7 @@ def upgrade() -> None: layer_extent GEOMETRY; BEGIN FOR rec IN - SELECT name, parent_layer + SELECT uuid, parent_layer FROM districtrmap LOOP BEGIN From cfb933b2b6a6bfd2956f898addde74bc3a0d630c Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 8 Nov 2024 13:46:59 -0600 Subject: [PATCH 26/50] Mobile UI Enhancements (#128) --- app/src/app/components/Map.tsx | 2 +- .../app/components/sidebar/ColorPicker.jsx | 2 +- app/src/app/components/sidebar/DataPanels.tsx | 66 ++++++++++++++++ .../components/sidebar/MapModeSelector.jsx | 2 +- .../components/sidebar/MobileColorPicker.jsx | 32 ++++++++ .../app/components/sidebar/MobileTopNav.tsx | 76 +++++++++++++++++++ .../components/sidebar/RecentMapsModal.tsx | 11 ++- app/src/app/components/sidebar/Sidebar.tsx | 72 ++++++++++-------- app/src/app/map/page.tsx | 20 ++--- app/src/app/page.tsx | 8 +- app/src/app/utils/events/mapEvents.ts | 9 ++- 11 files changed, 252 insertions(+), 48 deletions(-) create mode 100644 app/src/app/components/sidebar/DataPanels.tsx create mode 100644 app/src/app/components/sidebar/MobileColorPicker.jsx create mode 100644 app/src/app/components/sidebar/MobileTopNav.tsx diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index f33be4c3e..fd047fcfb 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -80,7 +80,7 @@ export const MapComponent: React.FC = () => { return (

{colorArray.map((color, i) => ( , + }, + { + title: "layers", + label: "Data layers", + content: , + }, + { + title: "evaluation", + label: "Evaluation", + content: Unimplemented , + }, +]; + +const DataPanels: React.FC = ({ + defaultPanel = defaultPanels[0].title, + panels = defaultPanels, +}) => { + return ( + + + {panels.map((panel) => ( + + {panel.label} + + ))} + + + {panels.map((panel) => ( + + {panel.content} + + ))} + + + ); +}; + +export default DataPanels; diff --git a/app/src/app/components/sidebar/MapModeSelector.jsx b/app/src/app/components/sidebar/MapModeSelector.jsx index d2dcd2b47..d24677e8c 100644 --- a/app/src/app/components/sidebar/MapModeSelector.jsx +++ b/app/src/app/components/sidebar/MapModeSelector.jsx @@ -39,7 +39,7 @@ export function MapModeSelector() { defaultValue="default" value={activeTool} onValueChange={handleRadioChange} - columns={{ initial: "1", sm: "3" }} + columns={{ initial: "3" }} > {activeTools.map((tool) => ( diff --git a/app/src/app/components/sidebar/MobileColorPicker.jsx b/app/src/app/components/sidebar/MobileColorPicker.jsx new file mode 100644 index 000000000..9f88ef58e --- /dev/null +++ b/app/src/app/components/sidebar/MobileColorPicker.jsx @@ -0,0 +1,32 @@ +import React, { useState } from "react"; +import { Button, IconButton, Popover } from "@radix-ui/themes"; // Import Popover from Radix +import { ColorPicker } from "./ColorPicker"; +import { colorScheme } from "@/app/constants/colors"; +import { useMapStore } from "@/app/store/mapStore"; +import { ColorWheelIcon } from "@radix-ui/react-icons"; + +export const MobileColorPicker = () => { + const [open, setOpen] = useState(false); + const selectedZone = useMapStore((state) => state.selectedZone); + + const zoneIndex = selectedZone ? selectedZone - 1 : 0; + const color = colorScheme[zoneIndex]; + + return ( + + + + + + + + + + + ); +}; diff --git a/app/src/app/components/sidebar/MobileTopNav.tsx b/app/src/app/components/sidebar/MobileTopNav.tsx new file mode 100644 index 000000000..e438cbc0c --- /dev/null +++ b/app/src/app/components/sidebar/MobileTopNav.tsx @@ -0,0 +1,76 @@ +import { Box, Button, Flex, Heading } from "@radix-ui/themes"; +import React, { useRef, useState } from "react"; +import { Cross2Icon, GearIcon } from "@radix-ui/react-icons"; +import DataPanels from "./DataPanels"; + +const MobileTopNav = () => { + const [dataPanelOpen, setDataPanelOpen] = useState(false); + const handleToggleDataPanel = () => setDataPanelOpen((prev) => !prev); + + const boxRef = useRef(null); + const topBarHeight = + boxRef.current?.getClientRects()?.[0]?.height || 44.90625; + const isLandscape = typeof window !== 'undefined' && window.matchMedia("(orientation: landscape)").matches; + + return ( + + + + Districtr + + + + + + + + + {dataPanelOpen && ( + + + + )} + + ); +}; + +export default MobileTopNav; diff --git a/app/src/app/components/sidebar/RecentMapsModal.tsx b/app/src/app/components/sidebar/RecentMapsModal.tsx index 3226e2b0b..294749539 100644 --- a/app/src/app/components/sidebar/RecentMapsModal.tsx +++ b/app/src/app/components/sidebar/RecentMapsModal.tsx @@ -14,7 +14,14 @@ import { } from '@radix-ui/themes'; import {usePathname, useSearchParams, useRouter} from 'next/navigation'; import {DocumentObject} from '../../utils/api/apiHandlers'; +import { styled } from '@stitches/react'; type NamedDocumentObject = DocumentObject & {name?: string}; + +const DialogContentContainer = styled(Dialog.Content, { + maxWidth: "calc(100vw - 2rem)", + maxHeight: "calc(100vh-2rem)" +}) + export const RecentMapsModal = () => { const router = useRouter(); const pathname = usePathname(); @@ -46,7 +53,7 @@ export const RecentMapsModal = () => { Recents - + Recent Maps @@ -84,7 +91,7 @@ export const RecentMapsModal = () => { ))} - + ); }; diff --git a/app/src/app/components/sidebar/Sidebar.tsx b/app/src/app/components/sidebar/Sidebar.tsx index 4b0fc833a..3623e74bf 100644 --- a/app/src/app/components/sidebar/Sidebar.tsx +++ b/app/src/app/components/sidebar/Sidebar.tsx @@ -1,56 +1,66 @@ +import React from "react"; +import DataPanels from "./DataPanels"; import {Box, Flex, Heading} from '@radix-ui/themes'; import {MapModeSelector} from './MapModeSelector'; import {ColorPicker} from './ColorPicker'; import {ResetMapButton} from './ResetMapButton'; import {GerryDBViewSelector} from './GerryDBViewSelector'; -import {HorizontalBar} from './charts/HorizontalBarChart'; import {useMapStore} from '@/app/store/mapStore'; -import {Tabs, Text} from '@radix-ui/themes'; -import Layers from './Layers'; import PaintByCounty from './PaintByCounty'; import {BrushSizeSelector} from './BrushSizeSelector'; +import {MobileColorPicker} from "./MobileColorPicker" export default function SidebarComponent() { const activeTool = useMapStore(state => state.activeTool); return ( - + - + Districtr - {activeTool === 'brush' || activeTool === 'eraser' ? ( -
+ {activeTool === "brush" || activeTool === "eraser" ? ( +
+
- {' '} -
- ) : null} - {activeTool === 'brush' ? ( -
- + {" "} +
+ {activeTool === "brush" ? ( +
+ + + + + + +
+ ) : null}
) : null} - - - Population - Data layers - Evaluation - - - - - - - - - - Unimplemented - - - + + + ); diff --git a/app/src/app/map/page.tsx b/app/src/app/map/page.tsx index a2d3592c5..94df8d25b 100644 --- a/app/src/app/map/page.tsx +++ b/app/src/app/map/page.tsx @@ -1,18 +1,20 @@ -'use client'; -import React from 'react'; -import {MapContextMenu} from '../components/ContextMenu'; -import {MapComponent} from '../components/Map'; -import SidebarComponent from '../components/sidebar/Sidebar'; -import {QueryClientProvider} from '@tanstack/react-query'; -import {queryClient} from '../utils/api/queryClient'; +"use client"; +import React from "react"; +import { MapContextMenu } from "../components/ContextMenu"; +import { MapComponent } from "../components/Map"; +import SidebarComponent from "../components/sidebar/Sidebar"; +import MobileTopNav from "../components/sidebar/MobileTopNav"; +import { QueryClientProvider } from "@tanstack/react-query"; +import { queryClient } from "../utils/api/queryClient"; export default function Map() { if (queryClient) { return ( -
- +
+ +
diff --git a/app/src/app/page.tsx b/app/src/app/page.tsx index c6b60f8bb..97e644c6b 100644 --- a/app/src/app/page.tsx +++ b/app/src/app/page.tsx @@ -7,8 +7,12 @@ import {queryClient} from './utils/api/queryClient'; export default function Home() { return ( -
-
+
+
diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index aa6ad6187..ac988f103 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -164,9 +164,14 @@ export const handleMapMouseMove = ( mapStore.brushSize, paintLayers, ); + // sourceCapabilities exists on the UIEvent constructor, which does not appear + // properly tpyed in the default map events + // https://developer.mozilla.org/en-US/docs/Web/API/UIEvent/sourceCapabilities + const isTouchEvent = 'touches' in e || (e.originalEvent as any)?.sourceCapabilities?.firesTouchEvents + const isBrushingTool = sourceLayer && ["brush", "eraser", "shatter"].includes(activeTool); - if (isBrushingTool) { + if (isBrushingTool && !isTouchEvent) { setHoverFeatures(selectedFeatures); } @@ -246,6 +251,8 @@ export const mapEvents = [ { action: "mouseover", handler: handleMapMouseOver }, { action: "mouseleave", handler: handleMapMouseLeave }, { action: "touchleave", handler: handleMapMouseLeave }, + { action: "touchend", handler: handleMapMouseUp }, + { action: "touchcancel", handler: handleMapMouseUp }, { action: "mouseout", handler: handleMapMouseOut }, { action: "mousemove", handler: handleMapMouseMove }, { action: "touchmove", handler: handleMapMouseMove }, From f857b09fa1f9282a4051f70a30f54eefee100f4a Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 11 Nov 2024 15:39:37 -0600 Subject: [PATCH 27/50] Reset Map (#126) --- .../app/components/sidebar/ResetMapButton.tsx | 11 ++--- app/src/app/store/mapRenderSubs.ts | 8 +++ app/src/app/store/mapStore.ts | 36 +++++++++++++- app/src/app/utils/api/apiHandlers.ts | 28 +++++++++++ app/src/app/utils/api/mutations.ts | 19 +++++++ app/src/app/utils/helpers.ts | 49 +++++++++++++++++++ backend/app/main.py | 20 ++++++++ backend/tests/test_main.py | 9 ++++ 8 files changed, 171 insertions(+), 9 deletions(-) diff --git a/app/src/app/components/sidebar/ResetMapButton.tsx b/app/src/app/components/sidebar/ResetMapButton.tsx index 06f8af11b..14a811fe3 100644 --- a/app/src/app/components/sidebar/ResetMapButton.tsx +++ b/app/src/app/components/sidebar/ResetMapButton.tsx @@ -2,16 +2,11 @@ import {useMapStore} from '@/app/store/mapStore'; import {Button} from '@radix-ui/themes'; export function ResetMapButton() { - const mapStore = useMapStore.getState(); - - const handleClickResetMap = () => { - mapStore.setFreshMap(true); - // clear map metrics - mapStore.setMapMetrics(null); - }; + const handleClickResetMap = useMapStore(state => state.handleReset) + const noZonesAreAssigned = useMapStore(state => !state.zoneAssignments.size) return ( - ); diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 9b906c349..73e80d585 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -5,6 +5,7 @@ import { PARENT_LAYERS, CHILD_LAYERS, getLayerFilter, + BLOCK_SOURCE_ID, } from '@constants/layers'; import { ColorZoneAssignmentsState, @@ -50,6 +51,13 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { layersToFilter.forEach(layerId => mapRef.setFilter(layerId, getLayerFilter(layerId, shatterIds)) ); + shatterIds.parents.forEach((id) => { + mapRef?.removeFeatureState({ + source: BLOCK_SOURCE_ID, + id, + sourceLayer: state.mapDocument?.parent_layer, + }); + }); mapRef.once('render', () => { setMapLock(false); diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 5bfd82129..748372ece 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -23,10 +23,11 @@ import { LayerVisibility, PaintEventHandler, getFeaturesInBbox, + resetZoneColors, setZones, } from "../utils/helpers"; import { getRenderSubscriptions } from "./mapRenderSubs"; -import { patchShatter } from "../utils/api/mutations"; +import { patchReset, patchShatter } from "../utils/api/mutations"; import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; import { getMapMetricsSubs } from "./metricsSubs"; import { getMapEditSubs } from "./mapEditSubs"; @@ -80,6 +81,7 @@ export interface MapStore { resetZoneAssignments: () => void; zonePopulations: Map; setZonePopulations: (zone: Zone, population: number) => void; + handleReset: () => void; accumulatedGeoids: Set; setAccumulatedGeoids: (geoids: MapStore["accumulatedGeoids"]) => void; brushSize: number; @@ -232,6 +234,38 @@ export const useMapStore = create( zoneAssignments, }); }, + handleReset: async () => { + const {mapDocument, getMapRef, zoneAssignments, shatterIds} = get(); + const document_id = mapDocument?.document_id + + if (!document_id) { + console.log("No document ID to reset."); + return; + } + set({ + mapLock: true, + appLoadingState: "loading", + }); + const resetResponse = await patchReset.mutate(document_id); + + if (resetResponse.document_id === document_id) { + const initialState = useMapStore.getInitialState(); + resetZoneColors({ + zoneAssignments, + mapRef: getMapRef(), + mapDocument, + shatterIds + }) + + set({ + zonePopulations: initialState.zonePopulations, + zoneAssignments: initialState.zoneAssignments, + shatterIds: initialState.shatterIds, + appLoadingState: "loaded", + mapLock: false, + }); + } + }, setShatterIds: ( existingParents, existingChildren, diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 377251532..bc4be27ac 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -189,6 +189,17 @@ export interface AssignmentsCreate { assignments_upserted: number; } +/** + * Reset assignments response + * @interface +* @property {boolean} success - Confirming if the operation succeeded + * @property {string} document_id - Document ID where assignments were dropped + */ +export interface AssignmentsReset { + success: boolean; + document_id: string; +} + /** * * @param assignments @@ -206,6 +217,23 @@ export const patchUpdateAssignments: ( }); }; +/** + * + * @param assignments + * @returns server object containing the updated assignments per geoid + */ +export const patchUpdateReset: ( + document_id: string, +) => Promise = async (document_id) => { + return await axios + .patch(`${process.env.NEXT_PUBLIC_API_URL}/api/update_assignments/${document_id}/reset`, { + document_id + }) + .then((res) => { + return res.data; + }); +}; + /** * Shatter result * @interface diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts index 6bdc1d580..4f29e5d58 100644 --- a/app/src/app/utils/api/mutations.ts +++ b/app/src/app/utils/api/mutations.ts @@ -2,9 +2,11 @@ import {MutationObserver} from '@tanstack/query-core'; import {queryClient} from './queryClient'; import { AssignmentsCreate, + AssignmentsReset, createMapDocument, patchShatterParents, patchUpdateAssignments, + patchUpdateReset, } from '@/app/utils/api/apiHandlers'; import {useMapStore} from '@/app/store/mapStore'; import {mapMetrics} from './queries'; @@ -43,6 +45,23 @@ export const patchUpdates = new MutationObserver(queryClient, { }, }); + +export const patchReset = new MutationObserver(queryClient, { + mutationFn: patchUpdateReset, + onMutate: () => { + console.log("Reseting map"); + }, + onError: (error) => { + console.log("Error reseting map: ", error); + }, + onSuccess: (data: AssignmentsReset) => { + console.log( + `Successfully reset ${data.document_id}` + ); + mapMetrics.refetch(); + }, +}); + export const document = new MutationObserver(queryClient, { mutationFn: createMapDocument, onMutate: () => { diff --git a/app/src/app/utils/helpers.ts b/app/src/app/utils/helpers.ts index 8953242a4..48a6c5c2e 100644 --- a/app/src/app/utils/helpers.ts +++ b/app/src/app/utils/helpers.ts @@ -322,6 +322,55 @@ export const colorZoneAssignments = ( }); }; +/** + * resetZoneColors + * Resets the zone colors for the specified feature IDs on the map. + * + * This function sets the feature state for each ID in the provided set or array to indicate that + * the zone color should be reset. It checks if the map document is available and determines + * the appropriate source layer based on the existence of child layers and shatter IDs. + * + * @param {Set | string[]} ids - A set or array of feature IDs for which to reset the zone colors. + * @param {ReturnType} mapRef - The maplibre map instance used to set the feature state. + * @param {MapStore['mapDocument']} mapDocument - The map document containing layer information. + * @param {MapStore['shatterIds']} shatterIds - The shatter IDs used to determine layer types. + */ +export const resetZoneColors = ({ + ids, zoneAssignments, mapRef, mapDocument, shatterIds +}: { + ids?: Set | string[], + zoneAssignments?: MapStore['zoneAssignments'] + mapRef: ReturnType, + mapDocument: MapStore['mapDocument'], + shatterIds: MapStore['shatterIds'] +}) => { + const idsToReset = ids ? Array.from(ids) : zoneAssignments ? Array.from(zoneAssignments.keys()) : null + if (!mapDocument || !mapRef || !idsToReset) return + const childLayerExists = mapDocument?.child_layer + const shatterIdsExist = shatterIds.parents.size + const getSourceLayer = childLayerExists && shatterIdsExist + ? (id: string) => { + return shatterIds.children.has(id) + ? mapDocument.child_layer! + : mapDocument.parent_layer + } + : (_: string) => mapDocument.parent_layer + idsToReset.forEach(id => { + const sourceLayer = getSourceLayer(id) + mapRef?.setFeatureState( + { + source: BLOCK_SOURCE_ID, + id, + sourceLayer, + }, + { + selected: true, + zone: null, + } + ); + }) +} + // property changes on which to re-color assignments export const colorZoneAssignmentTriggers = [ 'zoneAssignments', diff --git a/backend/app/main.py b/backend/app/main.py index 2b4657a5b..4631839b7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -174,6 +174,26 @@ async def shatter_parent( return result +@app.patch( + "/api/update_assignments/{document_id}/reset", status_code=status.HTTP_200_OK +) +async def reset_map(document_id: str, session: Session = Depends(get_session)): + # Drop the partition for the given assignments + partition_name = f'"document.assignments_{document_id}"' + session.execute(text(f"DROP TABLE IF EXISTS {partition_name} CASCADE;")) + + # Recreate the partition + session.execute( + text(f""" + CREATE TABLE {partition_name} PARTITION OF document.assignments + FOR VALUES IN ('{document_id}'); + """) + ) + session.commit() + + return {"message": "Assignments partition reset", "document_id": document_id} + + # called by getAssignments in apiHandlers.ts @app.get("/api/get_assignments/{document_id}", response_model=list[AssignmentsResponse]) async def get_assignments(document_id: str, session: Session = Depends(get_session)): diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 8ada690af..ae2622cfc 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -374,6 +374,15 @@ def test_patch_assignments_twice(client, document_id): assert data[1]["geo_id"] == "202090434001003" +def test_patch_reset_assignments(client, document_id): + test_patch_assignments(client, document_id) + response = client.patch(f"/api/update_assignments/{document_id}/reset") + assert response.status_code == 200 + assignments = client.get(f"/api/get_assignments/{document_id}") + assert assignments.status_code == 200 + assert len(assignments.json()) == 0 + + def test_get_document_population_totals_null_assignments( client, document_id, ks_demo_view_census_blocks ): From 6cc5e6e47461750dd7cfd4b2b291c93977dd94f9 Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 12 Nov 2024 09:27:40 -0600 Subject: [PATCH 28/50] Enhanced shattering (#129) Co-authored-by: Raphael Paul Laude --- app/.prettierrc.js | 15 +- app/README.md | 19 +- app/next.config.mjs | 24 +- app/package-lock.json | 26 +- app/package.json | 6 +- app/sentry.client.config.ts | 4 +- app/sentry.edge.config.ts | 4 +- app/sentry.server.config.ts | 5 +- app/src/app/components/ContextMenu.tsx | 62 +- app/src/app/components/Map.tsx | 53 +- .../app/components/sidebar/ColorPicker.jsx | 90 - .../app/components/sidebar/ColorPicker.tsx | 134 ++ app/src/app/components/sidebar/DataPanels.tsx | 36 +- .../sidebar/ExitBlockViewButtons.tsx | 18 + app/src/app/components/sidebar/Layers.tsx | 20 +- .../components/sidebar/MapModeSelector.jsx | 69 +- .../components/sidebar/MobileColorPicker.jsx | 4 +- .../app/components/sidebar/MobileTopNav.tsx | 36 +- app/src/app/components/sidebar/Picker.jsx | 29 +- .../components/sidebar/RecentMapsModal.tsx | 12 +- .../app/components/sidebar/ResetMapButton.tsx | 4 +- app/src/app/components/sidebar/Sidebar.tsx | 51 +- .../app/components/sidebar/ZoneLockPicker.tsx | 36 + app/src/app/components/sidebar/ZonePicker.tsx | 79 + .../components/sidebar/charts/ResizableBox.js | 26 +- app/src/app/constants/basemapLayers.ts | 2099 ++++++----------- app/src/app/constants/configuration.ts | 12 +- app/src/app/constants/layers.ts | 167 +- app/src/app/constants/types.ts | 33 +- app/src/app/globals.css | 34 +- app/src/app/layout.tsx | 23 +- app/src/app/map/page.tsx | 16 +- app/src/app/page.tsx | 14 +- app/src/app/pages/s/[sessionId].tsx | 4 +- app/src/app/store/mapEditSubs.ts | 60 +- app/src/app/store/mapRenderSubs.ts | 154 +- app/src/app/store/mapStore.ts | 741 ++++-- app/src/app/store/types.ts | 7 + app/src/app/utils/api/apiHandlers.ts | 35 +- app/src/app/utils/api/mutations.ts | 38 +- app/src/app/utils/api/queryParamsListener.ts | 3 + app/src/app/utils/events/handlers.ts | 90 +- app/src/app/utils/events/mapEvents.ts | 175 +- app/src/app/utils/helpers.ts | 170 +- app/src/app/utils/zone-helpers.ts | 27 + app/tailwind.config.ts | 17 +- app/tsconfig.json | 1 - .../65a4fc0a727d_add_unshatter_udf.py | 38 + backend/app/main.py | 30 +- backend/app/models.py | 4 + backend/app/sql/unshatter_parent.sql | 44 + backend/tests/test_utils.py | 37 + 52 files changed, 2699 insertions(+), 2236 deletions(-) delete mode 100644 app/src/app/components/sidebar/ColorPicker.jsx create mode 100644 app/src/app/components/sidebar/ColorPicker.tsx create mode 100644 app/src/app/components/sidebar/ExitBlockViewButtons.tsx create mode 100644 app/src/app/components/sidebar/ZoneLockPicker.tsx create mode 100644 app/src/app/components/sidebar/ZonePicker.tsx create mode 100644 app/src/app/store/types.ts create mode 100644 app/src/app/utils/zone-helpers.ts create mode 100644 backend/app/alembic/versions/65a4fc0a727d_add_unshatter_udf.py create mode 100644 backend/app/sql/unshatter_parent.sql diff --git a/app/.prettierrc.js b/app/.prettierrc.js index 5dc5d8a8e..0e3276ee3 100644 --- a/app/.prettierrc.js +++ b/app/.prettierrc.js @@ -1,3 +1,16 @@ module.exports = { - ...require('prettier-airbnb-config'), + // https://github.com/Parker-Ledoux/prettier-airbnb-config + "$schema": "http://json.schemastore.org/prettierrc", + "arrowParens": "avoid", + "bracketSpacing": false, + "jsxBracketSameLine": false, + "jsxSingleQuote": false, + "printWidth": 100, + "proseWrap": "always", + "quoteProps": "as-needed", + "semi": true, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "es5", + "useTabs": false }; diff --git a/app/README.md b/app/README.md index 80e9fa38e..d00d2f1d1 100644 --- a/app/README.md +++ b/app/README.md @@ -1,4 +1,5 @@ -This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). +This is a [Next.js](https://nextjs.org/) project bootstrapped with +[`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). ## Getting Started @@ -22,9 +23,11 @@ bun dev Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. -You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. +You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the +file. -This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font. +This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to +automatically optimize and load Inter, a custom Google Font. ## Learn More @@ -33,10 +36,14 @@ To learn more about Next.js, take a look at the following resources: - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. -You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your +feedback and contributions are welcome! ## Deploy on Vercel -The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. +The easiest way to deploy your Next.js app is to use the +[Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) +from the creators of Next.js. -Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. +Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more +details. diff --git a/app/next.config.mjs b/app/next.config.mjs index 5db6eadb4..145a32e20 100644 --- a/app/next.config.mjs +++ b/app/next.config.mjs @@ -1,12 +1,12 @@ -import { withSentryConfig } from "@sentry/nextjs"; +import {withSentryConfig} from '@sentry/nextjs'; /** @type {import('next').NextConfig} */ const nextConfig = { async redirects() { return [ // Basic redirect { - source: "/", - destination: "/map", + source: '/', + destination: '/map', permanent: true, }, ]; @@ -16,12 +16,12 @@ const nextConfig = { }, resolve: { alias: { - "@src": "app/src", - "@components": "app/src/components", - "@utils": "app/src/utils", - "@api": "app/src/api", - "@store": "app/src/store", - "@constants": "app/src/constants", + '@src': 'app/src', + '@components': 'app/src/components', + '@utils': 'app/src/utils', + '@api': 'app/src/api', + '@store': 'app/src/store', + '@constants': 'app/src/constants', }, }, }; @@ -30,8 +30,8 @@ export default withSentryConfig(nextConfig, { // For all available options, see: // https://github.com/getsentry/sentry-webpack-plugin#options - org: "mggg-districtr", - project: "districtr-v2-app", + org: 'mggg-districtr', + project: 'districtr-v2-app', // Only print logs for uploading source maps in CI silent: !process.env.CI, @@ -51,7 +51,7 @@ export default withSentryConfig(nextConfig, { // This can increase your server load as well as your hosting bill. // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client- // side errors will fail. - tunnelRoute: "/monitoring", + tunnelRoute: '/monitoring', // Hides source maps from generated client bundles hideSourceMaps: true, diff --git a/app/package-lock.json b/app/package-lock.json index ce236f06c..8e9d77c75 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -16,6 +16,7 @@ "@sentry/nextjs": "^8.26.0", "@stitches/react": "^1.2.8", "@tanstack/react-query": "^5.51.11", + "@turf/bbox": "^7.1.0", "@turf/boolean-within": "^7.1.0", "@turf/helpers": "^7.1.0", "@turf/point-on-feature": "^7.1.0", @@ -45,7 +46,7 @@ "jest": "^29.7.0", "nock": "^13.5.4", "postcss": "^8", - "prettier-airbnb-config": "^1.0.0", + "prettier": "^3.3.3", "tailwindcss": "^3.4.1", "typescript": "^5" } @@ -10940,25 +10941,18 @@ } }, "node_modules/prettier": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", - "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", "dev": true, - "peer": true, "bin": { - "prettier": "bin-prettier.js" + "prettier": "bin/prettier.cjs" }, "engines": { - "node": ">=4" - } - }, - "node_modules/prettier-airbnb-config": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/prettier-airbnb-config/-/prettier-airbnb-config-1.0.0.tgz", - "integrity": "sha512-FsEe38fJftYi19AskzuOWbw8zBnXDcZwv2Uw8BI65ROyK4hE6+iojXVFu+FiYNfrRBL+UZJbl2XGk56Ur2QGsA==", - "dev": true, - "peerDependencies": { - "prettier": "^1.18.2" + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" } }, "node_modules/pretty-format": { diff --git a/app/package.json b/app/package.json index 44793d00b..417629e60 100644 --- a/app/package.json +++ b/app/package.json @@ -6,7 +6,8 @@ "dev": "next dev", "build": "next build", "start": "next start", - "lint": "next lint" + "lint": "next lint", + "prettier": "prettier --write src/**/*.{tsx,ts} --config .prettierrc.js" }, "dependencies": { "@emotion/react": "^11.13.0", @@ -17,6 +18,7 @@ "@sentry/nextjs": "^8.26.0", "@stitches/react": "^1.2.8", "@tanstack/react-query": "^5.51.11", + "@turf/bbox": "^7.1.0", "@turf/boolean-within": "^7.1.0", "@turf/helpers": "^7.1.0", "@turf/point-on-feature": "^7.1.0", @@ -46,7 +48,7 @@ "jest": "^29.7.0", "nock": "^13.5.4", "postcss": "^8", - "prettier-airbnb-config": "^1.0.0", + "prettier": "^3.3.3", "tailwindcss": "^3.4.1", "typescript": "^5" } diff --git a/app/sentry.client.config.ts b/app/sentry.client.config.ts index e33f81673..889aec390 100644 --- a/app/sentry.client.config.ts +++ b/app/sentry.client.config.ts @@ -2,10 +2,10 @@ // The config you add here will be used whenever a users loads a page in their browser. // https://docs.sentry.io/platforms/javascript/guides/nextjs/ -import * as Sentry from "@sentry/nextjs"; +import * as Sentry from '@sentry/nextjs'; Sentry.init({ - dsn: "https://e42d7ccd60a760cd2885b3f394b3a73b@o4507623009091584.ingest.us.sentry.io/4507794033082368", + dsn: 'https://e42d7ccd60a760cd2885b3f394b3a73b@o4507623009091584.ingest.us.sentry.io/4507794033082368', // Adjust this value in production, or use tracesSampler for greater control tracesSampleRate: 1, diff --git a/app/sentry.edge.config.ts b/app/sentry.edge.config.ts index 9f7e6dd65..e486b94c8 100644 --- a/app/sentry.edge.config.ts +++ b/app/sentry.edge.config.ts @@ -3,10 +3,10 @@ // Note that this config is unrelated to the Vercel Edge Runtime and is also required when running locally. // https://docs.sentry.io/platforms/javascript/guides/nextjs/ -import * as Sentry from "@sentry/nextjs"; +import * as Sentry from '@sentry/nextjs'; Sentry.init({ - dsn: "https://e42d7ccd60a760cd2885b3f394b3a73b@o4507623009091584.ingest.us.sentry.io/4507794033082368", + dsn: 'https://e42d7ccd60a760cd2885b3f394b3a73b@o4507623009091584.ingest.us.sentry.io/4507794033082368', // Adjust this value in production, or use tracesSampler for greater control tracesSampleRate: 1, diff --git a/app/sentry.server.config.ts b/app/sentry.server.config.ts index f7129bd6c..2b46d3228 100644 --- a/app/sentry.server.config.ts +++ b/app/sentry.server.config.ts @@ -2,10 +2,10 @@ // The config you add here will be used whenever the server handles a request. // https://docs.sentry.io/platforms/javascript/guides/nextjs/ -import * as Sentry from "@sentry/nextjs"; +import * as Sentry from '@sentry/nextjs'; Sentry.init({ - dsn: "https://e42d7ccd60a760cd2885b3f394b3a73b@o4507623009091584.ingest.us.sentry.io/4507794033082368", + dsn: 'https://e42d7ccd60a760cd2885b3f394b3a73b@o4507623009091584.ingest.us.sentry.io/4507794033082368', // Adjust this value in production, or use tracesSampler for greater control tracesSampleRate: 1, @@ -15,5 +15,4 @@ Sentry.init({ // Uncomment the line below to enable Spotlight (https://spotlightjs.com) // spotlight: process.env.NODE_ENV === 'development', - }); diff --git a/app/src/app/components/ContextMenu.tsx b/app/src/app/components/ContextMenu.tsx index ecaeea7d9..7fe0a5167 100644 --- a/app/src/app/components/ContextMenu.tsx +++ b/app/src/app/components/ContextMenu.tsx @@ -1,19 +1,45 @@ import React from 'react'; import {ContextMenu, Text} from '@radix-ui/themes'; import {useMapStore} from '@/app/store/mapStore'; +import {CHILD_LAYERS, PARENT_LAYERS} from '../constants/layers'; export const MapContextMenu: React.FC = () => { const mapDocument = useMapStore(state => state.mapDocument); const contextMenu = useMapStore(state => state.contextMenu); const handleShatter = useMapStore(state => state.handleShatter); + const lockedFeatures = useMapStore(state => state.lockedFeatures); + const lockFeature = useMapStore(state => state.lockFeature); + const shatterMappings = useMapStore(state => state.shatterMappings); + + const canShatter = Boolean( + mapDocument?.parent_layer && + mapDocument.child_layer && + mapDocument.child_layer !== contextMenu?.data.sourceLayer + ); + if (!contextMenu) return null; + const isChild = CHILD_LAYERS.includes(contextMenu.data.layer.id); + const id = contextMenu.data.id?.toString() || ''; + const parent = + (isChild && + Object.entries(shatterMappings).find(([key, value]) => { + return value.has(id); + })?.[0]) || + false; + const shatterableId = isChild && parent ? parent : contextMenu?.data?.id; + const featureIsLocked = lockedFeatures.has(id); const handleSelect = () => { - if (!mapDocument || contextMenu?.data?.id === undefined) return; - handleShatter(mapDocument.document_id, [contextMenu.data.id.toString()]); + if (!mapDocument || !shatterableId) return; + const shatterData = isChild ? {id: shatterableId} : contextMenu.data; + handleShatter(mapDocument.document_id, [shatterData]); contextMenu.close(); }; + const handleLock = () => { + lockFeature(id, !featureIsLocked); + }; + return ( { left: contextMenu.x, }} > - {contextMenu.data.id && ( - - - {contextMenu.data.id} - - + + + {id} + + + {!isChild && ( + + Break to Blocks + )} - - Shatter + + {featureIsLocked ? 'Unlock' : 'Lock'} + + {!!parent && ( + <> + + + Parent: {parent} + + + + Break Parent to Blocks + + + )} ); diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index fd047fcfb..908a70666 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -1,33 +1,30 @@ -import type { Map, MapLayerEventType } from "maplibre-gl"; -import maplibregl, { - MapLayerMouseEvent, - MapLayerTouchEvent, -} from "maplibre-gl"; -import "maplibre-gl/dist/maplibre-gl.css"; -import { Protocol } from "pmtiles"; -import type { MutableRefObject } from "react"; -import React, { useEffect, useRef } from "react"; -import { MAP_OPTIONS } from "../constants/configuration"; -import { mapEvents } from "../utils/events/mapEvents"; -import { INTERACTIVE_LAYERS } from "../constants/layers"; -import { useMapStore } from "../store/mapStore"; +import type {Map, MapLayerEventType} from 'maplibre-gl'; +import maplibregl, {MapLayerMouseEvent, MapLayerTouchEvent} from 'maplibre-gl'; +import 'maplibre-gl/dist/maplibre-gl.css'; +import {Protocol} from 'pmtiles'; +import type {MutableRefObject} from 'react'; +import React, {useEffect, useRef} from 'react'; +import {MAP_OPTIONS} from '../constants/configuration'; +import {mapEvents} from '../utils/events/mapEvents'; +import {INTERACTIVE_LAYERS} from '../constants/layers'; +import {useMapStore} from '../store/mapStore'; export const MapComponent: React.FC = () => { const map: MutableRefObject = useRef(null); const mapContainer: MutableRefObject = useRef(null); - const mapLock = useMapStore((state) => state.mapLock); - const setMapRef = useMapStore((state) => state.setMapRef); - const mapOptions = useMapStore((state) => state.mapOptions); + const mapLock = useMapStore(state => state.mapLock); + const setMapRef = useMapStore(state => state.setMapRef); + const mapOptions = useMapStore(state => state.mapOptions); useEffect(() => { let protocol = new Protocol(); - maplibregl.addProtocol("pmtiles", protocol.tile); + maplibregl.addProtocol('pmtiles', protocol.tile); return () => { - maplibregl.removeProtocol("pmtiles"); + maplibregl.removeProtocol('pmtiles'); }; }, []); - useEffect(() => { + const fitMapToBounds = () => { if (map.current && mapOptions.bounds) { if (mapOptions.bounds) { map.current.fitBounds(mapOptions.bounds, { @@ -35,11 +32,12 @@ export const MapComponent: React.FC = () => { }); } } - }, [mapOptions]); + }; + useEffect(fitMapToBounds, [mapOptions.bounds]); useEffect(() => { if (map.current || !mapContainer.current) return; - + map.current = new maplibregl.Map({ container: mapContainer.current, style: MAP_OPTIONS.style, @@ -47,16 +45,17 @@ export const MapComponent: React.FC = () => { zoom: MAP_OPTIONS.zoom, maxZoom: MAP_OPTIONS.maxZoom, }); + fitMapToBounds(); map.current.scrollZoom.setWheelZoomRate(1 / 300); map.current.scrollZoom.setZoomRate(1 / 300); map.current.addControl(new maplibregl.NavigationControl()); - map.current.on("load", () => { + map.current.on('load', () => { setMapRef(map); }); - INTERACTIVE_LAYERS.forEach((layer) => { - mapEvents.forEach((action) => { + INTERACTIVE_LAYERS.forEach(layer => { + mapEvents.forEach(action => { if (map.current) { map.current?.on( action.action as keyof MapLayerEventType, @@ -70,8 +69,8 @@ export const MapComponent: React.FC = () => { }); return () => { - mapEvents.forEach((action) => { - map.current?.off(action.action, (e) => { + mapEvents.forEach(action => { + map.current?.off(action.action, e => { action.handler(e, map.current); }); }); @@ -81,7 +80,7 @@ export const MapComponent: React.FC = () => { return (
diff --git a/app/src/app/components/sidebar/ColorPicker.jsx b/app/src/app/components/sidebar/ColorPicker.jsx deleted file mode 100644 index 55a2f4894..000000000 --- a/app/src/app/components/sidebar/ColorPicker.jsx +++ /dev/null @@ -1,90 +0,0 @@ -import React, { useState } from "react"; -import { _colorScheme, colorScheme } from "../../constants/colors"; -import { Button } from "@radix-ui/themes"; -import { styled } from "@stitches/react"; -import * as RadioGroup from "@radix-ui/react-radio-group"; -import { blackA } from "@radix-ui/colors"; -import { useMapStore } from "../../store/mapStore"; - -export function ColorPicker() { - const selectedZone = useMapStore((state) => state.selectedZone); - const setSelectedZone = useMapStore((state) => state.setSelectedZone); - const setZoneAssignments = useMapStore((state) => state.setZoneAssignments); - const accumulatedGeoids = useMapStore((state) => state.accumulatedGeoids); - const resetAccumulatedBlockPopulations = useMapStore((state) => state.resetAccumulatedBlockPopulations); - - const colorArray = colorScheme; - if (!colorArray) return null; - const handleRadioChange = (value) => { - console.log( - "setting accumulated geoids to old zone", - selectedZone, - "new zone is", - value, - ); - setZoneAssignments(selectedZone, accumulatedGeoids); - setSelectedZone(value); - resetAccumulatedBlockPopulations(); - }; - return ( -
- - {colorArray.map((color, i) => ( - - - - ))} - -
- ); -} - -const StyledColorPicker = styled(Button, { - width: 25, - height: 25, - borderRadius: 10, - margin: 5, - "&:selected": { - border: "2px solid", - }, -}); - -const RadioGroupItem = styled(RadioGroup.Item, { - width: 20, - height: 20, - borderRadius: "100%", - "&:hover": { backgroundColor: blackA.blackA4 }, - "&:focus": { boxShadow: `0 0 0 2px black` }, - margin: 2.5, - alignItems: "center", - border: "1px solid #ccc", - borderRadius: "8px", - cursor: "pointer", -}); - -const RadioGroupIndicator = styled(RadioGroup.Indicator, { - // display: "flex", - alignItems: "center", - justifyContent: "center", - width: "100%", - height: "100%", - position: "relative", - textAlign: "-webkit-center", - "&::after": { - content: '""', - display: "block", - width: 7, - height: 7, - borderRadius: "50%", - backgroundColor: "#fff", - }, -}); - -const RadioGroupRoot = styled(RadioGroup.Root, {}); diff --git a/app/src/app/components/sidebar/ColorPicker.tsx b/app/src/app/components/sidebar/ColorPicker.tsx new file mode 100644 index 000000000..39f2b72f7 --- /dev/null +++ b/app/src/app/components/sidebar/ColorPicker.tsx @@ -0,0 +1,134 @@ +import React from 'react'; +import {Button, Checkbox, CheckboxGroup} from '@radix-ui/themes'; +import {styled} from '@stitches/react'; +import * as RadioGroup from '@radix-ui/react-radio-group'; +import {blackA} from '@radix-ui/colors'; + +type ColorPickerProps = T extends true + ? { + defaultValue: number[]; + value?: number[]; + onValueChange: (indices: number[], color: string[]) => void; + colorArray: string[]; + multiple: true; + } + : { + defaultValue: number; + value?: number; + onValueChange: (i: number, color: string) => void; + colorArray: string[]; + multiple?: false; + }; + +export const ColorPicker = ({ + defaultValue, + value, + onValueChange, + colorArray, + multiple, +}: ColorPickerProps) => { + if (multiple) { + return ( +
+ colorArray[i])} + value={value?.map(i => colorArray[i]) || []} + onValueChange={values => { + const indices = values.map(f => colorArray.indexOf(f)); + onValueChange(indices, values); + }} + > + {colorArray.map((color, i) => ( + + {/* */} + + ))} + +
+ ); + } + + return ( +
+ { + const index = colorArray.indexOf(value); + if (index !== -1) onValueChange(index, value); + }} + value={value !== undefined ? colorArray[value] : undefined} + defaultValue={colorArray[defaultValue]} + > + {colorArray.map((color, i) => ( + + + + ))} + +
+ ); +}; + +const StyledColorPicker = styled(Button, { + width: 25, + height: 25, + borderRadius: 10, + margin: 5, + '&:selected': { + border: '2px solid', + }, +}); + +const groupItemCSS = { + width: 20, + height: 20, + '&:hover': {backgroundColor: blackA.blackA4}, + '&:focus': {boxShadow: `0 0 0 2px black`}, + margin: 2.5, + alignItems: 'center', + border: '1px solid #ccc', + borderRadius: '8px', + cursor: 'pointer', +}; +const RadioGroupItem = styled(RadioGroup.Item, groupItemCSS); +const CheckboxGroupItem = styled(CheckboxGroup.Item, { + ...groupItemCSS, + margin: 0.5, + backgroundColor: 'var(--accent-indicator)', + '& svg': { + backgroundColor: 'var(--accent-indicator)', + }, +}); + +const groupIndicatorCSS = { + // display: "flex", + alignItems: 'center', + justifyContent: 'center', + width: '100%', + height: '100%', + position: 'relative', + textAlign: '-webkit-center', + '&::after': { + content: '""', + display: 'block', + width: 7, + height: 7, + borderRadius: '50%', + backgroundColor: '#fff', + }, +}; +const RadioGroupIndicator = styled(RadioGroup.Indicator, groupIndicatorCSS); +// const CheckBoxGroupIndicator = styled(Checkbox.., groupIndicatorCSS); + +const groupRootCSS = {}; +const RadioGroupRoot = styled(RadioGroup.Root, groupRootCSS); +const CheckboxGroupRoot = styled(CheckboxGroup.Root, { + ...groupRootCSS, + flexDirection: 'row', + flexWrap: 'wrap', + justifyContent: 'center', +}); diff --git a/app/src/app/components/sidebar/DataPanels.tsx b/app/src/app/components/sidebar/DataPanels.tsx index 1bae09492..ec16ce4ec 100644 --- a/app/src/app/components/sidebar/DataPanels.tsx +++ b/app/src/app/components/sidebar/DataPanels.tsx @@ -1,13 +1,13 @@ -import { Box, Flex, Heading } from "@radix-ui/themes"; -import { MapModeSelector } from "./MapModeSelector"; -import { ColorPicker } from "./ColorPicker"; -import { ResetMapButton } from "./ResetMapButton"; -import { GerryDBViewSelector } from "./GerryDBViewSelector"; -import { HorizontalBar } from "./charts/HorizontalBarChart"; -import { useMapStore } from "@/app/store/mapStore"; -import { Tabs, Text } from "@radix-ui/themes"; -import Layers from "./Layers"; -import React from "react"; +import {Box, Flex, Heading} from '@radix-ui/themes'; +import {MapModeSelector} from './MapModeSelector'; +import {ColorPicker} from './ColorPicker'; +import {ResetMapButton} from './ResetMapButton'; +import {GerryDBViewSelector} from './GerryDBViewSelector'; +import {HorizontalBar} from './charts/HorizontalBarChart'; +import {useMapStore} from '@/app/store/mapStore'; +import {Tabs, Text} from '@radix-ui/themes'; +import Layers from './Layers'; +import React from 'react'; interface DataPanelSpec { title: string; @@ -23,18 +23,18 @@ interface DataPanelsProps { const defaultPanels: DataPanelSpec[] = [ { - title: "population", - label: "Population", + title: 'population', + label: 'Population', content: , }, { - title: "layers", - label: "Data layers", + title: 'layers', + label: 'Data layers', content: , }, { - title: "evaluation", - label: "Evaluation", + title: 'evaluation', + label: 'Evaluation', content: Unimplemented , }, ]; @@ -46,14 +46,14 @@ const DataPanels: React.FC = ({ return ( - {panels.map((panel) => ( + {panels.map(panel => ( {panel.label} ))} - {panels.map((panel) => ( + {panels.map(panel => ( {panel.content} diff --git a/app/src/app/components/sidebar/ExitBlockViewButtons.tsx b/app/src/app/components/sidebar/ExitBlockViewButtons.tsx new file mode 100644 index 000000000..f32ea1c13 --- /dev/null +++ b/app/src/app/components/sidebar/ExitBlockViewButtons.tsx @@ -0,0 +1,18 @@ +import {useMapStore} from '@/app/store/mapStore'; +import {Button} from '@radix-ui/themes'; + +export function ExitBlockViewButtons() { + const captiveIds = useMapStore(store => store.captiveIds); + const exitBlockView = useMapStore(store => store.exitBlockView); + + return captiveIds.size ? ( + <> + + + + ) : null; +} diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index f96b4ed4c..adab03804 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -22,6 +22,10 @@ export default function Layers() { const mapDocument = useMapStore(state => state.mapDocument); const visibleLayerIds = useMapStore(state => state.visibleLayerIds); const updateVisibleLayerIds = useMapStore(state => state.updateVisibleLayerIds); + const toggleHighlightBrokenDistricts = useMapStore(state => state.toggleHighlightBrokenDistricts); + const toggleLockAllAreas = useMapStore(state => state.toggleLockAllAreas); + const parentsAreBroken = useMapStore(state => state.shatterIds.parents.size); + const mapOptions = useMapStore(state => state.mapOptions); const toggleLayers = (layerIds: string[]) => { if (!mapRef) return; @@ -37,7 +41,11 @@ export default function Layers() { Show numbering for painted districts + toggleHighlightBrokenDistricts()} + > + Highlight Broken Voter Districts + + toggleLockAllAreas()}> + Lock All Painted Areas + Boundaries diff --git a/app/src/app/components/sidebar/MapModeSelector.jsx b/app/src/app/components/sidebar/MapModeSelector.jsx index d24677e8c..032829126 100644 --- a/app/src/app/components/sidebar/MapModeSelector.jsx +++ b/app/src/app/components/sidebar/MapModeSelector.jsx @@ -1,35 +1,42 @@ -import React from "react"; -import * as RadioGroup from "@radix-ui/react-radio-group"; -import { styled } from "@stitches/react"; -import { useMapStore } from "@store/mapStore"; -import { RadioCards, Box } from "@radix-ui/themes"; +import React from 'react'; +import * as RadioGroup from '@radix-ui/react-radio-group'; +import {styled} from '@stitches/react'; +import {useMapStore} from '@store/mapStore'; +import {RadioCards, Box} from '@radix-ui/themes'; import { EraserIcon, Pencil2Icon, HandIcon, - BorderSplitIcon, -} from "@radix-ui/react-icons"; -import { RecentMapsModal } from "@components/sidebar/RecentMapsModal"; + LockOpen1Icon, + ViewGridIcon, +} from '@radix-ui/react-icons'; +import {RecentMapsModal} from '@components/sidebar/RecentMapsModal'; export function MapModeSelector() { - const activeTool = useMapStore((state) => state.activeTool); - const setActiveTool = useMapStore((state) => state.setActiveTool); - const mapDocument = useMapStore((state) => state.mapDocument); + const activeTool = useMapStore(state => state.activeTool); + const setActiveTool = useMapStore(state => state.setActiveTool); + const mapDocument = useMapStore(state => state.mapDocument); if (!activeTool) return null; const activeTools = [ - { mode: "pan", disabled: false, label: "Pan", icon: }, - { mode: "brush", disabled: false, label: "Brush", icon: }, - { mode: "eraser", disabled: false, label: "Erase", icon: }, + {mode: 'pan', disabled: false, label: 'Pan', icon: }, + {mode: 'brush', disabled: false, label: 'Brush', icon: }, + {mode: 'eraser', disabled: false, label: 'Erase', icon: }, { - mode: "shatter", + mode: 'shatter', disabled: !mapDocument?.child_layer, - label: "Shatter", - icon: , + label: 'Break', + icon: , + }, + { + mode: 'lock', + disabled: false, + label: 'Lock', + icon: , }, ]; - const handleRadioChange = (value) => { + const handleRadioChange = value => { setActiveTool(value); }; @@ -41,13 +48,9 @@ export function MapModeSelector() { onValueChange={handleRadioChange} columns={{ initial: "3" }} > - {activeTools.map((tool) => ( + {activeTools.map(tool => ( - + {tool.icon} {tool.label} @@ -60,18 +63,18 @@ export function MapModeSelector() { } const RadioGroupRoot = styled(RadioGroup.Root, { - display: "grid", - flexDirection: "column", + display: 'grid', + flexDirection: 'column', gap: 10, }); const RadioGroupItem = styled(RadioGroup.Item, { - display: "grid", - alignItems: "center", - padding: "1rem", - border: "1px solid #ccc", - borderRadius: "8px", - cursor: "pointer", + display: 'grid', + alignItems: 'center', + padding: '1rem', + border: '1px solid #ccc', + borderRadius: '8px', + cursor: 'pointer', }); -const Flex = styled("div", { display: "grid" }); +const Flex = styled('div', {display: 'grid'}); diff --git a/app/src/app/components/sidebar/MobileColorPicker.jsx b/app/src/app/components/sidebar/MobileColorPicker.jsx index 9f88ef58e..25077e398 100644 --- a/app/src/app/components/sidebar/MobileColorPicker.jsx +++ b/app/src/app/components/sidebar/MobileColorPicker.jsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; import { Button, IconButton, Popover } from "@radix-ui/themes"; // Import Popover from Radix -import { ColorPicker } from "./ColorPicker"; +import { ZonePicker } from "./ZonePicker"; import { colorScheme } from "@/app/constants/colors"; import { useMapStore } from "@/app/store/mapStore"; import { ColorWheelIcon } from "@radix-ui/react-icons"; @@ -25,7 +25,7 @@ export const MobileColorPicker = () => { - + ); diff --git a/app/src/app/components/sidebar/MobileTopNav.tsx b/app/src/app/components/sidebar/MobileTopNav.tsx index e438cbc0c..9598662ca 100644 --- a/app/src/app/components/sidebar/MobileTopNav.tsx +++ b/app/src/app/components/sidebar/MobileTopNav.tsx @@ -1,32 +1,28 @@ -import { Box, Button, Flex, Heading } from "@radix-ui/themes"; -import React, { useRef, useState } from "react"; -import { Cross2Icon, GearIcon } from "@radix-ui/react-icons"; -import DataPanels from "./DataPanels"; +import {Box, Button, Flex, Heading} from '@radix-ui/themes'; +import React, {useRef, useState} from 'react'; +import {Cross2Icon, GearIcon} from '@radix-ui/react-icons'; +import DataPanels from './DataPanels'; const MobileTopNav = () => { const [dataPanelOpen, setDataPanelOpen] = useState(false); - const handleToggleDataPanel = () => setDataPanelOpen((prev) => !prev); + const handleToggleDataPanel = () => setDataPanelOpen(prev => !prev); const boxRef = useRef(null); - const topBarHeight = - boxRef.current?.getClientRects()?.[0]?.height || 44.90625; - const isLandscape = typeof window !== 'undefined' && window.matchMedia("(orientation: landscape)").matches; + const topBarHeight = boxRef.current?.getClientRects()?.[0]?.height || 44.90625; + const isLandscape = + typeof window !== 'undefined' && window.matchMedia('(orientation: landscape)').matches; return ( - - + + Districtr { state.selectedZone); - const setSelectedZone = useMapStore((state) => state.setSelectedZone); - const setZoneAssignments = useMapStore((state) => state.setZoneAssignments); - const accumulatedGeoids = useMapStore((state) => state.accumulatedGeoids); + const selectedZone = useMapStore(state => state.selectedZone); + const setSelectedZone = useMapStore(state => state.setSelectedZone); + const setZoneAssignments = useMapStore(state => state.setZoneAssignments); + const accumulatedGeoids = useMapStore(state => state.accumulatedGeoids); - const handlePickerValueChange = (value) => { - console.log( - "setting accumulated geoids to old zone", - selectedZone, - "new zone is", - value - ); + const handlePickerValueChange = value => { + console.log('setting accumulated geoids to old zone', selectedZone, 'new zone is', value); setZoneAssignments(selectedZone, accumulatedGeoids); setSelectedZone(value); }; // to be refactored const options = [ - { value: 1, label: "First Zone" }, - { value: 2, label: "Second Zone" }, - { value: 3, label: "Third Zone" }, + {value: 1, label: 'First Zone'}, + {value: 2, label: 'Second Zone'}, + {value: 3, label: 'Third Zone'}, ]; return ( diff --git a/app/src/app/components/sidebar/RecentMapsModal.tsx b/app/src/app/components/sidebar/RecentMapsModal.tsx index 294749539..e09432654 100644 --- a/app/src/app/components/sidebar/RecentMapsModal.tsx +++ b/app/src/app/components/sidebar/RecentMapsModal.tsx @@ -14,13 +14,13 @@ import { } from '@radix-ui/themes'; import {usePathname, useSearchParams, useRouter} from 'next/navigation'; import {DocumentObject} from '../../utils/api/apiHandlers'; -import { styled } from '@stitches/react'; +import {styled} from '@stitches/react'; type NamedDocumentObject = DocumentObject & {name?: string}; const DialogContentContainer = styled(Dialog.Content, { - maxWidth: "calc(100vw - 2rem)", - maxHeight: "calc(100vh-2rem)" -}) + maxWidth: 'calc(100vw - 2rem)', + maxHeight: 'calc(100vh-2rem)', +}); export const RecentMapsModal = () => { const router = useRouter(); @@ -28,7 +28,7 @@ export const RecentMapsModal = () => { const searchParams = useSearchParams(); const mapDocument = useMapStore(store => store.mapDocument); const userMaps = useMapStore(store => store.userMaps); - const upcertUserMap = useMapStore(store => store.upcertUserMap); + const upsertUserMap = useMapStore(store => store.upsertUserMap); const setMapDocument = useMapStore(store => store.setMapDocument); const [dialogOpen, setDialogOpen] = React.useState(false); @@ -80,7 +80,7 @@ export const RecentMapsModal = () => { key={i} active={mapDocument?.document_id === userMap.document_id} onChange={userMapData => - upcertUserMap({ + upsertUserMap({ userMapData, userMapDocumentId: userMap.document_id, }) diff --git a/app/src/app/components/sidebar/ResetMapButton.tsx b/app/src/app/components/sidebar/ResetMapButton.tsx index 14a811fe3..37d1e5481 100644 --- a/app/src/app/components/sidebar/ResetMapButton.tsx +++ b/app/src/app/components/sidebar/ResetMapButton.tsx @@ -2,8 +2,8 @@ import {useMapStore} from '@/app/store/mapStore'; import {Button} from '@radix-ui/themes'; export function ResetMapButton() { - const handleClickResetMap = useMapStore(state => state.handleReset) - const noZonesAreAssigned = useMapStore(state => !state.zoneAssignments.size) + const handleClickResetMap = useMapStore(state => state.handleReset); + const noZonesAreAssigned = useMapStore(state => !state.zoneAssignments.size); return ( + + + + + + Reset Map + + Are you sure? This will reset all zone assignments and broken geographies. Resetting your + map cannot be undone. + + + + + + + + + + + + ); } From 0bb21e69cbec61fefd8882c16d1277714a98f667 Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 14 Nov 2024 07:58:01 -0600 Subject: [PATCH 33/50] Fix logic for checking individual locked areas (#177) --- app/src/app/components/sidebar/ZonePicker.tsx | 4 +-- app/src/app/store/mapRenderSubs.ts | 32 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/app/src/app/components/sidebar/ZonePicker.tsx b/app/src/app/components/sidebar/ZonePicker.tsx index 0d71aaa92..e9915bc22 100644 --- a/app/src/app/components/sidebar/ZonePicker.tsx +++ b/app/src/app/components/sidebar/ZonePicker.tsx @@ -19,9 +19,9 @@ export function ZonePicker() { const handleRadioChange = (index: number, _color: string) => { const value = index + 1; console.log('setting accumulated geoids to old zone', selectedZone, 'new zone is', value); - setZoneAssignments(selectedZone, accumulatedGeoids); + // setZoneAssignments(selectedZone, accumulatedGeoids); setSelectedZone(value); - resetAccumulatedBlockPopulations(); + // resetAccumulatedBlockPopulations(); }; return ( diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 09bc00067..4d22b5599 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -102,8 +102,14 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ], (curr, prev) => { colorZoneAssignments(curr, prev); - const {captiveIds, shatterIds, getMapRef, setLockedFeatures, mapRenderingState} = - useMapStore.getState(); + const { + captiveIds, + shatterIds, + getMapRef, + setLockedFeatures, + lockedFeatures, + mapRenderingState, + } = useMapStore.getState(); const mapRef = getMapRef(); if (!mapRef || mapRenderingState !== 'loaded') return; [...PARENT_LAYERS, ...CHILD_LAYERS].forEach(layerId => { @@ -120,6 +126,8 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ); }); const [lockPaintedAreas, prevLockPaintedAreas] = [curr[6], prev[6]]; + const sameLockedAreas = + JSON.stringify(lockPaintedAreas) === JSON.stringify(prevLockPaintedAreas); const zoneAssignments = curr[0]; // if lockPaintedAreas, lock all zones if (lockPaintedAreas === true) { @@ -128,15 +136,29 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { .filter(([key, value]) => value !== null) .map(([key]) => key) ); - setLockedFeatures(new Set(nonNullZones)); + setLockedFeatures(nonNullZones); // now unlocked, was previously locked } else if (Array.isArray(lockPaintedAreas)) { + const previousWasArray = Array.isArray(prevLockPaintedAreas); const nonNullZones = new Set( [...zoneAssignments.entries()] - .filter(([key, value]) => lockPaintedAreas.includes(value)) + .filter( + ([key, value]) => + // locked zones include assignment zone + lockPaintedAreas.includes(value) || + // locked zones are the same, and this individual feature was previously locked + (sameLockedAreas && lockedFeatures.has(key)) || + // locked zones are changed, BUT this individual feature is not in a zone + // that was previously locked + (!sameLockedAreas && + previousWasArray && + !lockPaintedAreas.includes(value) && + !prevLockPaintedAreas.includes(value) && + lockedFeatures.has(key)) + ) .map(([key]) => key) ); - setLockedFeatures(new Set(nonNullZones)); + setLockedFeatures(nonNullZones); } else if (!lockPaintedAreas && prevLockPaintedAreas) { setLockedFeatures(new Set()); } From 90da1b734e1646463fe253e58d223ad1777a686f Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 14 Nov 2024 07:59:02 -0600 Subject: [PATCH 34/50] Highlight unassigned geographies (#173) --- app/src/app/components/sidebar/Layers.tsx | 7 +++ app/src/app/constants/layers.ts | 69 ++++++++++++++--------- app/src/app/store/mapRenderSubs.ts | 28 ++++++++- app/src/app/store/mapStore.ts | 7 +++ app/src/app/store/types.ts | 1 + 5 files changed, 85 insertions(+), 27 deletions(-) diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index adab03804..4f3620fe2 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -26,6 +26,7 @@ export default function Layers() { const toggleLockAllAreas = useMapStore(state => state.toggleLockAllAreas); const parentsAreBroken = useMapStore(state => state.shatterIds.parents.size); const mapOptions = useMapStore(state => state.mapOptions); + const setMapOptions = useMapStore(state => state.setMapOptions); const toggleLayers = (layerIds: string[]) => { if (!mapRef) return; @@ -45,6 +46,7 @@ export default function Layers() { visibleLayerIds.includes(BLOCK_LAYER_ID) ? '1' : '', parentsAreBroken && mapOptions.showBrokenDistricts ? '3' : '', mapOptions.lockPaintedAreas === true ? '4' : '', + mapOptions.higlightUnassigned === true ? 'higlightUnassigned' : '' ]} > Highlight Broken Voter Districts + setMapOptions({ + higlightUnassigned: !mapOptions.higlightUnassigned + })}> + Highlight Unassigned Districts + toggleLockAllAreas()}> Lock All Painted Areas diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index cc7504ed0..23de6eb20 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -3,6 +3,7 @@ import { ExpressionSpecification, FilterSpecification, LayerSpecification, + LineLayerSpecification, } from 'maplibre-gl'; import {Map} from 'maplibre-gl'; import {getBlocksSource} from './sources'; @@ -13,6 +14,7 @@ import {colorScheme} from './colors'; export const BLOCK_SOURCE_ID = 'blocks'; export const BLOCK_LAYER_ID = 'blocks'; export const BLOCK_LAYER_ID_HIGHLIGHT = BLOCK_LAYER_ID + '-highlight'; +export const BLOCK_LAYER_ID_HIGHLIGHT_CHILD = BLOCK_LAYER_ID + '-highlight-child'; export const BLOCK_LAYER_ID_CHILD = 'blocks-child'; export const BLOCK_HOVER_LAYER_ID = `${BLOCK_LAYER_ID}-hover`; export const BLOCK_HOVER_LAYER_ID_CHILD = `${BLOCK_LAYER_ID_CHILD}-hover`; @@ -21,7 +23,7 @@ export const INTERACTIVE_LAYERS = [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CH export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; -export const CHILD_LAYERS = [BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD]; +export const CHILD_LAYERS = [BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD, BLOCK_LAYER_ID_HIGHLIGHT_CHILD]; export const DEFAULT_PAINT_STYLE: ExpressionSpecification = [ 'case', @@ -66,6 +68,9 @@ export function getLayerFill( ): DataDrivenPropertyValueSpecification { const innerFillSpec = [ 'case', + // is broken parent + ['boolean', ['feature-state', 'broken'], false], + 0, // geography is locked ['boolean', ['feature-state', 'locked'], false], 0.35, @@ -124,8 +129,9 @@ export function getLayerFill( } export function getHighlightLayerSpecification( sourceLayer: string, - layerId: string -): LayerSpecification { + layerId: string, + highlightUnassgned?: boolean +): LineLayerSpecification { return { id: layerId, source: BLOCK_SOURCE_ID, @@ -143,15 +149,29 @@ export function getHighlightLayerSpecification( '#000000', // Black color when focused ['boolean', ['feature-state', 'highlighted'], false], '#e5ff00', // yellow color when highlighted + ['boolean', ['feature-state', 'highlighted'], false], + '#e5ff00', // yellow color when highlighted + // @ts-ignore right behavior, wrong types + ['==', ['feature-state', 'zone'], null], + '#FF0000', // optionally red color when zone is not assigned '#000000', // Default color ], 'line-width': [ 'case', - ['boolean', ['feature-state', 'focused'], false], - 5, // Width of 5 when focused - ['boolean', ['feature-state', 'highlighted'], false], - 5, // Width of 5 when highlighted - 0, // Default width + ['boolean', ['feature-state', 'broken'], false], + 0, // none when broken parent + [ + 'any', + ['boolean', ['feature-state', 'focused'], false], + ['boolean', ['feature-state', 'highlighted'], false], + ['all', + // @ts-ignore correct logic, wrong types + ['==', ['feature-state', 'zone'], null], + ['boolean', !!highlightUnassgned] + ] + ], + 3.5, + 0, // Default width if none of the conditions are met ], }, }; @@ -229,6 +249,10 @@ const addBlockLayers = (map: Map | null, mapDocument: DocumentObject) => { getBlocksHoverLayerSpecification(mapDocument.child_layer, BLOCK_HOVER_LAYER_ID_CHILD), LABELS_BREAK_LAYER_ID ); + map?.addLayer( + getHighlightLayerSpecification(mapDocument.child_layer, BLOCK_LAYER_ID_HIGHLIGHT_CHILD), + LABELS_BREAK_LAYER_ID + ); } map?.addLayer(getHighlightLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID_HIGHLIGHT)); useMapStore.getState().setMapRenderingState('loaded'); @@ -245,24 +269,17 @@ export function removeBlockLayers(map: Map | null) { return; } useMapStore.getState().setMapRenderingState('loading'); - if (map.getLayer(BLOCK_LAYER_ID)) { - map.removeLayer(BLOCK_LAYER_ID); - } - if (map.getLayer(BLOCK_LAYER_ID_HIGHLIGHT)) { - map.removeLayer(BLOCK_LAYER_ID_HIGHLIGHT); - } - if (map.getLayer(BLOCK_HOVER_LAYER_ID)) { - map.removeLayer(BLOCK_HOVER_LAYER_ID); - } - if (map.getLayer(BLOCK_LAYER_ID_CHILD)) { - map.removeLayer(BLOCK_LAYER_ID_CHILD); - } - if (map.getLayer(BLOCK_HOVER_LAYER_ID_CHILD)) { - map.removeLayer(BLOCK_HOVER_LAYER_ID_CHILD); - } - if (map.getSource(BLOCK_SOURCE_ID)) { - map.removeSource(BLOCK_SOURCE_ID); - } + [ + BLOCK_LAYER_ID, + BLOCK_LAYER_ID_HIGHLIGHT, + BLOCK_HOVER_LAYER_ID, + BLOCK_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_LAYER_ID_HIGHLIGHT_CHILD, + BLOCK_SOURCE_ID, + ].forEach(layer => { + map.getLayer(layer) && map.removeLayer(layer); + }); } export {addBlockLayers}; diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 4d22b5599..2a1a98b52 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -8,6 +8,9 @@ import { getLayerFilter, getLayerFill, BLOCK_SOURCE_ID, + BLOCK_LAYER_ID_HIGHLIGHT, + getHighlightLayerSpecification, + BLOCK_LAYER_ID_HIGHLIGHT_CHILD, } from '../constants/layers'; import { ColorZoneAssignmentsState, @@ -56,10 +59,12 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ); // remove zone from parents shatterIds.parents.forEach(id => { - mapRef?.removeFeatureState({ + mapRef?.setFeatureState({ source: BLOCK_SOURCE_ID, id, sourceLayer: mapDocument?.parent_layer, + }, { + broken: true }); }); @@ -278,6 +283,26 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { }); } ); + + const highlightUnassignedSub = useMapStore.subscribe( + state => state.mapOptions.higlightUnassigned, + (higlightUnassigned) => { + const {getMapRef, mapDocument} = useMapStore.getState(); + const mapRef = getMapRef(); + if (!mapRef || !mapDocument?.parent_layer) return; + // set the layer BLOCK_LAYER_ID_HIGHLIGHT style to be the return from getHighlightLayerSpecification + const paintStyle = getHighlightLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID_HIGHLIGHT, higlightUnassigned)['paint'] + if (!paintStyle) return + if(mapRef.getLayer(BLOCK_LAYER_ID_HIGHLIGHT)){ + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT, 'line-width', paintStyle['line-width']); + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT, 'line-color', paintStyle['line-color']); + } + if(mapRef.getLayer(BLOCK_LAYER_ID_HIGHLIGHT_CHILD)){ + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT_CHILD, 'line-width', paintStyle['line-width']); + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT_CHILD, 'line-color', paintStyle['line-color']); + } + } + ); return [ addLayerSubMapDocument, _shatterMapSideEffectRender, @@ -285,5 +310,6 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { _zoneAssignmentMapSideEffectRender, _updateMapCursor, _applyFocusFeatureState, + highlightUnassignedSub, ]; }; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 6806d1931..9c48b8c50 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -551,6 +551,13 @@ export const useMapStore = create( delete shatterMappings[parent.parentId]; newShatterIds.parents.delete(parent.parentId); newZoneAssignments.set(parent.parentId, parent.zone!); + mapRef?.setFeatureState({ + source: BLOCK_SOURCE_ID, + id: parent.parentId, + sourceLayer: mapDocument?.parent_layer, + }, { + broken: false + }); }); set({ diff --git a/app/src/app/store/types.ts b/app/src/app/store/types.ts index 81e76c018..47ad70c55 100644 --- a/app/src/app/store/types.ts +++ b/app/src/app/store/types.ts @@ -2,6 +2,7 @@ import {NullableZone} from '../constants/types'; export type DistrictrMapOptions = { showBrokenDistricts?: boolean; + higlightUnassigned?: boolean; lockPaintedAreas: boolean | Array; mode: 'default' | 'break'; }; From 8cbb3254b0a11e9974b32dedf0508f2cde76ee0f Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 14 Nov 2024 08:54:10 -0600 Subject: [PATCH 35/50] Highlight bugfix (#179) --- app/src/app/constants/layers.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 23de6eb20..ca992f8f6 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -130,7 +130,7 @@ export function getLayerFill( export function getHighlightLayerSpecification( sourceLayer: string, layerId: string, - highlightUnassgned?: boolean + highlightUnassigned?: boolean ): LineLayerSpecification { return { id: layerId, @@ -158,8 +158,6 @@ export function getHighlightLayerSpecification( ], 'line-width': [ 'case', - ['boolean', ['feature-state', 'broken'], false], - 0, // none when broken parent [ 'any', ['boolean', ['feature-state', 'focused'], false], @@ -167,7 +165,8 @@ export function getHighlightLayerSpecification( ['all', // @ts-ignore correct logic, wrong types ['==', ['feature-state', 'zone'], null], - ['boolean', !!highlightUnassgned] + ['boolean', !!highlightUnassigned], + ['!', ['boolean', ['feature-state', 'broken'], false]], ] ], 3.5, From ee0e86a6e1d42bd9b292d67cc9c9834a5bf07e7b Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 14 Nov 2024 16:00:18 -0600 Subject: [PATCH 36/50] Bugfix - remove source (#180) --- app/src/app/constants/layers.ts | 28 +++++++++++++++------------- app/src/app/store/mapStore.ts | 21 +++++++++++++++------ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index ca992f8f6..93461404b 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -23,7 +23,11 @@ export const INTERACTIVE_LAYERS = [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CH export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; -export const CHILD_LAYERS = [BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD, BLOCK_LAYER_ID_HIGHLIGHT_CHILD]; +export const CHILD_LAYERS = [ + BLOCK_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_LAYER_ID_HIGHLIGHT_CHILD, +]; export const DEFAULT_PAINT_STYLE: ExpressionSpecification = [ 'case', @@ -66,11 +70,11 @@ export function getLayerFill( captiveIds?: Set, shatterIds?: Set ): DataDrivenPropertyValueSpecification { - const innerFillSpec = [ + const innerFillSpec = ([ 'case', // is broken parent ['boolean', ['feature-state', 'broken'], false], - 0, + 0, // geography is locked ['boolean', ['feature-state', 'locked'], false], 0.35, @@ -108,7 +112,7 @@ export function getLayerFill( ['boolean', ['feature-state', 'hover'], false], 0.6, 0.2, - ] as unknown as DataDrivenPropertyValueSpecification; + ] as unknown) as DataDrivenPropertyValueSpecification; if (captiveIds?.size) { return [ 'case', @@ -162,12 +166,13 @@ export function getHighlightLayerSpecification( 'any', ['boolean', ['feature-state', 'focused'], false], ['boolean', ['feature-state', 'highlighted'], false], - ['all', + [ + 'all', // @ts-ignore correct logic, wrong types ['==', ['feature-state', 'zone'], null], ['boolean', !!highlightUnassigned], ['!', ['boolean', ['feature-state', 'broken'], false]], - ] + ], ], 3.5, 0, // Default width if none of the conditions are met @@ -255,12 +260,6 @@ const addBlockLayers = (map: Map | null, mapDocument: DocumentObject) => { } map?.addLayer(getHighlightLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID_HIGHLIGHT)); useMapStore.getState().setMapRenderingState('loaded'); - - // update map bounds based on document extent - useMapStore.getState().setMapOptions({ - bounds: mapDocument.extent as [number, number, number, number], - container: useMapStore.getState().mapOptions.container, - }); }; export function removeBlockLayers(map: Map | null) { @@ -275,10 +274,13 @@ export function removeBlockLayers(map: Map | null) { BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD, BLOCK_LAYER_ID_HIGHLIGHT_CHILD, - BLOCK_SOURCE_ID, ].forEach(layer => { map.getLayer(layer) && map.removeLayer(layer); }); + + [BLOCK_SOURCE_ID].forEach(source => { + map.getSource(source) && map.removeSource(source); + }); } export {addBlockLayers}; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 9c48b8c50..19cabb0ce 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -358,17 +358,26 @@ export const useMapStore = create( setMapViews: mapViews => set({mapViews}), mapDocument: null, setMapDocument: mapDocument => { - const currentMapDocument = get().mapDocument; + const { + mapDocument: currentMapDocument, + setFreshMap, + resetZoneAssignments, + upsertUserMap, + mapOptions + } = get(); if (currentMapDocument?.document_id === mapDocument.document_id) { return; } - get().setFreshMap(true); - get().resetZoneAssignments(); - get().upsertUserMap({ - mapDocument, - }); + setFreshMap(true) + resetZoneAssignments() + upsertUserMap({mapDocument}) + set({ mapDocument: mapDocument, + mapOptions: { + ...mapOptions, + bounds: mapDocument.extent + }, shatterIds: {parents: new Set(), children: new Set()}, }); }, From 9968de21f5fc987d569b0ddeff748d48f0724e25 Mon Sep 17 00:00:00 2001 From: Anna Bailliekova Date: Mon, 18 Nov 2024 10:11:56 -0600 Subject: [PATCH 37/50] Fixes #30 Summary stats metadata endpoint (#87) Co-authored-by: Raphael Paul Laude Co-authored-by: nofurtherinformation --- app/package-lock.json | 23 +- app/package.json | 4 +- app/src/app/components/sidebar/DataPanels.tsx | 12 +- app/src/app/components/sidebar/Evaluation.tsx | 325 ++++++++++++++++++ app/src/app/store/mapStore.ts | 51 ++- app/src/app/store/metricsSubs.ts | 3 +- app/src/app/utils/api/apiHandlers.ts | 106 +++++- app/src/app/utils/api/mutations.ts | 3 + app/src/app/utils/api/queries.ts | 51 ++- app/src/app/utils/numbers.ts | 33 ++ app/src/app/utils/summaryStats.ts | 31 ++ ...a_create_summary_stat_metadata_endpoint.py | 30 ++ ...d35_add_available_stats_to_districtrmap.py | 50 +++ .../f86991e63a62_summary_stat_udfs.py | 39 +++ backend/app/main.py | 101 ++++++ backend/app/models.py | 54 ++- .../app/sql/available_summary_stat_udf.sql | 82 +++++ backend/app/sql/summary_stats_p1.sql | 42 +++ backend/app/sql/summary_stats_p1_totals.sql | 39 +++ backend/app/sql/summary_stats_p4.sql | 44 +++ backend/app/sql/summary_stats_p4_totals.sql | 41 +++ backend/app/utils.py | 68 +++- backend/cli.py | 26 ++ backend/load_data.py | 4 +- ...o_view_census_blocks_summary_stats.geojson | 17 + ...iew_census_blocks_summary_stats_p4.geojson | 17 + backend/tests/test_main.py | 181 +++++++++- backend/tests/test_utils.py | 49 +++ 28 files changed, 1481 insertions(+), 45 deletions(-) create mode 100644 app/src/app/components/sidebar/Evaluation.tsx create mode 100644 app/src/app/utils/numbers.ts create mode 100644 app/src/app/utils/summaryStats.ts create mode 100644 backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py create mode 100644 backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py create mode 100644 backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py create mode 100644 backend/app/sql/available_summary_stat_udf.sql create mode 100644 backend/app/sql/summary_stats_p1.sql create mode 100644 backend/app/sql/summary_stats_p1_totals.sql create mode 100644 backend/app/sql/summary_stats_p4.sql create mode 100644 backend/app/sql/summary_stats_p4_totals.sql create mode 100644 backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson create mode 100644 backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson diff --git a/app/package-lock.json b/app/package-lock.json index 0ffe56b48..4d1b8d6a9 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -19,6 +19,7 @@ "@turf/bbox": "^7.1.0", "@turf/helpers": "^7.1.0", "axios": "^1.7.2", + "d3-scale-chromatic": "^3.1.0", "idb-keyval": "^6.2.1", "lodash": "^4.17.21", "maplibre-gl": "^4.4.1", @@ -35,6 +36,7 @@ "@flydotio/dockerfile": "^0.5.8", "@testing-library/jest-dom": "^6.4.6", "@testing-library/react": "^16.0.0", + "@types/d3-scale-chromatic": "^3.0.3", "@types/lodash": "^4.17.5", "@types/node": "^20", "@types/react": "^18", @@ -4403,7 +4405,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/bbox/-/bbox-7.1.0.tgz", "integrity": "sha512-PdWPz9tW86PD78vSZj2fiRaB8JhUHy6piSa/QXb83lucxPK+HTAdzlDQMTKj5okRCU8Ox/25IR2ep9T8NdopRA==", - "license": "MIT", "dependencies": { "@turf/helpers": "^7.1.0", "@turf/meta": "^7.1.0", @@ -4418,7 +4419,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/helpers/-/helpers-7.1.0.tgz", "integrity": "sha512-dTeILEUVeNbaEeoZUOhxH5auv7WWlOShbx7QSd4s0T4Z0/iz90z9yaVCtZOLbU89umKotwKaJQltBNO9CzVgaQ==", - "license": "MIT", "dependencies": { "@types/geojson": "^7946.0.10", "tslib": "^2.6.2" @@ -4431,7 +4431,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/meta/-/meta-7.1.0.tgz", "integrity": "sha512-ZgGpWWiKz797Fe8lfRj7HKCkGR+nSJ/5aKXMyofCvLSc2PuYJs/qyyifDPWjASQQCzseJ7AlF2Pc/XQ/3XkkuA==", - "license": "MIT", "dependencies": { "@turf/helpers": "^7.1.0", "@types/geojson": "^7946.0.10" @@ -4532,6 +4531,12 @@ "@types/d3-time": "*" } }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.3.tgz", + "integrity": "sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==", + "dev": true + }, "node_modules/@types/d3-shape": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz", @@ -6171,6 +6176,18 @@ "node": ">=12" } }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", diff --git a/app/package.json b/app/package.json index a61167bbb..90abcf044 100644 --- a/app/package.json +++ b/app/package.json @@ -21,6 +21,7 @@ "@turf/bbox": "^7.1.0", "@turf/helpers": "^7.1.0", "axios": "^1.7.2", + "d3-scale-chromatic": "^3.1.0", "idb-keyval": "^6.2.1", "lodash": "^4.17.21", "maplibre-gl": "^4.4.1", @@ -37,6 +38,7 @@ "@flydotio/dockerfile": "^0.5.8", "@testing-library/jest-dom": "^6.4.6", "@testing-library/react": "^16.0.0", + "@types/d3-scale-chromatic": "^3.0.3", "@types/lodash": "^4.17.5", "@types/node": "^20", "@types/react": "^18", @@ -50,4 +52,4 @@ "tailwindcss": "^3.4.1", "typescript": "^5" } -} +} \ No newline at end of file diff --git a/app/src/app/components/sidebar/DataPanels.tsx b/app/src/app/components/sidebar/DataPanels.tsx index ec16ce4ec..f99b7d269 100644 --- a/app/src/app/components/sidebar/DataPanels.tsx +++ b/app/src/app/components/sidebar/DataPanels.tsx @@ -1,11 +1,7 @@ -import {Box, Flex, Heading} from '@radix-ui/themes'; -import {MapModeSelector} from './MapModeSelector'; -import {ColorPicker} from './ColorPicker'; -import {ResetMapButton} from './ResetMapButton'; -import {GerryDBViewSelector} from './GerryDBViewSelector'; +import {Box} from '@radix-ui/themes'; +import Evaluation from '@components/sidebar/Evaluation'; import {HorizontalBar} from './charts/HorizontalBarChart'; -import {useMapStore} from '@/app/store/mapStore'; -import {Tabs, Text} from '@radix-ui/themes'; +import {Tabs} from '@radix-ui/themes'; import Layers from './Layers'; import React from 'react'; @@ -35,7 +31,7 @@ const defaultPanels: DataPanelSpec[] = [ { title: 'evaluation', label: 'Evaluation', - content: Unimplemented , + content: , }, ]; diff --git a/app/src/app/components/sidebar/Evaluation.tsx b/app/src/app/components/sidebar/Evaluation.tsx new file mode 100644 index 000000000..af31c3a09 --- /dev/null +++ b/app/src/app/components/sidebar/Evaluation.tsx @@ -0,0 +1,325 @@ +import React, {useMemo, useState} from 'react'; +import {useMapStore} from '@/app/store/mapStore'; +import {useQuery} from '@tanstack/react-query'; +import { + CleanedP1ZoneSummaryStats, + CleanedP1ZoneSummaryStatsKeys, + getP1SummaryStats, + P1ZoneSummaryStats, + P1ZoneSummaryStatsKeys, +} from '@/app/utils/api/apiHandlers'; +import {Button, Checkbox, CheckboxGroup} from '@radix-ui/themes'; +import {Heading, Flex, Spinner, Text} from '@radix-ui/themes'; +import {queryClient} from '@utils/api/queryClient'; +import {formatNumber, NumberFormats} from '@/app/utils/numbers'; +import {colorScheme} from '@/app/constants/colors'; +import { + getEntryTotal, + getStdDevColor, + stdDevArray, + stdDevColors, + sumArray, +} from '@utils/summaryStats'; +import {interpolateBlues, interpolateGreys} from 'd3-scale-chromatic'; + +type EvalModes = 'share' | 'count' | 'totpop'; +type ColumnConfiguration> = Array<{label: string; column: keyof T}>; +type EvaluationProps = { + columnConfig?: ColumnConfiguration; +}; + +// const calculateColumn = ( +// mode: EvalModes, +// entry: P1ZoneSummaryStats, +// totals: P1ZoneSummaryStats, +// column: keyof Omit +// ) => { +// const count = entry[column]; +// switch (mode) { +// case 'count': +// return count; +// case 'pct': +// return count / entry['total']; +// case 'share': +// return count / totals[column]; +// } +// }; + +const defaultColumnConfig: ColumnConfiguration = [ + { + label: 'White', + column: 'white_pop', + }, + { + label: 'Black', + column: 'black_pop', + }, + { + label: 'Asian', + column: 'asian_pop', + }, + { + label: 'Am. Indian', + column: 'amin_pop', + }, + { + label: 'Pacific Isl.', + column: 'nhpi_pop', + }, + { + label: 'Other', + column: 'other_pop', + }, +]; + +const modeButtonConfig: Array<{label: string; value: EvalModes}> = [ + { + label: 'Population by Share', + value: 'share', + }, + { + label: 'Population by Count', + value: 'count', + }, + // { + // label: "Population by Percent of Zone", + // value: 'totpop' + // } +]; + +const numberFormats: Record = { + share: 'percent', + count: 'string', + totpop: 'percent', +}; + +const getColConfig = (evalMode: EvalModes) => { + switch (evalMode) { + case 'share': + return (col: keyof P1ZoneSummaryStats) => `${col}_pct` as keyof CleanedP1ZoneSummaryStats; + default: + return (col: keyof P1ZoneSummaryStats) => col; + } +}; + +const Evaluation: React.FC = ({columnConfig = defaultColumnConfig}) => { + const [evalMode, setEvalMode] = useState('share'); + // const [showAverages, setShowAverages] = useState(true); + // const [showStdDev, setShowStdDev] = useState(false); + const [colorBg, setColorBg] = useState(true); + const [showUnassigned, setShowUnassigned] = useState(true); + + const numberFormat = numberFormats[evalMode]; + const columnGetter = getColConfig(evalMode); + const totPop = useMapStore(state => state.summaryStats.totpop?.data); + const mapDocument = useMapStore(state => state.mapDocument); + const assignmentsHash = useMapStore(state => state.assignmentsHash); + + const {data, error, isLoading} = useQuery( + { + queryKey: ['p1SummaryStats', mapDocument, assignmentsHash], + queryFn: () => mapDocument && getP1SummaryStats(mapDocument), + enabled: !!mapDocument, + staleTime: 0, + placeholderData: previousData => previousData, + }, + queryClient + ); + + const { + unassigned, + maxValues, + // averages, + // stdDevs + } = useMemo(() => { + if (!data?.results || !totPop) { + return {}; + } + let maxValues: Record = {}; + + let unassigned: Record = { + ...totPop, + zone: -999, + total: getEntryTotal(totPop), + }; + P1ZoneSummaryStatsKeys.forEach(key => { + let total = unassigned[key]; + maxValues[key] = -Math.pow(10, 12); + data.results.forEach(row => { + total -= row[key]; + maxValues[key] = Math.max(row[key], maxValues[key]); + }); + unassigned[`${key}_pct`] = total / unassigned[key]; + unassigned[key] = total; + }); + // const averages: Record = {}; + // const stdDevs: Record = {}; + // CleanedP1ZoneSummaryStatsKeys.forEach(key => { + // const values = data.results.map(row => row[key]); + // averages[key] = sumArray(values) / data.results.length; + // stdDevs[key] = stdDevArray(values); + // }); + return { + unassigned, + maxValues, + // averages, + // stdDevs + }; + }, [data?.results, totPop]); + + if (!data || !maxValues || (mapDocument && !mapDocument.available_summary_stats)) { + return Summary statistics are not available for this map.; + } + + if (error) { + return ( +
+

Summary Statistics

+

There was an error loading the summary statistics.

+
+ ); + } + const rows = unassigned && showUnassigned ? [...data.results, unassigned] : data.results; + return ( +
+ + {modeButtonConfig.map((mode, i) => ( + + ))} + {isLoading && } + + + + + setShowUnassigned(v => !v)}> + Show Unassigned Population + + {/* setShowAverages(v => !v)}> + Show Zone Averages + + setShowStdDev(v => !v)}> + Show Zone Std. Dev. + */} + setColorBg(v => !v)}> + +

Color Cells By Values

+ {/* {colorByStdDev && ( + + {Object.entries(stdDevColors) + .sort((a, b) => +a[0] - +b[0]) + .map(([stdev, backgroundColor], i) => ( + + {+stdev > 0 ? `+${stdev}`: stdev} + + ))} + + )} */} + + + + +
+ + + + + {columnConfig.map((f, i) => ( + + ))} + + + + {/* {!!(averages && showAverages) && ( + + + {columnConfig.map((f, i) => ( + + ))} + + )} + {!!(stdDevs && showStdDev) && ( + + + {columnConfig.map((f, i) => ( + + ))} + + )} */} + {rows + .sort((a, b) => a.zone - b.zone) + .map(row => { + const isUnassigned = row.zone === -999; + const zoneName = isUnassigned ? 'None' : row.zone; + const backgroundColor = isUnassigned ? '#DDDDDD' : colorScheme[row.zone - 1]; + + return ( + + + {columnConfig.map((f, i) => { + const column = columnGetter(f.column); + const colorValue = + evalMode === 'count' ? row[column] / maxValues[column] : row[column]; + const backgroundColor = + colorBg && !isUnassigned + ? interpolateGreys(colorValue) + .replace('rgb', 'rgba') + .replace(')', ',0.5)') + : 'initial'; + return ( + + ); + })} + + ); + })} + +
Zone + {f.label} +
+ Zone Averages + + {formatNumber(averages[columnGetter(f.column)], numberFormat)} +
+ Zone Std. Dev. + + {formatNumber(stdDevs[columnGetter(f.column)], numberFormat)} +
+ + {zoneName} + + {formatNumber(row[column], numberFormat)} +
+
+
+ ); +}; + +export default Evaluation; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 19cabb0ce..dccfb99d7 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -8,6 +8,7 @@ import { Assignment, DistrictrMap, DocumentObject, + P1TotPopSummaryStats, ShatterResult, ZonePopulation, } from '../utils/api/apiHandlers'; @@ -22,19 +23,19 @@ import { getFeaturesInBbox, resetZoneColors, setZones, -} from '../utils/helpers'; -import {getRenderSubscriptions} from './mapRenderSubs'; +} from "../utils/helpers"; +import { getRenderSubscriptions } from "./mapRenderSubs"; +import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; +import { getMapMetricsSubs } from "./metricsSubs"; +import { getMapEditSubs } from "./mapEditSubs"; +import { getQueriesResultsSubs } from "../utils/api/queries"; +import { persistOptions } from "./persistConfig"; import {patchReset, patchShatter, patchUnShatter} from '../utils/api/mutations'; -import {getSearchParamsObersver} from '../utils/api/queryParamsListener'; -import {getMapMetricsSubs} from './metricsSubs'; -import {getMapEditSubs} from './mapEditSubs'; import bbox from '@turf/bbox'; import {BLOCK_SOURCE_ID} from '../constants/layers'; -import {getMapViewsSubs} from '../utils/api/queries'; -import {persistOptions} from './persistConfig'; -import {onlyUnique} from '../utils/arrays'; import {DistrictrMapOptions} from './types'; -import {queryClient} from '../utils/api/queryClient'; +import { onlyUnique } from '../utils/arrays'; +import { queryClient } from '../utils/api/queryClient'; const combineSetValues = (setRecord: Record>, keys?: string[]) => { const combinedSet = new Set(); // Create a new set to hold combined values @@ -77,6 +78,15 @@ export interface MapStore { */ mapDocument: DocumentObject | null; setMapDocument: (mapDocument: DocumentObject) => void; + summaryStats: { + totpop?: { + data: P1TotPopSummaryStats + } + }, + setSummaryStat: ( + stat: T, + value: MapStore['summaryStats'][T] + ) => void, // SHATTERING /** * A subset of IDs that a user is working on in a focused view. @@ -222,6 +232,8 @@ export interface MapStore { resetAccumulatedBlockPopulations: () => void; zoneAssignments: Map; // geoid -> zone setZoneAssignments: (zone: NullableZone, gdbPaths: Set) => void; + assignmentsHash: string; + setAssignmentsHash: (hash: string) => void; loadZoneAssignments: (assigments: Assignment[]) => void; resetZoneAssignments: () => void; zonePopulations: Map; @@ -381,6 +393,15 @@ export const useMapStore = create( shatterIds: {parents: new Set(), children: new Set()}, }); }, + summaryStats: {}, + setSummaryStat: (stat, value) => { + set({ + summaryStats: { + ...get().summaryStats, + [stat]: value + } + }) + }, // TODO: Refactor to something like this // featureStates: { // locked: [], @@ -612,12 +633,8 @@ export const useMapStore = create( userMaps.splice(i, 1, userMapData); // Replace the map at index i with the new data } else { const urlParams = new URL(window.location.href).searchParams; - urlParams.delete('document_id'); // Remove the document_id parameter - window.history.pushState( - {}, - '', - window.location.pathname + '?' + urlParams.toString() - ); // Update the URL without document_id + urlParams.delete("document_id"); // Remove the document_id parameter + window.history.pushState({}, '', window.location.pathname + '?' + urlParams.toString()); // Update the URL without document_id userMaps.splice(i, 1); } } @@ -766,6 +783,8 @@ export const useMapStore = create( selectedZone: 1, setSelectedZone: zone => set({selectedZone: zone}), zoneAssignments: new Map(), + assignmentsHash: "", + setAssignmentsHash: (hash) => set({ assignmentsHash: hash }), accumulatedGeoids: new Set(), setAccumulatedGeoids: accumulatedGeoids => set({accumulatedGeoids}), setZoneAssignments: (zone, geoids) => { @@ -867,6 +886,6 @@ export const useMapStore = create( // these need to initialize after the map store getRenderSubscriptions(useMapStore); getMapMetricsSubs(useMapStore); -getMapViewsSubs(useMapStore); +getQueriesResultsSubs(useMapStore); getMapEditSubs(useMapStore); getSearchParamsObersver(); diff --git a/app/src/app/store/metricsSubs.ts b/app/src/app/store/metricsSubs.ts index 25a8c7713..7329a6787 100644 --- a/app/src/app/store/metricsSubs.ts +++ b/app/src/app/store/metricsSubs.ts @@ -1,4 +1,4 @@ -import {updateMapMetrics} from '../utils/api/queries'; +import {updateMapMetrics, updateTotPop} from '../utils/api/queries'; import {useMapStore as _useMapStore} from './mapStore'; export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { @@ -7,6 +7,7 @@ export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { mapDocument => { if (mapDocument) { updateMapMetrics(mapDocument); + updateTotPop(mapDocument) } } ); diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 4b7736a27..e9886b4ae 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import 'maplibre-gl'; import {useMapStore} from '@/app/store/mapStore'; +import { getEntryTotal } from '../summaryStats'; export const FormatAssignments = () => { const assignments = Array.from(useMapStore.getState().zoneAssignments.entries()).map( @@ -61,6 +62,7 @@ export interface DocumentObject { created_at: string; updated_at: string | null; extent: [number, number, number, number]; // [minx, miny, maxx, maxy] + available_summary_stats: string[]; } /** @@ -105,7 +107,7 @@ export const getDocument: (document_id: string) => Promise = asy }; export const getAssignments: ( - mapDocument: DocumentObject + mapDocument: DocumentObject | null ) => Promise = async mapDocument => { if (mapDocument) { return await axios @@ -149,6 +151,108 @@ export const getZonePopulations: ( } }; +export interface SummaryStatsResult { + summary_stat: string; + results: T; +} + +/** + * P1ZoneSummaryStats + * + * @interface + * @property {number} zone - The zone. + * @property {number} total_pop - The total population. + */ +export interface P1ZoneSummaryStats { + zone: number; + other_pop: number; + asian_pop: number; + amin_pop: number; + nhpi_pop: number; + black_pop: number; + white_pop: number; +} +export type P1TotPopSummaryStats = Omit + +export const P1ZoneSummaryStatsKeys = [ + 'other_pop', + 'asian_pop', + 'amin_pop', + 'nhpi_pop', + 'black_pop', + 'white_pop' +] as const + +export const CleanedP1ZoneSummaryStatsKeys = [ + ...P1ZoneSummaryStatsKeys, + 'total', + 'other_pop_pct', + 'asian_pop_pct', + 'amin_pop_pct', + 'nhpi_pop_pct', + 'black_pop_pct', + 'white_pop_pct', +] as const + +export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { + total: number; + other_pop_pct: number; + asian_pop_pct: number; + amin_pop_pct: number; + nhpi_pop_pct: number; + black_pop_pct: number; + white_pop_pct: number; +} + +/** + * Get P1 zone stats from the server. + * @param mapDocument - DocumentObject, the document object + * @returns Promise + */ +export const getP1SummaryStats: ( + mapDocument: DocumentObject +) => Promise> = async mapDocument => { + if (mapDocument) { + return await axios + .get>(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/P1`) + .then(res => { + const results = res.data.results.map(row => { + const total = getEntryTotal(row) + return P1ZoneSummaryStatsKeys.reduce((acc, key) => { + acc[`${key}_pct`] = acc[key] / total; + return acc; + }, { + ...row, + total + }) as CleanedP1ZoneSummaryStats + }) + return { + ...res.data, + results + } + }) + } else { + throw new Error('No document provided'); + } +}; + +/** + * Get P1 zone stats from the server. + * @param mapDocument - DocumentObject, the document object + * @returns Promise + */ +export const getP1TotPopSummaryStats: ( + mapDocument: DocumentObject | null +) => Promise> = async mapDocument => { + if (mapDocument) { + return await axios + .get>(`${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/P1/${mapDocument.parent_layer}`) + .then(res => res.data) + } else { + throw new Error('No document provided'); + } +}; + /** * Get available DistrictrMap views from the server. * @param limit - number, the number of views to return (default 10, max 100) diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts index 30ae99dff..a8dbc0ecc 100644 --- a/app/src/app/utils/api/mutations.ts +++ b/app/src/app/utils/api/mutations.ts @@ -27,6 +27,7 @@ export const patchShatter = new MutationObserver(queryClient, { }, onSuccess: data => { console.log(`Successfully shattered parents into ${data.children.length} children`); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); return data; }, }); @@ -60,6 +61,7 @@ export const patchUpdates = new MutationObserver(queryClient, { }, onSuccess: (data: AssignmentsCreate) => { console.log(`Successfully upserted ${data.assignments_upserted} assignments`); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); mapMetrics.refetch(); // remove trailing shattered features // This needs to happen AFTER the updates are done @@ -96,6 +98,7 @@ export const document = new MutationObserver(queryClient, { }, onSuccess: data => { useMapStore.getState().setMapDocument(data); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); useMapStore.getState().setAppLoadingState('loaded'); const documentUrl = new URL(window.location.toString()); documentUrl.searchParams.set('document_id', data.document_id); diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index e50f7996c..2549b3c0c 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -9,12 +9,32 @@ import { getDocument, getZonePopulations, ZonePopulation, + SummaryStatsResult, + getP1TotPopSummaryStats, + P1TotPopSummaryStats, } from './apiHandlers'; import {MapStore, useMapStore} from '@/app/store/mapStore'; const INITIAL_VIEW_LIMIT = 30; const INITIAL_VIEW_OFFSET = 0; +/** + * A utility function that returns a query function based on a nullable parameter. + * + * @param callback - A function that takes a parameter of type ParamT and returns a Promise of type ResultT. + * @param nullableParam - An optional parameter of type ParamT. If this parameter is not provided or is falsy, the function will return a function that returns null. + * + * @returns A function that, when called, will either return null (if nullableParam is not provided) + * or call the callback function with the nullableParam and return its result. + * + * @template ParamT - The type of the parameter that the callback function accepts. + * @template ResultT - The type of the result that the callback function returns. + */ +const getNullableParamQuery = (callback: (param: ParamT) => Promise, nullableParam?: ParamT) => { + if (!nullableParam) return () => null; + return async () => await callback(nullableParam); +}; + export const mapMetrics = new QueryObserver(queryClient, { queryKey: ['_zonePopulations'], queryFn: skipToken, @@ -43,12 +63,19 @@ export const updateMapViews = (limit: number, offset: number) => { }); }; -export const getMapViewsSubs = (_useMapStore: typeof useMapStore) => { +export const getQueriesResultsSubs = (_useMapStore: typeof useMapStore) => { mapViewsQuery.subscribe(result => { if (result) { _useMapStore.getState().setMapViews(result); } }); + fetchTotPop.subscribe(response => { + if (response?.data?.results) { + useMapStore.getState().setSummaryStat('totpop', { data: response.data.results}); + } else { + useMapStore.getState().setSummaryStat('totpop', undefined) + } + }); }; const getDocumentFunction = (documentId?: string) => { @@ -81,19 +108,14 @@ updateDocumentFromId.subscribe(mapDocument => { } }); -const getFetchAssignmentsQuery = (mapDocument?: MapStore['mapDocument']) => { - if (!mapDocument) return () => null; - return async () => await getAssignments(mapDocument); -}; - export const fetchAssignments = new QueryObserver(queryClient, { queryKey: ['assignments'], - queryFn: getFetchAssignmentsQuery(), + queryFn: getNullableParamQuery(getAssignments) }); export const updateAssignments = (mapDocument: DocumentObject) => { fetchAssignments.setOptions({ - queryFn: getFetchAssignmentsQuery(mapDocument), + queryFn: getNullableParamQuery(getAssignments, mapDocument), queryKey: ['assignments', performance.now()], }); }; @@ -103,3 +125,16 @@ fetchAssignments.subscribe(assignments => { useMapStore.getState().loadZoneAssignments(assignments.data); } }); + +export const fetchTotPop = new QueryObserver | null>(queryClient, { + queryKey: ['gerrydb_tot_pop'], + queryFn: getNullableParamQuery>(getP1TotPopSummaryStats), +}); + +export const updateTotPop = (mapDocument: DocumentObject | null) => { + fetchTotPop.setOptions({ + queryFn: getNullableParamQuery(getP1TotPopSummaryStats, mapDocument), + queryKey: ['gerrydb_tot_pop', mapDocument?.gerrydb_table], + }); +}; + diff --git a/app/src/app/utils/numbers.ts b/app/src/app/utils/numbers.ts new file mode 100644 index 000000000..fb6443070 --- /dev/null +++ b/app/src/app/utils/numbers.ts @@ -0,0 +1,33 @@ +const percentFormatter = new Intl.NumberFormat('en-US', { + style: 'percent', + minimumFractionDigits: 0, + maximumFractionDigits: 1, +}) +const compactFormatter = new Intl.NumberFormat('en-US', { + notation: 'compact', + compactDisplay: 'short', +}) + +const stringFormatter = (n: number) => (Math.round(n)).toLocaleString() + +export type NumberFormats = 'percent' | 'string' | 'compact' +export const formatNumber = ( + value: number | undefined, + format: NumberFormats +) => { + if (value === undefined) { + return value + } + switch(format){ + case 'percent': + return percentFormatter.format(value) + case 'string': // Added case for 'string' + return stringFormatter(value) // Format as string + case 'compact': // Added case for 'compact' + return compactFormatter.format(value) // Format as compact + default: + const exhaustiveCheck: never = format; + throw new Error(`Unhandled format case: ${exhaustiveCheck}`); + + } +} \ No newline at end of file diff --git a/app/src/app/utils/summaryStats.ts b/app/src/app/utils/summaryStats.ts new file mode 100644 index 000000000..5b663967a --- /dev/null +++ b/app/src/app/utils/summaryStats.ts @@ -0,0 +1,31 @@ +import { P1ZoneSummaryStats } from "./api/apiHandlers"; + +export const getEntryTotal = (entry: Omit) => + Object.entries(entry).reduce((total, [key, value]) => { + if (key !== 'zone') { + return total + value; // Sum values of properties except 'zone' + } + return total; // Return total unchanged for 'zone' + }, 0); + + +export const sumArray = (arr: number[]) => arr.reduce((total, value) => total + value, 0); +export const stdDevArray = (arr: number[]) => { + const mean = sumArray(arr) / arr.length; // Calculate mean + const variance = arr.reduce((total, value) => total + Math.pow(value - mean, 2), 0) / arr.length; // Calculate variance + return Math.sqrt(variance); // Return standard deviation +} + +export const stdDevColors = { + [-2]: '#5e3c9977', + [-1]: '#b2abd277', + [0]: "#ffffff", + [1]: '#fdb86377', + [2]: '#e6610177' +} as const + +export const getStdDevColor = (value: number) => { + const floorValue = value > 0 ? Math.floor(value) : Math.ceil(value) + const cleanValue= (floorValue < -2 ? -2 : floorValue > 2 ? 2 : floorValue) as keyof typeof stdDevColors + return stdDevColors[cleanValue] || 'none' +} \ No newline at end of file diff --git a/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py b/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py new file mode 100644 index 000000000..4fc8ca915 --- /dev/null +++ b/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py @@ -0,0 +1,30 @@ +"""create summary stat metadata endpoint + +Revision ID: 5d9f7335f98a +Revises: 65a4fc0a727d +Create Date: 2024-09-11 10:15:07.929311 + +""" + +from typing import Sequence, Union + +from alembic import op +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "5d9f7335f98a" +down_revision: Union[str, None] = "65a4fc0a727d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with open(SQL_DIR / "available_summary_stat_udf.sql", "r") as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + sql = "DROP FUNCTION IF EXISTS get_available_summary_stats;" + op.execute(sql) diff --git a/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py new file mode 100644 index 000000000..091aa3bf3 --- /dev/null +++ b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py @@ -0,0 +1,50 @@ +"""add available stats to districtrmap + +Revision ID: c3541f016d35 +Revises: 5d9f7335f98a +Create Date: 2024-11-10 12:56:36.766141 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c3541f016d35" +down_revision: Union[str, None] = "5d9f7335f98a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "districtrmap", + sa.Column("available_summary_stats", sa.ARRAY(sa.TEXT()), nullable=True), + ) + + op.execute( + sa.text(""" + UPDATE districtrmap d + SET available_summary_stats = ( + SELECT + CASE WHEN d.child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer)) + END + ) + """) + ) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("districtrmap", "available_summary_stats") + # ### end Alembic commands ### diff --git a/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py b/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py new file mode 100644 index 000000000..29c1aa36a --- /dev/null +++ b/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py @@ -0,0 +1,39 @@ +"""summary stat udfs + +Revision ID: f86991e63a62 +Revises: c3541f016d35 +Create Date: 2024-11-10 14:17:46.753393 + +""" + +from typing import Sequence, Union + +from alembic import op +from pathlib import Path +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "f86991e63a62" +down_revision: Union[str, None] = "c3541f016d35" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + for udf in [ + "summary_stats_p1.sql", + "summary_stats_p1_totals.sql", + "summary_stats_p4.sql", + "summary_stats_p4_totals.sql", + ]: + with Path(SQL_DIR, udf).open() as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + op.execute("DROP FUNCTION IF EXISTS get_summary_stats_p1") + op.execute("DROP FUNCTION IF EXISTS get_summary_p1_totals") + op.execute("DROP FUNCTION IF EXISTS get_summary_stats_p4") + op.execute("DROP FUNCTION IF EXISTS get_summary_p4_totals") diff --git a/backend/app/main.py b/backend/app/main.py index 57b6b8c33..72407b4d1 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -26,6 +26,11 @@ DistrictrMapPublic, ParentChildEdges, ShatterResult, + SummaryStatisticType, + SummaryStatsP1, + PopulationStatsP1, + SummaryStatsP4, + PopulationStatsP4, ) if settings.ENVIRONMENT == "production": @@ -101,6 +106,7 @@ async def create_document( DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore + DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore ) .where(Document.document_id == document_id) .join( @@ -260,6 +266,7 @@ async def get_document(document_id: str, session: Session = Depends(get_session) DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore + DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore ) # pyright: ignore .where(Document.document_id == document_id) .join( @@ -301,6 +308,100 @@ async def get_total_population( ) +@app.get("/api/document/{document_id}/{summary_stat}") +async def get_summary_stat( + document_id: str, summary_stat: str, session: Session = Depends(get_session) +): + try: + _summary_stat = SummaryStatisticType[summary_stat] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid summary_stat: {summary_stat}", + ) + + try: + stmt, SummaryStatsModel = { + "P1": ( + text( + "SELECT * from get_summary_stats_p1(:document_id) WHERE zone is not null" + ), + SummaryStatsP1, + ), + "P4": { + text( + "SELECT * from get_summary_stats_p4(:document_id) WHERE zone is not null" + ), + SummaryStatsP4, + }, + }[summary_stat] + except KeyError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Summary stats not implemented for {summary_stat}", + ) + + try: + results = session.execute(stmt, {"document_id": document_id}).fetchall() + return { + "summary_stat": _summary_stat.value, + "results": [SummaryStatsModel.from_orm(row) for row in results], + } + except ProgrammingError as e: + logger.error(e) + error_text = str(e) + if f"Table name not found for document_id: {document_id}" in error_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Document with ID {document_id} not found", + ) + + +@app.get("/api/districtrmap/summary_stats/{summary_stat}/{gerrydb_table}") +async def get_gerrydb_summary_stat( + summary_stat: str, gerrydb_table: str, session: Session = Depends(get_session) +): + try: + _summary_stat = SummaryStatisticType[summary_stat] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid summary_stat: {summary_stat}", + ) + + try: + summary_stat_udf, SummaryStatsModel = { + "P1": ("get_summary_p1_totals", PopulationStatsP1), + "P4": ("get_summary_p4_totals", PopulationStatsP4), + }[summary_stat] + except KeyError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Summary stats not implemented for {summary_stat}", + ) + + stmt = text( + f"""SELECT * + FROM {summary_stat_udf}(:gerrydb_table)""" + ).bindparams( + bindparam(key="gerrydb_table", type_=String), + ) + try: + results = session.execute(stmt, {"gerrydb_table": gerrydb_table}).fetchone() + return { + "summary_stat": _summary_stat.value, + "results": SummaryStatsModel.from_orm(results), + } + except ProgrammingError as e: + logger.error(e) + error_text = str(e) + if f"Table {gerrydb_table} does not exist in gerrydb schema" in error_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Gerrydb Table with ID {gerrydb_table} not found", + ) + + @app.get("/api/gerrydb/views", response_model=list[DistrictrMapPublic]) async def get_projects( *, diff --git a/backend/app/models.py b/backend/app/models.py index 16fff215f..b40c20aee 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,6 +1,6 @@ from datetime import datetime from typing import Optional -from pydantic import UUID4, BaseModel +from pydantic import UUID4, BaseModel, ConfigDict from sqlmodel import ( Field, ForeignKey, @@ -13,9 +13,11 @@ MetaData, String, ) -from sqlalchemy.types import ARRAY +from sqlalchemy.types import ARRAY, TEXT from sqlalchemy import Float from app.constants import DOCUMENT_SCHEMA +from enum import Enum +from typing import Any class UUIDType(UUID): @@ -44,6 +46,13 @@ class TimeStampMixin(SQLModel): ) +class SummaryStatisticType(Enum): + P1 = "Population by Race" + P2 = "Hispanic or Latino, and Not Hispanic or Latino by Race" + P3 = "Voting Age Population by Race" + P4 = "Hispanic or Latino, and Not Hispanic or Latino by Race Voting Age Population" + + class DistrictrMap(TimeStampMixin, SQLModel, table=True): uuid: str = Field(sa_column=Column(UUIDType, unique=True, primary_key=True)) name: str = Field(nullable=False) @@ -64,11 +73,14 @@ class DistrictrMap(TimeStampMixin, SQLModel, table=True): String, ForeignKey("gerrydbtable.name"), default=None, nullable=True ) ) - extent: list[float] = Field(sa_column=Column(ARRAY(Float), nullable=True)) + extent: list[float] | None = Field(sa_column=Column(ARRAY(Float), nullable=True)) # schema? will need to contrain the schema # where does this go? # when you create the view, pull the columns that you need # we'll want discrete management steps + available_summary_stats: list[SummaryStatisticType] | None = Field( + sa_column=Column(ARRAY(TEXT), nullable=True, default=[]) + ) class DistrictrMapPublic(BaseModel): @@ -78,6 +90,7 @@ class DistrictrMapPublic(BaseModel): child_layer: str | None = None tiles_s3_path: str | None = None num_districts: int | None = None + available_summary_stats: list[str] | None = None class GerryDBTable(TimeStampMixin, SQLModel, table=True): @@ -130,6 +143,7 @@ class DocumentPublic(BaseModel): created_at: datetime updated_at: datetime extent: list[float] | None = None + available_summary_stats: list[str] | None = None class AssignmentsBase(SQLModel): @@ -175,3 +189,37 @@ class ShatterResult(BaseModel): class ZonePopulation(BaseModel): zone: int total_pop: int + + +class SummaryStats(BaseModel): + summary_stat: SummaryStatisticType + results: list[Any] + + +class PopulationStatsP1(BaseModel): + model_config = ConfigDict(from_attributes=True) + other_pop: int + asian_pop: int + amin_pop: int + nhpi_pop: int + black_pop: int + white_pop: int + + +class SummaryStatsP1(PopulationStatsP1): + zone: int + + +class PopulationStatsP4(BaseModel): + model_config = ConfigDict(from_attributes=True) + hispanic_vap: int + non_hispanic_asian_vap: int + non_hispanic_amin_vap: int + non_hispanic_nhpi_vap: int + non_hispanic_black_vap: int + non_hispanic_white_vap: int + non_hispanic_other_vap: int + + +class SummaryStatsP4(PopulationStatsP4): + zone: int diff --git a/backend/app/sql/available_summary_stat_udf.sql b/backend/app/sql/available_summary_stat_udf.sql new file mode 100644 index 000000000..7992c03e5 --- /dev/null +++ b/backend/app/sql/available_summary_stat_udf.sql @@ -0,0 +1,82 @@ +CREATE OR REPLACE FUNCTION get_available_summary_stats(gerrydb_table_name TEXT) +RETURNS TABLE (summary_stat TEXT) AS $$ +DECLARE + p1 BOOLEAN; + p2 BOOLEAN; + p3 BOOLEAN; + p4 BOOLEAN; +BEGIN + SELECT count(column_name) = 6 INTO p1 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('other_pop', + 'asian_pop', + 'amin_pop', + 'nhpi_pop', + 'black_pop', + 'white_pop') + ; + + SELECT count(column_name) = 6 INTO p3 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('other_vap', + 'asian_vap', + 'amin_vap', + 'nhpi_vap', + 'black_vap', + 'white_vap') + ; + + SELECT count(column_name) = 7 INTO p2 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('hispanic_pop', + 'non_hispanic_asian_pop', + 'non_hispanic_amin_pop', + 'non_hispanic_nhpi_pop', + 'non_hispanic_black_pop', + 'non_hispanic_white_pop', + 'non_hispanic_other_pop' + ) + ; + + SELECT count(column_name) = 7 INTO p4 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('hispanic_vap', + 'non_hispanic_asian_vap', + 'non_hispanic_amin_vap', + 'non_hispanic_nhpi_vap', + 'non_hispanic_black_vap', + 'non_hispanic_white_vap', + 'non_hispanic_other_vap' + ) + ; + + RETURN QUERY + + SELECT 'P1' as summary_stat + WHERE p1 + + UNION + + SELECT 'P2' + WHERE p2 + + UNION + + SELECT 'P3' + WHERE p3 + + UNION + + SELECT 'P4' + WHERE p4; + +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p1.sql b/backend/app/sql/summary_stats_p1.sql new file mode 100644 index 000000000..bc9fa994e --- /dev/null +++ b/backend/app/sql/summary_stats_p1.sql @@ -0,0 +1,42 @@ +CREATE OR REPLACE FUNCTION get_summary_stats_p1(document_id UUID) +RETURNS TABLE ( + zone TEXT, + other_pop BIGINT, + asian_pop BIGINT, + amin_pop BIGINT, + nhpi_pop BIGINT, + black_pop BIGINT, + white_pop BIGINT +) AS $$ +DECLARE + doc_districtrmap RECORD; + sql_query TEXT; +BEGIN + SELECT districtrmap.* INTO doc_districtrmap + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF doc_districtrmap.gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.other_pop, 0))::BIGINT AS other_pop, + SUM(COALESCE(blocks.asian_pop, 0))::BIGINT AS asian_pop, + SUM(COALESCE(blocks.amin_pop, 0))::BIGINT AS amin_pop, + SUM(COALESCE(blocks.nhpi_pop, 0))::BIGINT AS nhpi_pop, + SUM(COALESCE(blocks.black_pop, 0))::BIGINT AS black_pop, + SUM(COALESCE(blocks.white_pop, 0))::BIGINT AS white_pop + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', doc_districtrmap.gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p1_totals.sql b/backend/app/sql/summary_stats_p1_totals.sql new file mode 100644 index 000000000..9b65c9ad9 --- /dev/null +++ b/backend/app/sql/summary_stats_p1_totals.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE FUNCTION get_summary_p1_totals(gerrydb_table TEXT) +RETURNS TABLE ( + other_pop BIGINT, + asian_pop BIGINT, + amin_pop BIGINT, + nhpi_pop BIGINT, + black_pop BIGINT, + white_pop BIGINT +) AS $$ +DECLARE + table_exists BOOLEAN; + sql_query TEXT; + +BEGIN + -- Check if the table exists + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'gerrydb' + AND table_name = $1 + ) INTO table_exists; + + IF NOT table_exists THEN + RAISE EXCEPTION 'Table % does not exist in gerrydb schema', $1; + END IF; + + sql_query := format(' + SELECT + SUM(COALESCE(other_pop, 0))::BIGINT AS other_pop, + SUM(COALESCE(asian_pop, 0))::BIGINT AS asian_pop, + SUM(COALESCE(amin_pop, 0))::BIGINT AS amin_pop, + SUM(COALESCE(nhpi_pop, 0))::BIGINT AS nhpi_pop, + SUM(COALESCE(black_pop, 0))::BIGINT AS black_pop, + SUM(COALESCE(white_pop, 0))::BIGINT AS white_pop + FROM gerrydb.%I + ', $1); + RETURN QUERY EXECUTE sql_query; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p4.sql b/backend/app/sql/summary_stats_p4.sql new file mode 100644 index 000000000..1c75bfbcf --- /dev/null +++ b/backend/app/sql/summary_stats_p4.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION get_summary_stats_p4(document_id UUID) +RETURNS TABLE ( + zone TEXT, + hispanic_vap BIGINT, + non_hispanic_asian_vap BIGINT, + non_hispanic_amin_vap BIGINT, + non_hispanic_nhpi_vap BIGINT, + non_hispanic_black_vap BIGINT, + non_hispanic_white_vap BIGINT, + non_hispanic_other_vap BIGINT +) AS $$ +DECLARE + doc_districtrmap RECORD; + sql_query TEXT; +BEGIN + SELECT districtrmap.* INTO doc_districtrmap + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF doc_districtrmap.gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.hispanic_vap, 0))::BIGINT AS hispanic_vap, + SUM(COALESCE(blocks.non_hispanic_asian_vap, 0))::BIGINT AS non_hispanic_asian_vap, + SUM(COALESCE(blocks.non_hispanic_amin_vap, 0))::BIGINT AS non_hispanic_amin_vap, + SUM(COALESCE(blocks.non_hispanic_nhpi_vap, 0))::BIGINT AS non_hispanic_nhpi_vap, + SUM(COALESCE(blocks.non_hispanic_black_vap, 0))::BIGINT AS non_hispanic_black_vap, + SUM(COALESCE(blocks.non_hispanic_white_vap, 0))::BIGINT AS non_hispanic_white_vap, + SUM(COALESCE(blocks.non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', doc_districtrmap.gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p4_totals.sql b/backend/app/sql/summary_stats_p4_totals.sql new file mode 100644 index 000000000..a9c57f12d --- /dev/null +++ b/backend/app/sql/summary_stats_p4_totals.sql @@ -0,0 +1,41 @@ +CREATE OR REPLACE FUNCTION get_summary_p4_totals(gerrydb_table TEXT) +RETURNS TABLE ( + hispanic_vap BIGINT, + non_hispanic_asian_vap BIGINT, + non_hispanic_amin_vap BIGINT, + non_hispanic_nhpi_vap BIGINT, + non_hispanic_black_vap BIGINT, + non_hispanic_white_vap BIGINT, + non_hispanic_other_vap BIGINT +) AS $$ +DECLARE + table_exists BOOLEAN; + sql_query TEXT; + +BEGIN + -- Check if the table exists + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'gerrydb' + AND table_name = $1 + ) INTO table_exists; + + IF NOT table_exists THEN + RAISE EXCEPTION 'Table % does not exist in gerrydb schema', $1; + END IF; + + sql_query := format(' + SELECT + SUM(COALESCE(hispanic_vap, 0))::BIGINT AS hispanic_vap, + SUM(COALESCE(non_hispanic_asian_vap, 0))::BIGINT AS non_hispanic_asian_vap, + SUM(COALESCE(non_hispanic_amin_vap, 0))::BIGINT AS non_hispanic_amin_vap, + SUM(COALESCE(non_hispanic_nhpi_vap, 0))::BIGINT AS non_hispanic_nhpi_vap, + SUM(COALESCE(non_hispanic_black_vap, 0))::BIGINT AS non_hispanic_black_vap, + SUM(COALESCE(non_hispanic_white_vap, 0))::BIGINT AS non_hispanic_white_vap, + SUM(COALESCE(non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap + FROM gerrydb.%I + ', $1); + RETURN QUERY EXECUTE sql_query; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/utils.py b/backend/app/utils.py index 007e3a3e2..adb9a8eb5 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -5,7 +5,7 @@ import logging -from app.models import UUIDType +from app.models import SummaryStatisticType, UUIDType logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -202,3 +202,69 @@ def add_extent_to_districtrmap( END $$; """) session.execute(stmt) + + +def get_available_summary_stats(session: Session, gerrydb_table_name: str): + """ + Get the available summary statistics for a given gerrydb table. + + Args: + session: The database session. + gerrydb_table_name: The name of the gerrydb table. + """ + stmt = text("SELECT * FROM get_available_summary_stats(:gerrydb_table_name)") + return session.execute( + stmt, + { + "gerrydb_table_name": gerrydb_table_name, + }, + ).all() + + +def add_available_summary_stats_to_districtrmap( + session: Session, districtr_map_uuid: str, summary_stats: list[str] | None = None +) -> list[SummaryStatisticType] | None: + """ + Add the available summary statistics to the districtr map. + + Args: + session: The database session. + districtr_map_uuid: The UUID of the districtr map. + summary_stats: The summary statistics to add. + """ + if summary_stats is not None: + raise NotImplementedError( + "Manually adding summary stats to a districtr map is not yet implemented." + ) + + stmt = text( + """ + UPDATE districtrmap + SET available_summary_stats = + CASE WHEN child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(parent_layer)) + END + WHERE uuid = :districtr_map_uuid + RETURNING available_summary_stats + """ + ).bindparams( + bindparam(key="districtr_map_uuid", type_=UUIDType), + ) + result = session.execute( + stmt, + { + "districtr_map_uuid": districtr_map_uuid, + "summary_stats": summary_stats, + }, + ) + (available_summary_stats,) = result.one() + logger.info( + f"Updated available summary stats for districtr map {districtr_map_uuid} to {available_summary_stats}" + ) + return available_summary_stats diff --git a/backend/cli.py b/backend/cli.py index 11b592109..5aab2a6fa 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -13,6 +13,7 @@ create_shatterable_gerrydb_view as _create_shatterable_gerrydb_view, create_parent_child_edges as _create_parent_child_edges, add_extent_to_districtrmap as _add_extent_to_districtrmap, + add_available_summary_stats_to_districtrmap as _add_available_summary_stats_to_districtrmap, ) logger = logging.getLogger(__name__) @@ -233,6 +234,10 @@ def create_districtr_map( session=session, districtr_map_uuid=districtr_map_uuid, bounds=bounds ) + _add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + session.commit() logger.info(f"Districtr map created successfully {districtr_map_uuid}") @@ -292,5 +297,26 @@ def add_extent_to_districtr_map(districtr_map: str, bounds: list[float] | None = session.close() +@cli.command("add-available-summary-stats-to-districtr-map") +@click.option("--districtr-map", "-d", help="Districtr map name", required=True) +def add_available_summary_stats_to_districtr_map(districtr_map: str): + session = next(get_session()) + stmt = text( + "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" + ) + (districtr_map_uuid,) = session.execute( + stmt, params={"districtrmap_name": districtr_map} + ).one() + print(f"Found districtmap uuid: {districtr_map_uuid}") + + _add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + + session.commit() + logger.info("Updated available summary stats successfully.") + session.close() + + if __name__ == "__main__": cli() diff --git a/backend/load_data.py b/backend/load_data.py index 2e9faf981..bc1ac99e8 100755 --- a/backend/load_data.py +++ b/backend/load_data.py @@ -76,8 +76,8 @@ def load_sample_data(config): ) result = session.execute(exists_query).scalar() - if result > 0: - print(f"###\Districtr map {name} already exists.\n###") + if result is not None and result > 0: + print(f"Districtr map {name} already exists.") else: subprocess.run( [ diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson new file mode 100644 index 000000000..572cefc2e --- /dev/null +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson @@ -0,0 +1,17 @@ +{ +"type": "FeatureCollection", +"name": "SELECT", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "other_pop": 12, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "other_pop": 1, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "other_pop": 5, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "other_pop": 24, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "other_pop": 32, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +] +} diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson new file mode 100644 index 000000000..d84d67435 --- /dev/null +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson @@ -0,0 +1,17 @@ +{ +"type": "FeatureCollection", +"name": "SELECT", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 12, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 1, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 5, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 24, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 32, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +] +} diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 530a1dd1b..170fa242b 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -11,7 +11,7 @@ OGR2OGR_PG_CONNECTION_STRING, FIXTURES_PATH, ) -from app.utils import create_districtr_map +from app.utils import create_districtr_map, add_available_summary_stats_to_districtrmap def test_read_main(client): @@ -30,6 +30,8 @@ def test_get_session(): GERRY_DB_FIXTURE_NAME = "ks_demo_view_census_blocks" GERRY_DB_TOTAL_VAP_FIXTURE_NAME = "ks_demo_view_census_blocks_total_vap" GERRY_DB_NO_POP_FIXTURE_NAME = "ks_demo_view_census_blocks_no_pop" +GERRY_DB_P1_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats" +GERRY_DB_P4_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats_p4" ## Test DB @@ -450,3 +452,180 @@ def test_list_gerydb_views_offset_and_limit(client, districtr_maps): assert response.status_code == 200 data = response.json() assert len(data) == 1 + assert data[0]["name"] == "Districtr map ks_demo_view_census_blocks" + + +@pytest.fixture(name=GERRY_DB_P1_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats(session: Session): + layer = GERRY_DB_P1_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.execute(upsert_query, {"name": layer}) + + (districtr_map_uuid,) = create_districtr_map( + session=session, + name="DistrictMap with P1 view", + parent_layer_name=layer, + gerrydb_table_name=layer, + ) + add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + + session.commit() + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="document_id_p1_summary_stats") +def document_summary_stats_fixture(client, ks_demo_view_census_blocks_summary_stats): + response = client.post( + "/api/create_document", + json={ + "gerrydb_table": GERRY_DB_P1_FIXTURE_NAME, + }, + ) + document_id = response.json()["document_id"] + return document_id + + +def test_get_p1_summary_stats(client, document_id_p1_summary_stats): + # Set up assignments + document_id = document_id_p1_summary_stats + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": 2}, + ] + }, + ) + + summary_stat = "P1" + response = client.get(f"/api/document/{document_id}/{summary_stat}") + data = response.json() + assert response.status_code == 200 + assert data.get("summary_stat") == "Population by Race" + results = data.get("results") + assert results is not None + assert len(results) == 2 + record_1, record_2 = data.get("results") + assert record_1.get("zone") == 1 + assert record_2.get("zone") == 2 + assert record_1.get("other_pop") == 13 + assert record_2.get("other_pop") == 24 + + +@pytest.fixture(name=GERRY_DB_P4_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats_p4(session: Session): + layer = GERRY_DB_P4_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.execute(upsert_query, {"name": layer}) + + (districtr_map_uuid,) = create_districtr_map( + session=session, + name="DistrictMap with P4 view", + parent_layer_name=layer, + gerrydb_table_name=layer, + ) + summary_stats = add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + assert summary_stats == ["P4"], f"Expected P4 to be available, got {summary_stats}" + + session.commit() + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="document_id_p4_summary_stats") +def document_p4_summary_stats_fixture( + client, ks_demo_view_census_blocks_summary_stats_p4 +): + response = client.post( + "/api/create_document", + json={ + "gerrydb_table": GERRY_DB_P4_FIXTURE_NAME, + }, + ) + document_id = response.json()["document_id"] + return document_id + + +def test_get_p4_summary_stats(client, document_id_p4_summary_stats): + # Set up assignments + document_id = str(document_id_p4_summary_stats) + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": 2}, + ] + }, + ) + + summary_stat = "P4" + response = client.get(f"/api/document/{document_id}/{summary_stat}") + data = response.json() + assert response.status_code == 200 + assert ( + data.get("summary_stat") + == "Hispanic or Latino, and Not Hispanic or Latino by Race Voting Age Population" + ) + results = data.get("results") + assert results is not None + assert len(results) == 2 + record_1, record_2 = data.get("results") + assert record_1.get("zone") == 1 + assert record_2.get("zone") == 2 + assert record_1.get("hispanic_vap") == 13 + assert record_2.get("hispanic_vap") == 24 diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 883fcc631..7e03b154a 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -5,6 +5,7 @@ create_shatterable_gerrydb_view, create_parent_child_edges, add_extent_to_districtrmap, + get_available_summary_stats, ) from sqlmodel import Session import subprocess @@ -126,6 +127,42 @@ def districtr_map_fixture( return inserted_districtr_map +GERRY_DB_P1_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats" + + +@pytest.fixture(name=GERRY_DB_P1_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats(session: Session): + layer = GERRY_DB_P1_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.begin() + session.execute(upsert_query, {"name": GERRY_DB_P1_FIXTURE_NAME}) + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + # FOR THE TESTS BELOW I NEED TO ADD ACTUAL ASSERTIONS @@ -240,6 +277,18 @@ def test_shattering(client, session: Session, document_id): assert all(d["zone"] == 1 for d in data["children"]) +def test_get_available_summary_stats( + session: Session, ks_demo_view_census_blocks_summary_stats +): + result = get_available_summary_stats(session, GERRY_DB_P1_FIXTURE_NAME) + assert len(result) == 1 + (summary_stats_available,) = result + assert summary_stats_available + assert len(summary_stats_available) == 1 + (summary_stat,) = summary_stats_available + assert summary_stat == "P1" + + def test_unshatter_process(client, document_id): response = client.patch( "/api/update_assignments", From 4539490659ab182fdb91138030a50ce8f85b20e2 Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 18 Nov 2024 10:43:18 -0600 Subject: [PATCH 38/50] Wording updates (#184) --- app/src/app/components/sidebar/Layers.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index 4f3620fe2..e1b565000 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -71,12 +71,12 @@ export default function Layers() { disabled={!parentsAreBroken} onClick={() => toggleHighlightBrokenDistricts()} > - Highlight Broken Voter Districts + Highlight broken precincts setMapOptions({ higlightUnassigned: !mapOptions.higlightUnassigned })}> - Highlight Unassigned Districts + Highlight unassigned units toggleLockAllAreas()}> Lock All Painted Areas From 60153d50753af34d88cae14c1484a3e3b93d8c3b Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Mon, 18 Nov 2024 12:10:52 -0500 Subject: [PATCH 39/50] Soft delete DistrictrMap (#183) --- .../2494caf34886_soft_delete_districtrmap.py | 109 ++++++++++++++++++ backend/app/main.py | 11 +- backend/app/models.py | 3 + backend/app/sql/create_districtr_map_udf.sql | 19 +-- backend/app/utils.py | 9 +- backend/load_data.py | 4 +- backend/requirements.txt | 4 +- backend/tests/test_main.py | 41 +++++++ 8 files changed, 173 insertions(+), 27 deletions(-) create mode 100644 backend/app/alembic/versions/2494caf34886_soft_delete_districtrmap.py diff --git a/backend/app/alembic/versions/2494caf34886_soft_delete_districtrmap.py b/backend/app/alembic/versions/2494caf34886_soft_delete_districtrmap.py new file mode 100644 index 000000000..a352bdca1 --- /dev/null +++ b/backend/app/alembic/versions/2494caf34886_soft_delete_districtrmap.py @@ -0,0 +1,109 @@ +"""soft delete districtrmap + +Revision ID: 2494caf34886 +Revises: f86991e63a62 +Create Date: 2024-11-18 09:37:23.352006 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "2494caf34886" +down_revision: Union[str, None] = "f86991e63a62" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "districtrmap", + sa.Column("visible", sa.Boolean(), nullable=False, server_default="true"), + ) + op.execute( + sa.text( + """ + CREATE OR REPLACE FUNCTION create_districtr_map( + map_name VARCHAR, + gerrydb_table_name VARCHAR, + num_districts INTEGER, + tiles_s3_path VARCHAR, + parent_layer_name VARCHAR, + child_layer_name VARCHAR, + visibility BOOLEAN DEFAULT TRUE + ) + RETURNS UUID AS $$ + DECLARE + inserted_districtr_uuid UUID; + BEGIN + INSERT INTO districtrmap ( + created_at, + uuid, + name, + gerrydb_table_name, + num_districts, + tiles_s3_path, + parent_layer, + child_layer, + visible + ) + VALUES ( + now(), + gen_random_uuid(), + map_name, + gerrydb_table_name, + num_districts, + tiles_s3_path, + parent_layer_name, + child_layer_name, + visibility + ) + RETURNING uuid INTO inserted_districtr_uuid; + + RETURN inserted_districtr_uuid; + END; + $$ LANGUAGE plpgsql; + """ + ) + ) + + +def downgrade() -> None: + op.drop_column("districtrmap", "visible") + op.execute( + sa.text( + """ + CREATE OR REPLACE FUNCTION create_districtr_map( + map_name VARCHAR, + gerrydb_table_name VARCHAR, + num_districts INTEGER, + tiles_s3_path VARCHAR, + parent_layer_name VARCHAR, + child_layer_name VARCHAR + ) + RETURNS UUID AS $$ + DECLARE + inserted_districtr_uuid UUID; + BEGIN + INSERT INTO districtrmap ( + created_at, + uuid, + name, + gerrydb_table_name, + num_districts, + tiles_s3_path, + parent_layer, + child_layer + ) + VALUES (now(), gen_random_uuid(), $1, $2, $3, $4, $5, $6) + RETURNING uuid INTO inserted_districtr_uuid; + RETURN inserted_districtr_uuid; + END; + $$ LANGUAGE plpgsql; + """ + ) + ) diff --git a/backend/app/main.py b/backend/app/main.py index 72407b4d1..104ab2be1 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI, status, Depends, HTTPException, Query from sqlalchemy import text from sqlalchemy.exc import ProgrammingError -from sqlmodel import Session, String, select +from sqlmodel import Session, String, select, true from starlette.middleware.cors import CORSMiddleware from sqlalchemy.dialects.postgresql import insert import logging @@ -328,12 +328,12 @@ async def get_summary_stat( ), SummaryStatsP1, ), - "P4": { + "P4": ( text( "SELECT * from get_summary_stats_p4(:document_id) WHERE zone is not null" ), SummaryStatsP4, - }, + ), }[summary_stat] except KeyError: raise HTTPException( @@ -345,7 +345,7 @@ async def get_summary_stat( results = session.execute(stmt, {"document_id": document_id}).fetchall() return { "summary_stat": _summary_stat.value, - "results": [SummaryStatsModel.from_orm(row) for row in results], + "results": [SummaryStatsModel.model_validate(row) for row in results], } except ProgrammingError as e: logger.error(e) @@ -390,7 +390,7 @@ async def get_gerrydb_summary_stat( results = session.execute(stmt, {"gerrydb_table": gerrydb_table}).fetchone() return { "summary_stat": _summary_stat.value, - "results": SummaryStatsModel.from_orm(results), + "results": SummaryStatsModel.model_validate(results), } except ProgrammingError as e: logger.error(e) @@ -411,6 +411,7 @@ async def get_projects( ): gerrydb_views = session.exec( select(DistrictrMap) + .filter(DistrictrMap.visible == true()) # pyright: ignore .order_by(DistrictrMap.created_at.asc()) # pyright: ignore .offset(offset) .limit(limit) diff --git a/backend/app/models.py b/backend/app/models.py index b40c20aee..16c2d07d7 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -12,6 +12,7 @@ Column, MetaData, String, + Boolean, ) from sqlalchemy.types import ARRAY, TEXT from sqlalchemy import Float @@ -78,6 +79,7 @@ class DistrictrMap(TimeStampMixin, SQLModel, table=True): # where does this go? # when you create the view, pull the columns that you need # we'll want discrete management steps + visible: bool = Field(sa_column=Column(Boolean, nullable=False, default=True)) available_summary_stats: list[SummaryStatisticType] | None = Field( sa_column=Column(ARRAY(TEXT), nullable=True, default=[]) ) @@ -90,6 +92,7 @@ class DistrictrMapPublic(BaseModel): child_layer: str | None = None tiles_s3_path: str | None = None num_districts: int | None = None + visible: bool = True available_summary_stats: list[str] | None = None diff --git a/backend/app/sql/create_districtr_map_udf.sql b/backend/app/sql/create_districtr_map_udf.sql index 5c4d1cb49..1c1a5deae 100644 --- a/backend/app/sql/create_districtr_map_udf.sql +++ b/backend/app/sql/create_districtr_map_udf.sql @@ -4,19 +4,13 @@ CREATE OR REPLACE FUNCTION create_districtr_map( num_districts INTEGER, tiles_s3_path VARCHAR, parent_layer_name VARCHAR, - child_layer_name VARCHAR + child_layer_name VARCHAR, + visibility BOOLEAN DEFAULT TRUE ) RETURNS UUID AS $$ DECLARE inserted_districtr_uuid UUID; - extent GEOMETRY; BEGIN - EXECUTE format(' - SELECT ST_Extent(ST_Transform(geometry, 4326)) - FROM gerrydb.%I', - parent_layer_name - ) INTO extent; - INSERT INTO districtrmap ( created_at, uuid, @@ -26,7 +20,7 @@ BEGIN tiles_s3_path, parent_layer, child_layer, - extent + visible ) VALUES ( now(), @@ -37,12 +31,7 @@ BEGIN tiles_s3_path, parent_layer_name, child_layer_name, - ARRAY[ - ST_XMin(extent), - ST_YMin(extent), - ST_XMax(extent), - ST_YMax(extent) - ] + visibility ) RETURNING uuid INTO inserted_districtr_uuid; diff --git a/backend/app/utils.py b/backend/app/utils.py index adb9a8eb5..693a4a15f 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -1,7 +1,7 @@ from sqlalchemy import text from sqlalchemy import bindparam, Integer, String, Text from sqlalchemy.types import UUID -from sqlmodel import Session, Float +from sqlmodel import Session, Float, Boolean import logging @@ -19,6 +19,7 @@ def create_districtr_map( gerrydb_table_name: str | None = None, num_districts: int | None = None, tiles_s3_path: str | None = None, + visibility: bool = True, ) -> str: """ Create a new districtr map. @@ -31,6 +32,7 @@ def create_districtr_map( gerrydb_table_name: The name of the gerrydb table. num_districts: The number of districts. tiles_s3_path: The S3 path to the tiles. + visibility: The visibility of the map. Returns: The UUID of the inserted map. @@ -44,7 +46,8 @@ def create_districtr_map( :num_districts, :tiles_s3_path, :parent_layer_name, - :child_layer_name + :child_layer_name, + :visibility )""" ).bindparams( bindparam(key="map_name", type_=String), @@ -53,6 +56,7 @@ def create_districtr_map( bindparam(key="tiles_s3_path", type_=String), bindparam(key="parent_layer_name", type_=String), bindparam(key="child_layer_name", type_=String), + bindparam(key="visibility", type_=Boolean), ) (inserted_map_uuid,) = session.execute( @@ -64,6 +68,7 @@ def create_districtr_map( "tiles_s3_path": tiles_s3_path, "parent_layer_name": parent_layer_name, "child_layer_name": child_layer_name, + "visibility": visibility, }, ) return inserted_map_uuid # pyright: ignore diff --git a/backend/load_data.py b/backend/load_data.py index bc1ac99e8..3e7bbfcad 100755 --- a/backend/load_data.py +++ b/backend/load_data.py @@ -50,9 +50,7 @@ def load_sample_data(config): ) result = session.execute(exists_query).scalar() if result: - print( - f"###\nMaterialized view {view['gerrydb_table_name']} already exists.\n###" - ) + print(f"Materialized view {view['gerrydb_table_name']} already exists.") else: subprocess.run( [ diff --git a/backend/requirements.txt b/backend/requirements.txt index c8bb76dc9..795a8273e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -27,7 +27,7 @@ coverage==7.5.1 cryptography==42.0.5 dnspython==2.6.1 duckdb==0.10.1 -fastapi==0.110.0 +fastapi==0.115.5 geoalchemy2==0.15.2 h11==0.14.0 # via @@ -102,7 +102,7 @@ sqlalchemy==2.0.29 # geoalchemy2 # sqlmodel sqlmodel==0.0.16 -starlette==0.36.3 +starlette==0.41.2 # via fastapi text-unidecode==1.3 typing-extensions==4.10.0 diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 170fa242b..724089d74 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -49,6 +49,8 @@ def ks_demo_view_census_blocks_fixture(session: Session): os.path.join(FIXTURES_PATH, f"{layer}.geojson"), "-lco", "OVERWRITE=yes", + "-lco", + "GEOMETRY_NAME=geometry", "-nln", f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema ], @@ -90,6 +92,8 @@ def ks_demo_view_census_blocks_total_vap_fixture(session: Session): os.path.join(FIXTURES_PATH, f"{layer}.geojson"), "-lco", "OVERWRITE=yes", + "-lco", + "GEOMETRY_NAME=geometry", "-nln", f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema ], @@ -131,6 +135,8 @@ def ks_demo_view_census_blocks_no_pop_fixture(session: Session): os.path.join(FIXTURES_PATH, f"{layer}.geojson"), "-lco", "OVERWRITE=yes", + "-lco", + "GEOMETRY_NAME=geometry", "-nln", f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema ], @@ -452,6 +458,41 @@ def test_list_gerydb_views_offset_and_limit(client, districtr_maps): assert response.status_code == 200 data = response.json() assert len(data) == 1 + + +@pytest.fixture(name="districtr_maps_soft_deleted") +def districtr_map_soft_deleted_fixture( + session: Session, ks_demo_view_census_blocks_districtrmap: None +): + for i in range(2): + create_districtr_map( + session=session, + name=f"Districtr map {i}", + gerrydb_table_name=f"districtr_map_{i}", + parent_layer_name=GERRY_DB_FIXTURE_NAME, + visibility=bool( + i + ), # Should have one hidden (index 0) and one visible (index 1) + ) + session.commit() + + +def test_list_gerydb_views_soft_deleted_map( + client, session, districtr_maps_soft_deleted +): + response = client.get("/api/gerrydb/views") + assert response.status_code == 200 + data = response.json() + # One visible from `ks_demo_view_census_blocks_districtrmap` + # One hidden from `districtr_maps_soft_deleted` + # One visible from `districtr_maps_soft_deleted` + assert len(data) == 2 + + # Check that the hidden map is there + stmt = text("SELECT * FROM districtrmap WHERE not visible") + result = session.execute(stmt).one() + assert result is not None + assert not result[-1] # visible column is False assert data[0]["name"] == "Districtr map ks_demo_view_census_blocks" From b5d2d142ad692e3e69d3895f7642334594e1a7a4 Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 18 Nov 2024 13:10:46 -0600 Subject: [PATCH 40/50] Stop painting on mouse out/leave (#182) --- app/src/app/utils/events/mapEvents.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index b37093ca0..93b3f3197 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -120,17 +120,19 @@ export const handleMapMouseLeave = ( e: MapLayerMouseEvent | MapLayerTouchEvent, map: MapLibreMap | null ) => { - const mapStore = useMapStore.getState(); - const activeTool = mapStore.activeTool; - const sourceLayer = mapStore.mapDocument?.parent_layer; - const setHoverFeatures = mapStore.setHoverFeatures; + const {setHoverFeatures, setIsPainting} = useMapStore.getState(); setHoverFeatures([]); + setIsPainting(false) }; export const handleMapMouseOut = ( e: MapLayerMouseEvent | MapLayerTouchEvent, map: MapLibreMap | null -) => {}; +) => { + const {setHoverFeatures, setIsPainting} = useMapStore.getState(); + setHoverFeatures([]); + setIsPainting(false) +}; export const handleMapMouseMove = ( e: MapLayerMouseEvent | MapLayerTouchEvent, From 69fb2d2a2c3ac4b8b9dd2494a9968518e676454b Mon Sep 17 00:00:00 2001 From: mariogiampieri Date: Tue, 19 Nov 2024 00:06:46 -0500 Subject: [PATCH 41/50] Chart enhancements + stub of `places` and `districtingProblems` (#167) Co-authored-by: bailliekova Co-authored-by: Raphael Paul Laude Co-authored-by: nofurtherinformation --- .../app/components/sidebar/ColorPicker.tsx | 14 ++- .../sidebar/charts/HorizontalBarChart.tsx | 94 +++++++++++++--- app/src/app/store/mapStore.ts | 76 +++++++------ app/src/app/utils/api/apiHandlers.ts | 51 +++++---- app/src/app/utils/api/queries.ts | 41 ++++--- .../65a4fc0a727d_add_unshatter_udf.py | 1 - ...d35_add_available_stats_to_districtrmap.py | 6 +- backend/app/main.py | 19 +++- backend/tests/test_main.py | 30 +++-- pipelines/simple_elt/main.py | 103 +++++++++++++++++- 10 files changed, 325 insertions(+), 110 deletions(-) diff --git a/app/src/app/components/sidebar/ColorPicker.tsx b/app/src/app/components/sidebar/ColorPicker.tsx index 39f2b72f7..0d048e899 100644 --- a/app/src/app/components/sidebar/ColorPicker.tsx +++ b/app/src/app/components/sidebar/ColorPicker.tsx @@ -3,6 +3,7 @@ import {Button, Checkbox, CheckboxGroup} from '@radix-ui/themes'; import {styled} from '@stitches/react'; import * as RadioGroup from '@radix-ui/react-radio-group'; import {blackA} from '@radix-ui/colors'; +import {useMapStore} from '@/app/store/mapStore'; type ColorPickerProps = T extends true ? { @@ -27,6 +28,8 @@ export const ColorPicker = ({ colorArray, multiple, }: ColorPickerProps) => { + const mapDocument = useMapStore(state => state.mapDocument); + if (multiple) { return (
@@ -63,11 +66,12 @@ export const ColorPicker = ({ value={value !== undefined ? colorArray[value] : undefined} defaultValue={colorArray[defaultValue]} > - {colorArray.map((color, i) => ( - - - - ))} + {mapDocument && + colorArray.slice(0, mapDocument.num_districts ?? 0).map((color, i) => ( + + + + ))}
); diff --git a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx index 74ed8b4da..8fa91d975 100644 --- a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx +++ b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx @@ -1,7 +1,18 @@ import {useMapStore} from '@/app/store/mapStore'; import {Card, Flex, Heading, Text} from '@radix-ui/themes'; -import {BarChart, Bar, ResponsiveContainer, Tooltip, XAxis, YAxis, Cell} from 'recharts'; +import { + BarChart, + Bar, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, + Cell, + ReferenceLine, + Label, +} from 'recharts'; import {colorScheme} from '@/app/constants/colors'; +import {useState, useMemo} from 'react'; type TooltipInput = { active?: boolean; @@ -24,6 +35,38 @@ const CustomTooltip = ({active, payload: items}: TooltipInput) => { export const HorizontalBar = () => { const mapMetrics = useMapStore(state => state.mapMetrics); + const summaryStats = useMapStore(state => state.summaryStats); + const numDistricts = useMapStore(state => state.mapDocument?.num_districts); + const idealPopulation = summaryStats?.idealpop?.data; + const maxNumberOrderedBars = 40; // max number of zones to consider while keeping blank spaces for missing zones + const [totalExpectedBars, setTotalExpectedBars] = useState< + Array<{zone: number; total_pop: number}> + >([]); + + const calculateChartObject = () => { + if ((numDistricts ?? 0) < maxNumberOrderedBars) { + return mapMetrics && mapMetrics.data && numDistricts + ? Array.from({length: numDistricts}, (_, i) => i + 1).reduce( + (acc, district) => { + const totalPop = mapMetrics.data.reduce((acc, entry) => { + return entry.zone === district ? acc + entry.total_pop : acc; + }, 0); + return [...acc, {zone: district, total_pop: totalPop}]; + }, + [] as Array<{zone: number; total_pop: number}> + ) + : []; + } else { + return mapMetrics?.data ?? []; + } + }; + + useMemo(() => { + if (mapMetrics) { + const chartObject = calculateChartObject(); + setTotalExpectedBars(chartObject); + } + }, [mapMetrics]); if (mapMetrics?.isPending) { return
Loading...
; @@ -44,30 +87,49 @@ export const HorizontalBar = () => { return ( - Population by Zone + Population by district - - + + + idealPopulation + ? Math.round(Math.max(idealPopulation * 2, dataMax + 1000)) + : dataMax, + ]} tickFormatter={value => numberFormat.format(value)} /> - + } /> - {mapMetrics.data - .sort((a, b) => a.zone - b.zone) - .map((entry, index) => ( - - ))} + {totalExpectedBars && + totalExpectedBars + .sort((a, b) => a.zone - b.zone) + .map((entry, index) => ( + + ))} + + diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index dccfb99d7..d1864251b 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -23,19 +23,19 @@ import { getFeaturesInBbox, resetZoneColors, setZones, -} from "../utils/helpers"; -import { getRenderSubscriptions } from "./mapRenderSubs"; -import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; -import { getMapMetricsSubs } from "./metricsSubs"; -import { getMapEditSubs } from "./mapEditSubs"; -import { getQueriesResultsSubs } from "../utils/api/queries"; -import { persistOptions } from "./persistConfig"; +} from '../utils/helpers'; +import {getRenderSubscriptions} from './mapRenderSubs'; +import {getSearchParamsObersver} from '../utils/api/queryParamsListener'; +import {getMapMetricsSubs} from './metricsSubs'; +import {getMapEditSubs} from './mapEditSubs'; +import {getQueriesResultsSubs} from '../utils/api/queries'; +import {persistOptions} from './persistConfig'; import {patchReset, patchShatter, patchUnShatter} from '../utils/api/mutations'; import bbox from '@turf/bbox'; import {BLOCK_SOURCE_ID} from '../constants/layers'; import {DistrictrMapOptions} from './types'; -import { onlyUnique } from '../utils/arrays'; -import { queryClient } from '../utils/api/queryClient'; +import {onlyUnique} from '../utils/arrays'; +import {queryClient} from '../utils/api/queryClient'; const combineSetValues = (setRecord: Record>, keys?: string[]) => { const combinedSet = new Set(); // Create a new set to hold combined values @@ -80,13 +80,16 @@ export interface MapStore { setMapDocument: (mapDocument: DocumentObject) => void; summaryStats: { totpop?: { - data: P1TotPopSummaryStats - } - }, + data: P1TotPopSummaryStats; + }; + idealpop?: { + data: number; + }; + }; setSummaryStat: ( stat: T, value: MapStore['summaryStats'][T] - ) => void, + ) => void; // SHATTERING /** * A subset of IDs that a user is working on in a focused view. @@ -375,20 +378,20 @@ export const useMapStore = create( setFreshMap, resetZoneAssignments, upsertUserMap, - mapOptions + mapOptions, } = get(); if (currentMapDocument?.document_id === mapDocument.document_id) { return; } - setFreshMap(true) - resetZoneAssignments() - upsertUserMap({mapDocument}) - + setFreshMap(true); + resetZoneAssignments(); + upsertUserMap({mapDocument}); + set({ mapDocument: mapDocument, mapOptions: { ...mapOptions, - bounds: mapDocument.extent + bounds: mapDocument.extent, }, shatterIds: {parents: new Set(), children: new Set()}, }); @@ -398,9 +401,9 @@ export const useMapStore = create( set({ summaryStats: { ...get().summaryStats, - [stat]: value - } - }) + [stat]: value, + }, + }); }, // TODO: Refactor to something like this // featureStates: { @@ -581,13 +584,16 @@ export const useMapStore = create( delete shatterMappings[parent.parentId]; newShatterIds.parents.delete(parent.parentId); newZoneAssignments.set(parent.parentId, parent.zone!); - mapRef?.setFeatureState({ - source: BLOCK_SOURCE_ID, - id: parent.parentId, - sourceLayer: mapDocument?.parent_layer, - }, { - broken: false - }); + mapRef?.setFeatureState( + { + source: BLOCK_SOURCE_ID, + id: parent.parentId, + sourceLayer: mapDocument?.parent_layer, + }, + { + broken: false, + } + ); }); set({ @@ -633,8 +639,12 @@ export const useMapStore = create( userMaps.splice(i, 1, userMapData); // Replace the map at index i with the new data } else { const urlParams = new URL(window.location.href).searchParams; - urlParams.delete("document_id"); // Remove the document_id parameter - window.history.pushState({}, '', window.location.pathname + '?' + urlParams.toString()); // Update the URL without document_id + urlParams.delete('document_id'); // Remove the document_id parameter + window.history.pushState( + {}, + '', + window.location.pathname + '?' + urlParams.toString() + ); // Update the URL without document_id userMaps.splice(i, 1); } } @@ -783,8 +793,8 @@ export const useMapStore = create( selectedZone: 1, setSelectedZone: zone => set({selectedZone: zone}), zoneAssignments: new Map(), - assignmentsHash: "", - setAssignmentsHash: (hash) => set({ assignmentsHash: hash }), + assignmentsHash: '', + setAssignmentsHash: hash => set({assignmentsHash: hash}), accumulatedGeoids: new Set(), setAccumulatedGeoids: accumulatedGeoids => set({accumulatedGeoids}), setZoneAssignments: (zone, geoids) => { diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index e9886b4ae..3b475309c 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import 'maplibre-gl'; import {useMapStore} from '@/app/store/mapStore'; -import { getEntryTotal } from '../summaryStats'; +import {getEntryTotal} from '../summaryStats'; export const FormatAssignments = () => { const assignments = Array.from(useMapStore.getState().zoneAssignments.entries()).map( @@ -172,7 +172,7 @@ export interface P1ZoneSummaryStats { black_pop: number; white_pop: number; } -export type P1TotPopSummaryStats = Omit +export type P1TotPopSummaryStats = Omit; export const P1ZoneSummaryStatsKeys = [ 'other_pop', @@ -180,8 +180,8 @@ export const P1ZoneSummaryStatsKeys = [ 'amin_pop', 'nhpi_pop', 'black_pop', - 'white_pop' -] as const + 'white_pop', +] as const; export const CleanedP1ZoneSummaryStatsKeys = [ ...P1ZoneSummaryStatsKeys, @@ -192,7 +192,7 @@ export const CleanedP1ZoneSummaryStatsKeys = [ 'nhpi_pop_pct', 'black_pop_pct', 'white_pop_pct', -] as const +] as const; export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { total: number; @@ -214,23 +214,28 @@ export const getP1SummaryStats: ( ) => Promise> = async mapDocument => { if (mapDocument) { return await axios - .get>(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/P1`) + .get< + SummaryStatsResult + >(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/P1`) .then(res => { const results = res.data.results.map(row => { - const total = getEntryTotal(row) - return P1ZoneSummaryStatsKeys.reduce((acc, key) => { - acc[`${key}_pct`] = acc[key] / total; - return acc; - }, { - ...row, - total - }) as CleanedP1ZoneSummaryStats - }) - return { - ...res.data, - results - } - }) + const total = getEntryTotal(row); + return P1ZoneSummaryStatsKeys.reduce( + (acc, key) => { + acc[`${key}_pct`] = acc[key] / total; + return acc; + }, + { + ...row, + total, + } + ) as CleanedP1ZoneSummaryStats; + }); + return { + ...res.data, + results, + }; + }); } else { throw new Error('No document provided'); } @@ -246,8 +251,10 @@ export const getP1TotPopSummaryStats: ( ) => Promise> = async mapDocument => { if (mapDocument) { return await axios - .get>(`${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/P1/${mapDocument.parent_layer}`) - .then(res => res.data) + .get< + SummaryStatsResult + >(`${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/P1/${mapDocument.parent_layer}`) + .then(res => res.data); } else { throw new Error('No document provided'); } diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index 2549b3c0c..f4bbed4e6 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -13,6 +13,7 @@ import { getP1TotPopSummaryStats, P1TotPopSummaryStats, } from './apiHandlers'; +import {getEntryTotal} from '@/app/utils/summaryStats'; import {MapStore, useMapStore} from '@/app/store/mapStore'; const INITIAL_VIEW_LIMIT = 30; @@ -20,17 +21,20 @@ const INITIAL_VIEW_OFFSET = 0; /** * A utility function that returns a query function based on a nullable parameter. - * + * * @param callback - A function that takes a parameter of type ParamT and returns a Promise of type ResultT. * @param nullableParam - An optional parameter of type ParamT. If this parameter is not provided or is falsy, the function will return a function that returns null. - * - * @returns A function that, when called, will either return null (if nullableParam is not provided) + * + * @returns A function that, when called, will either return null (if nullableParam is not provided) * or call the callback function with the nullableParam and return its result. - * + * * @template ParamT - The type of the parameter that the callback function accepts. * @template ResultT - The type of the result that the callback function returns. */ -const getNullableParamQuery = (callback: (param: ParamT) => Promise, nullableParam?: ParamT) => { +const getNullableParamQuery = ( + callback: (param: ParamT) => Promise, + nullableParam?: ParamT +) => { if (!nullableParam) return () => null; return async () => await callback(nullableParam); }; @@ -71,9 +75,15 @@ export const getQueriesResultsSubs = (_useMapStore: typeof useMapStore) => { }); fetchTotPop.subscribe(response => { if (response?.data?.results) { - useMapStore.getState().setSummaryStat('totpop', { data: response.data.results}); + console.log(response?.data?.results); + useMapStore.getState().setSummaryStat('totpop', {data: response.data.results}); + useMapStore.getState().setSummaryStat('idealpop', { + data: + getEntryTotal(response.data.results) / + (useMapStore.getState().mapDocument?.num_districts ?? 1), + }); } else { - useMapStore.getState().setSummaryStat('totpop', undefined) + useMapStore.getState().setSummaryStat('totpop', undefined); } }); }; @@ -110,7 +120,7 @@ updateDocumentFromId.subscribe(mapDocument => { export const fetchAssignments = new QueryObserver(queryClient, { queryKey: ['assignments'], - queryFn: getNullableParamQuery(getAssignments) + queryFn: getNullableParamQuery(getAssignments), }); export const updateAssignments = (mapDocument: DocumentObject) => { @@ -126,10 +136,16 @@ fetchAssignments.subscribe(assignments => { } }); -export const fetchTotPop = new QueryObserver | null>(queryClient, { - queryKey: ['gerrydb_tot_pop'], - queryFn: getNullableParamQuery>(getP1TotPopSummaryStats), -}); +export const fetchTotPop = new QueryObserver | null>( + queryClient, + { + queryKey: ['gerrydb_tot_pop'], + queryFn: getNullableParamQuery< + MapStore['mapDocument'], + SummaryStatsResult + >(getP1TotPopSummaryStats), + } +); export const updateTotPop = (mapDocument: DocumentObject | null) => { fetchTotPop.setOptions({ @@ -137,4 +153,3 @@ export const updateTotPop = (mapDocument: DocumentObject | null) => { queryKey: ['gerrydb_tot_pop', mapDocument?.gerrydb_table], }); }; - diff --git a/backend/app/alembic/versions/65a4fc0a727d_add_unshatter_udf.py b/backend/app/alembic/versions/65a4fc0a727d_add_unshatter_udf.py index ab9041a44..6477c689e 100644 --- a/backend/app/alembic/versions/65a4fc0a727d_add_unshatter_udf.py +++ b/backend/app/alembic/versions/65a4fc0a727d_add_unshatter_udf.py @@ -19,7 +19,6 @@ revision: str = "65a4fc0a727d" down_revision: Union[str, None] = "dc391733e10a" branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: diff --git a/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py index 091aa3bf3..89844bac1 100644 --- a/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py +++ b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py @@ -26,7 +26,8 @@ def upgrade() -> None: ) op.execute( - sa.text(""" + sa.text( + """ UPDATE districtrmap d SET available_summary_stats = ( SELECT @@ -40,7 +41,8 @@ def upgrade() -> None: (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer)) END ) - """) + """ + ) ) diff --git a/backend/app/main.py b/backend/app/main.py index 104ab2be1..d207d0d66 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -94,6 +94,7 @@ async def create_document( {"gerrydb_table_name": data.gerrydb_table}, ) document_id = results.one()[0] # should be only one row, one column of results + stmt = ( select( Document.document_id, @@ -106,7 +107,9 @@ async def create_document( DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore - DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore + DistrictrMap.available_summary_stats.label( + "available_summary_stats" + ), # pyright: ignore ) .where(Document.document_id == document_id) .join( @@ -119,7 +122,7 @@ async def create_document( # Document id has a unique constraint so I'm not sure we need to hit the DB again here # more valuable would be to check that the assignments table doc = session.exec( - stmt + stmt, ).one() # again if we've got more than one, we have problems. if not doc.map_uuid: session.rollback() @@ -134,6 +137,7 @@ async def create_document( detail="Document creation failed", ) session.commit() + return doc @@ -218,10 +222,12 @@ async def reset_map(document_id: str, session: Session = Depends(get_session)): # Recreate the partition session.execute( - text(f""" + text( + f""" CREATE TABLE {partition_name} PARTITION OF document.assignments FOR VALUES IN ('{document_id}'); - """) + """ + ) ) session.commit() @@ -255,6 +261,7 @@ async def get_assignments(document_id: str, session: Session = Depends(get_sessi @app.get("/api/document/{document_id}", response_model=DocumentPublic) async def get_document(document_id: str, session: Session = Depends(get_session)): + stmt = ( select( Document.document_id, @@ -266,7 +273,9 @@ async def get_document(document_id: str, session: Session = Depends(get_session) DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore - DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore + DistrictrMap.available_summary_stats.label( + "available_summary_stats" + ), # pyright: ignore ) # pyright: ignore .where(Document.document_id == document_id) .join( diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 724089d74..0e092126f 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -61,13 +61,15 @@ def ks_demo_view_census_blocks_fixture(session: Session): def ks_demo_view_census_blocks_districtrmap_fixture( session: Session, ks_demo_view_census_blocks_total_vap: None ): - upsert_query = text(""" + upsert_query = text( + """ INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) ON CONFLICT (name) DO UPDATE SET updated_at = now() - """) + """ + ) session.begin() session.execute(upsert_query, {"name": GERRY_DB_FIXTURE_NAME}) @@ -104,13 +106,15 @@ def ks_demo_view_census_blocks_total_vap_fixture(session: Session): def ks_demo_view_census_blocks_total_vap_districtrmap_fixture( session: Session, ks_demo_view_census_blocks_total_vap: None ): - upsert_query = text(""" + upsert_query = text( + """ INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) ON CONFLICT (name) DO UPDATE SET updated_at = now() - """) + """ + ) session.begin() session.execute(upsert_query, {"name": GERRY_DB_TOTAL_VAP_FIXTURE_NAME}) @@ -147,13 +151,15 @@ def ks_demo_view_census_blocks_no_pop_fixture(session: Session): def ks_demo_view_census_blocks_no_pop_districtrmap_fixture( session: Session, ks_demo_view_census_blocks_no_pop: None ): - upsert_query = text(""" + upsert_query = text( + """ INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) ON CONFLICT (name) DO UPDATE SET updated_at = now() - """) + """ + ) session.begin() session.execute(upsert_query, {"name": GERRY_DB_NO_POP_FIXTURE_NAME}) @@ -513,13 +519,15 @@ def ks_demo_view_census_blocks_summary_stats(session: Session): ], ) - upsert_query = text(""" + upsert_query = text( + """ INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) ON CONFLICT (name) DO UPDATE SET updated_at = now() - """) + """ + ) session.execute(upsert_query, {"name": layer}) @@ -598,13 +606,15 @@ def ks_demo_view_census_blocks_summary_stats_p4(session: Session): ], ) - upsert_query = text(""" + upsert_query = text( + """ INSERT INTO gerrydbtable (uuid, name, updated_at) VALUES (gen_random_uuid(), :name, now()) ON CONFLICT (name) DO UPDATE SET updated_at = now() - """) + """ + ) session.execute(upsert_query, {"name": layer}) diff --git a/pipelines/simple_elt/main.py b/pipelines/simple_elt/main.py index 268a24c27..8327edb28 100644 --- a/pipelines/simple_elt/main.py +++ b/pipelines/simple_elt/main.py @@ -5,10 +5,11 @@ import os import click import logging +from urllib.request import urlretrieve from urllib.parse import urlparse from subprocess import run from typing import Iterable - +import json from files import download_and_unzip_zipfile, exists_in_s3, download_file_from_s3 from settings import settings @@ -88,7 +89,8 @@ def create_county_tiles(replace: bool = False, upload: bool = False): LOGGER.info("Creating county label centroids") label_fgb = settings.OUT_SCRATCH / f"{file_name}_label.fgb" if replace or not label_fgb.exists(): - duckdb.execute(f""" + duckdb.execute( + f""" INSTALL SPATIAL; LOAD spatial; COPY ( SELECT @@ -98,7 +100,8 @@ def create_county_tiles(replace: bool = False, upload: bool = False): FROM st_read('{fgb}') ) TO '{label_fgb}' WITH (FORMAT GDAL, DRIVER 'FlatGeobuf', SRS 'EPSG:4326') - """) + """ + ) LOGGER.info("Creating county label tiles") label_tiles = settings.OUT_SCRATCH / f"{file_name}_label.pmtiles" @@ -311,5 +314,99 @@ def merge_gerrydb_tilesets( ) +@cli.command("load-districtr-v1-places") +@click.option("--replace", is_flag=True, help="Replace existing files", default=False) +def load_districtr_v1_places(replace: bool = False) -> None: + """ + Load data from districtr_v1 endpoint to the s3 bucket for later ingestion + """ + districtr_places = urlretrieve( + "https://districtr.org/assets/data/landing_pages.json?v=2", + settings.OUT_SCRATCH / "districtr_v1_places.json", + ) + + s3_client = settings.get_s3_client() + + key = f"{S3_PREFIX}/districtr_places/districtr_v1_places.json" + print(s3_client, settings.S3_BUCKET, key) + s3_client.upload_file(districtr_places, settings.S3_BUCKET, key) + + +def upsert_places_and_problems(): + """ + Upsert places and problems from districtr_v1. + WIP/not functional port of load_dv1_places_and_problems_problems in #167 + """ + + raise NotImplementedError + + s3_client = settings.get_s3_client() + + key = f"{S3_PREFIX}/districtr_places/districtr_v1_places.json" + districtr_places = download_file_from_s3(s3_client, urlparse(key)) + + if not districtr_places: + LOGGER.error("Failed to download districtr_v1_places.json") + return + + with open(districtr_places, "r") as file: + places = json.load(file) + + for place in places: + url = place["state"] + LOGGER.info(f"Downloading problems for {url}") + try: + problems = urlretrieve( + f"https://districtr.org/assets/data/modules/{place['state'].lower()}.json", + settings.OUT_SCRATCH / f"{place['state'].lower()}_problems.json", + ) + key = ( + f"{S3_PREFIX}/districtr_problems/{place['state'].lower()}_problems.json" + ) + s3_client.upload_file(problems, settings.S3_BUCKET, key) + except Exception as e: + LOGGER.error(f"Failed to download problems for {url}: {e}") + continue + + # load districtr_v1 places and problems + load_districtr_v1_places() + load_districtr_v1_problems() + + +@cli.command("load-districtr-v1-problems") +@click.option("--replace", is_flag=True, help="Replace existing files", default=False) +def load_districtr_v1_problems(replace: bool = False) -> None: + """ + load problems definition json file for states from districtr_v1 and store in s3 bucket + """ + s3_client = settings.get_s3_client() + + # check if the districtr_places object exists in s3; if not, download it using load_districtr_v1_places + key = f"{S3_PREFIX}/districtr_places/districtr_v1_places.json" + if not exists_in_s3(s3_client, settings.S3_BUCKET, key): + load_districtr_v1_places() + + districtr_places = download_file_from_s3(s3_client, urlparse(key), replace) + + with open(districtr_places, "r") as file: + places = json.load(file) + + for place in places: + url = place["state"] + LOGGER.info(f"Downloading problems for {url}") + try: + problems = urlretrieve( + f"https://districtr.org/assets/data/modules/{place['state'].lower()}.json", + settings.OUT_SCRATCH / f"{place['state'].lower()}_problems.json", + ) + key = ( + f"{S3_PREFIX}/districtr_problems/{place['state'].lower()}_problems.json" + ) + s3_client.upload_file(problems, settings.S3_BUCKET, key) + except Exception as e: + LOGGER.error(f"Failed to download problems for {url}: {e}") + continue + + if __name__ == "__main__": cli() From 8fc81fb7d330009436e52d67bb64e1bdbb21c23f Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Tue, 19 Nov 2024 00:12:12 -0500 Subject: [PATCH 42/50] Add CLI command and utility function to update a DistrictrMap (#189) --- backend/app/models.py | 11 ++ backend/app/utils.py | 78 +++++++++++++- backend/cli.py | 198 ++++++++++++++++++++++-------------- backend/tests/test_utils.py | 32 ++++++ 4 files changed, 241 insertions(+), 78 deletions(-) diff --git a/backend/app/models.py b/backend/app/models.py index 16c2d07d7..b2f46d5c0 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -96,6 +96,17 @@ class DistrictrMapPublic(BaseModel): available_summary_stats: list[str] | None = None +class DistrictrMapUpdate(BaseModel): + gerrydb_table_name: str + name: str | None = None + parent_layer: str | None = None + child_layer: str | None = None + tiles_s3_path: str | None = None + num_districts: int | None = None + visible: bool | None = None + available_summary_stats: list[str] | None = None + + class GerryDBTable(TimeStampMixin, SQLModel, table=True): uuid: str = Field(sa_column=Column(UUIDType, unique=True, primary_key=True)) # Must correspond to the layer name in the tileset diff --git a/backend/app/utils.py b/backend/app/utils.py index 693a4a15f..f473c8153 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -1,11 +1,14 @@ -from sqlalchemy import text +from sqlalchemy import text, update from sqlalchemy import bindparam, Integer, String, Text from sqlalchemy.types import UUID from sqlmodel import Session, Float, Boolean import logging +from urllib.parse import ParseResult +import os +from app.core.config import settings -from app.models import SummaryStatisticType, UUIDType +from app.models import SummaryStatisticType, UUIDType, DistrictrMap, DistrictrMapUpdate logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -74,6 +77,41 @@ def create_districtr_map( return inserted_map_uuid # pyright: ignore +def update_districtrmap( + session: Session, + gerrydb_table_name: str, + **kwargs, +): + """ + Update a districtr map. + + Args: + session: The database session. + gerrydb_table_name: The name of the gerrydb table. + **kwargs: The fields to update. + + Returns: + The updated districtr map. + """ + data = DistrictrMapUpdate(gerrydb_table_name=gerrydb_table_name, **kwargs) + update_districtrmap = data.model_dump( + exclude_unset=True, exclude={"gerrydb_table_name"}, exclude_none=True + ) + + if not update_districtrmap.keys(): + raise KeyError("No fields to update") + + stmt = ( + update(DistrictrMap) + .where(DistrictrMap.gerrydb_table_name == data.gerrydb_table_name) # pyright: ignore + .values(update_districtrmap) + .returning(DistrictrMap) + ) + (updated_districtrmap,) = session.execute(stmt).one() + + return updated_districtrmap + + def create_shatterable_gerrydb_view( session: Session, parent_layer_name: str, @@ -273,3 +311,39 @@ def add_available_summary_stats_to_districtrmap( f"Updated available summary stats for districtr map {districtr_map_uuid} to {available_summary_stats}" ) return available_summary_stats + + +def download_file_from_s3(s3, url: ParseResult, replace=False) -> str: + """ + Download a file from S3 to the local volume path. + + Args: + s3: S3 client + url (ParseResult): URL of the file to download + replace (bool): If True, replace the file if it already exists + + Returns the path to the downloaded file. + """ + if not s3: + raise ValueError("S3 client is not available") + + file_name = url.path.lstrip("/") + logger.info("File name: %s", file_name) + object_information = s3.head_object(Bucket=url.netloc, Key=file_name) + + if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: + raise ValueError( + f"GeoPackage file {file_name} not found in S3 bucket {url.netloc}" + ) + + logger.info("Downloading GerryDB view. Got response:\n%s", object_information) + + path = os.path.join(settings.VOLUME_PATH, file_name) + + if os.path.exists(path) and not replace: + logger.info("File already exists. Skipping download.") + else: + logger.info("Downloading file...") + s3.download_file(url.netloc, file_name, path) + + return path diff --git a/backend/cli.py b/backend/cli.py index 5aab2a6fa..47f469eb8 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -2,10 +2,10 @@ import click import logging -from app.main import get_session +from app.core.db import engine from app.core.config import settings import subprocess -from urllib.parse import urlparse, ParseResult +from urllib.parse import urlparse from sqlalchemy import text from app.constants import GERRY_DB_SCHEMA from app.utils import ( @@ -14,51 +14,53 @@ create_parent_child_edges as _create_parent_child_edges, add_extent_to_districtrmap as _add_extent_to_districtrmap, add_available_summary_stats_to_districtrmap as _add_available_summary_stats_to_districtrmap, + update_districtrmap as _update_districtrmap, + download_file_from_s3, ) +from functools import wraps +from contextlib import contextmanager +from sqlmodel import Session +from typing import Callable, TypeVar, Any logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -@click.group() -def cli(): - pass +T = TypeVar("T") -def download_file_from_s3(s3, url: ParseResult, replace=False) -> str: - """ - Download a file from S3 to the local volume path. +@contextmanager +def session_scope(): + """Provide a transactional scope around a series of operations.""" + session = Session(engine) + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() - Args: - s3: S3 client - url (ParseResult): URL of the file to download - replace (bool): If True, replace the file if it already exists - Returns the path to the downloaded file. +def with_session(f: Callable[..., T]) -> Callable[..., T]: + """ + Decorator that handles database session creation and cleanup. + Compatible with Click commands. """ - if not s3: - raise ValueError("S3 client is not available") - - file_name = url.path.lstrip("/") - logger.info("File name: %s", file_name) - object_information = s3.head_object(Bucket=url.netloc, Key=file_name) - - if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: - raise ValueError( - f"GeoPackage file {file_name} not found in S3 bucket {url.netloc}" - ) - logger.info("Downloading GerryDB view. Got response:\n%s", object_information) + @wraps(f) + def decorator(*args: Any, **kwargs: Any) -> T: + with session_scope() as session: + kwargs["session"] = session + return f(*args, **kwargs) - path = os.path.join(settings.VOLUME_PATH, file_name) + return decorator - if os.path.exists(path) and not replace: - logger.info("File already exists. Skipping download.") - else: - logger.info("Downloading file...") - s3.download_file(url.netloc, file_name, path) - return path +@click.group() +def cli(): + pass @cli.command("import-gerrydb-view") @@ -66,7 +68,10 @@ def download_file_from_s3(s3, url: ParseResult, replace=False) -> str: @click.option("--gpkg", "-g", help="Path or URL to GeoPackage file", required=True) @click.option("--replace", "-f", help="Replace the file if it exists", is_flag=True) @click.option("--rm", "-r", help="Delete file after loading to postgres", is_flag=True) -def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): +@with_session +def import_gerrydb_view( + session: Session, layer: str, gpkg: str, replace: bool, rm: bool +): logger.info("Importing GerryDB view...") url = urlparse(gpkg) @@ -110,9 +115,6 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): logger.info("GerryDB view imported successfully") - _session = get_session() - session = next(_session) - upsert_query = text( """ INSERT INTO gerrydbtable (uuid, name, updated_at) @@ -123,50 +125,39 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): """ ) - try: - session.execute( - upsert_query, - { - "name": layer, - }, - ) - session.commit() - logger.info("GerryDB view upserted successfully.") - except Exception as e: - session.rollback() - logger.error("Failed to upsert GerryDB view. Got %s", e) - raise ValueError(f"Failed to upsert GerryDB view. Got {e}") - - session.close() + session.execute( + upsert_query, + { + "name": layer, + }, + ) + logger.info("GerryDB view upserted successfully.") @cli.command("create-parent-child-edges") @click.option("--districtr-map", "-d", help="Districtr map name", required=True) -def create_parent_child_edges(districtr_map: str): +@with_session +def create_parent_child_edges(session: Session, districtr_map: str): logger.info("Creating parent-child edges...") - session = next(get_session()) stmt = text( "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" ) (districtr_map_uuid,) = session.execute( stmt, params={"districtrmap_name": districtr_map} ).one() - print(f"Found districtmap uuid: {districtr_map_uuid}") + logger.info(f"Found districtmap uuid: {districtr_map_uuid}") + _create_parent_child_edges(session=session, districtr_map_uuid=districtr_map_uuid) - session.commit() logger.info("Parent-child relationship upserted successfully.") - session.close() - @cli.command("delete-parent-child-edges") @click.option("--districtr-map", "-d", help="Districtr map name", required=True) -def delete_parent_child_edges(districtr_map: str): +@with_session +def delete_parent_child_edges(session: Session, districtr_map: str): logger.info("Deleting parent-child edges...") - session = next(get_session()) - delete_query = text( """ DELETE FROM parentchildedges @@ -179,11 +170,8 @@ def delete_parent_child_edges(districtr_map: str): "districtr_map": districtr_map, }, ) - session.commit() logger.info("Parent-child relationship upserted successfully.") - session.close() - @cli.command("create-districtr-map") @click.option("--name", help="Name of the districtr map", required=True) @@ -204,7 +192,9 @@ def delete_parent_child_edges(districtr_map: str): default=None, nargs=4, ) +@with_session def create_districtr_map( + session: Session, name: str, parent_layer_name: str, child_layer_name: str | None, @@ -215,7 +205,6 @@ def create_districtr_map( bounds: list[float] | None = None, ): logger.info("Creating districtr map...") - session = next(get_session()) (districtr_map_uuid,) = _create_districtr_map( session=session, name=name, @@ -238,28 +227,88 @@ def create_districtr_map( session=session, districtr_map_uuid=districtr_map_uuid ) - session.commit() logger.info(f"Districtr map created successfully {districtr_map_uuid}") +@cli.command("update-districtr-map") +@click.option( + "--gerrydb-table-name", + "-n", + help="Name of the GerryDB table", + type=str, + required=True, +) +@click.option("--name", help="Name of the districtr map", type=str, required=False) +@click.option( + "--parent-layer-name", help="Parent gerrydb layer name", type=str, required=False +) +@click.option( + "--child-layer-name", help="Child gerrydb layer name", type=str, required=False +) +@click.option("--num-districts", help="Number of districts", type=str, required=False) +@click.option( + "--tiles-s3-path", help="S3 path to the tileset", type=str, required=False +) +@click.option("--visibility", "-v", help="Visibility", type=bool, required=False) +@click.option( + "--bounds", + "-b", + help="Bounds of the extent as `--bounds x_min y_min x_max y_max`", + required=False, + type=float, + default=None, + nargs=4, +) +@with_session +def update_districtr_map( + session: Session, + gerrydb_table_name: str, + name: str | None, + parent_layer_name: str | None, + child_layer_name: str | None, + num_districts: int | None, + tiles_s3_path: str | None, + visibility: bool = False, + bounds: list[float] | None = None, +): + logger.info("Updating districtr map...") + + _bounds = None + if bounds and len(bounds) == 4: + _bounds = bounds + + result = _update_districtrmap( + session=session, + gerrydb_table_name=gerrydb_table_name, + name=name, + parent_layer=parent_layer_name, + child_layer=child_layer_name, + num_districts=num_districts, + tiles_s3_path=tiles_s3_path, + visible=visibility, + bounds=_bounds, + ) + logger.info(f"Districtr map updated successfully {result}") + + @cli.command("create-shatterable-districtr-view") @click.option("--parent-layer-name", help="Parent gerrydb layer name", required=True) @click.option("--child-layer-name", help="Child gerrydb layer name", required=False) @click.option("--gerrydb-table-name", help="Name of the GerryDB table", required=False) +@with_session def create_shatterable_gerrydb_view( + session: Session, parent_layer_name: str, child_layer_name: str, gerrydb_table_name: str, ): logger.info("Creating materialized shatterable gerrydb view...") - session = next(get_session()) inserted_uuid = _create_shatterable_gerrydb_view( session=session, parent_layer_name=parent_layer_name, child_layer_name=child_layer_name, gerrydb_table_name=gerrydb_table_name, ) - session.commit() logger.info( f"Materialized shatterable gerrydb view created successfully {inserted_uuid}" ) @@ -276,10 +325,12 @@ def create_shatterable_gerrydb_view( default=None, nargs=4, ) -def add_extent_to_districtr_map(districtr_map: str, bounds: list[float] | None = None): +@with_session +def add_extent_to_districtr_map( + session: Session, districtr_map: str, bounds: list[float] | None = None +): logger.info(f"User provided bounds: {bounds}") - session = next(get_session()) stmt = text( "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" ) @@ -291,16 +342,13 @@ def add_extent_to_districtr_map(districtr_map: str, bounds: list[float] | None = _add_extent_to_districtrmap( session=session, districtr_map_uuid=districtr_map_uuid, bounds=bounds ) - session.commit() logger.info("Updated extent successfully.") - session.close() - @cli.command("add-available-summary-stats-to-districtr-map") @click.option("--districtr-map", "-d", help="Districtr map name", required=True) -def add_available_summary_stats_to_districtr_map(districtr_map: str): - session = next(get_session()) +@with_session +def add_available_summary_stats_to_districtr_map(session: Session, districtr_map: str): stmt = text( "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" ) @@ -313,9 +361,7 @@ def add_available_summary_stats_to_districtr_map(districtr_map: str): session=session, districtr_map_uuid=districtr_map_uuid ) - session.commit() logger.info("Updated available summary stats successfully.") - session.close() if __name__ == "__main__": diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 7e03b154a..e730ff424 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -6,10 +6,12 @@ create_parent_child_edges, add_extent_to_districtrmap, get_available_summary_stats, + update_districtrmap, ) from sqlmodel import Session import subprocess from app.constants import GERRY_DB_SCHEMA +from app.models import DistrictrMap from tests.constants import OGR2OGR_PG_CONNECTION_STRING, FIXTURES_PATH from sqlalchemy import text @@ -193,6 +195,36 @@ def test_create_districtr_map_some_nulls(session: Session, simple_parent_geos_ge session.commit() +@pytest.fixture(name="simple_parent_geos_districtrmap") +def simple_parent_geos_districtrmap_fixture( + session: Session, simple_parent_geos_gerrydb, simple_child_geos_gerrydb +): + gerrydb_name = "simple_geos_test" + (inserted_districtr_map,) = create_districtr_map( + session, + name="Simple shatterable layer", + gerrydb_table_name=gerrydb_name, + num_districts=10, + tiles_s3_path="tilesets/simple_shatterable_layer.pmtiles", + parent_layer_name="simple_parent_geos", + child_layer_name="simple_child_geos", + visibility=True, + ) + session.commit() + return gerrydb_name + + +def test_update_districtr_map(session: Session, simple_parent_geos_districtrmap): + result = update_districtrmap( + session=session, + gerrydb_table_name=simple_parent_geos_districtrmap, + visible=False, + ) + session.commit() + districtr_map = DistrictrMap.model_validate(result) + assert not districtr_map.visible + + def test_add_extent_to_districtrmap(session: Session, simple_parent_geos_gerrydb): (inserted_districtr_map,) = create_districtr_map( session, From 46f3e47a68213bedf0072914960eadb635b84e1c Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Tue, 19 Nov 2024 10:05:53 -0500 Subject: [PATCH 43/50] Two or more races pop (#190) --- app/src/app/components/sidebar/Evaluation.tsx | 96 ++----------------- app/src/app/utils/api/apiHandlers.ts | 4 + .../d90c9a1a246b_missing_two_or_more_races.py | 60 ++++++++++++ backend/app/main.py | 9 +- backend/app/models.py | 2 + .../app/sql/available_summary_stat_udf.sql | 22 +++-- backend/app/sql/summary_stats_p1.sql | 10 +- backend/app/sql/summary_stats_p1_totals.sql | 10 +- backend/app/sql/summary_stats_p4.sql | 10 +- backend/app/sql/summary_stats_p4_totals.sql | 10 +- ...o_view_census_blocks_summary_stats.geojson | 20 ++-- ...iew_census_blocks_summary_stats_p4.geojson | 20 ++-- backend/tests/test_main.py | 3 +- 13 files changed, 139 insertions(+), 137 deletions(-) create mode 100644 backend/app/alembic/versions/d90c9a1a246b_missing_two_or_more_races.py diff --git a/app/src/app/components/sidebar/Evaluation.tsx b/app/src/app/components/sidebar/Evaluation.tsx index af31c3a09..c39cebf27 100644 --- a/app/src/app/components/sidebar/Evaluation.tsx +++ b/app/src/app/components/sidebar/Evaluation.tsx @@ -3,24 +3,17 @@ import {useMapStore} from '@/app/store/mapStore'; import {useQuery} from '@tanstack/react-query'; import { CleanedP1ZoneSummaryStats, - CleanedP1ZoneSummaryStatsKeys, getP1SummaryStats, P1ZoneSummaryStats, P1ZoneSummaryStatsKeys, } from '@/app/utils/api/apiHandlers'; -import {Button, Checkbox, CheckboxGroup} from '@radix-ui/themes'; -import {Heading, Flex, Spinner, Text} from '@radix-ui/themes'; +import {Button, CheckboxGroup} from '@radix-ui/themes'; +import {Flex, Spinner, Text} from '@radix-ui/themes'; import {queryClient} from '@utils/api/queryClient'; import {formatNumber, NumberFormats} from '@/app/utils/numbers'; import {colorScheme} from '@/app/constants/colors'; -import { - getEntryTotal, - getStdDevColor, - stdDevArray, - stdDevColors, - sumArray, -} from '@utils/summaryStats'; -import {interpolateBlues, interpolateGreys} from 'd3-scale-chromatic'; +import {getEntryTotal} from '@utils/summaryStats'; +import {interpolateGreys} from 'd3-scale-chromatic'; type EvalModes = 'share' | 'count' | 'totpop'; type ColumnConfiguration> = Array<{label: string; column: keyof T}>; @@ -28,23 +21,6 @@ type EvaluationProps = { columnConfig?: ColumnConfiguration; }; -// const calculateColumn = ( -// mode: EvalModes, -// entry: P1ZoneSummaryStats, -// totals: P1ZoneSummaryStats, -// column: keyof Omit -// ) => { -// const count = entry[column]; -// switch (mode) { -// case 'count': -// return count; -// case 'pct': -// return count / entry['total']; -// case 'share': -// return count / totals[column]; -// } -// }; - const defaultColumnConfig: ColumnConfiguration = [ { label: 'White', @@ -66,6 +42,10 @@ const defaultColumnConfig: ColumnConfiguration = [ label: 'Pacific Isl.', column: 'nhpi_pop', }, + { + label: 'Two or More Races', + column: 'two_or_more_races_pop', + }, { label: 'Other', column: 'other_pop', @@ -81,10 +61,6 @@ const modeButtonConfig: Array<{label: string; value: EvalModes}> = [ label: 'Population by Count', value: 'count', }, - // { - // label: "Population by Percent of Zone", - // value: 'totpop' - // } ]; const numberFormats: Record = { @@ -104,8 +80,6 @@ const getColConfig = (evalMode: EvalModes) => { const Evaluation: React.FC = ({columnConfig = defaultColumnConfig}) => { const [evalMode, setEvalMode] = useState('share'); - // const [showAverages, setShowAverages] = useState(true); - // const [showStdDev, setShowStdDev] = useState(false); const [colorBg, setColorBg] = useState(true); const [showUnassigned, setShowUnassigned] = useState(true); @@ -152,18 +126,10 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf unassigned[`${key}_pct`] = total / unassigned[key]; unassigned[key] = total; }); - // const averages: Record = {}; - // const stdDevs: Record = {}; - // CleanedP1ZoneSummaryStatsKeys.forEach(key => { - // const values = data.results.map(row => row[key]); - // averages[key] = sumArray(values) / data.results.length; - // stdDevs[key] = stdDevArray(values); - // }); + return { unassigned, maxValues, - // averages, - // stdDevs }; }, [data?.results, totPop]); @@ -210,29 +176,9 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf setShowUnassigned(v => !v)}> Show Unassigned Population - {/* setShowAverages(v => !v)}> - Show Zone Averages - - setShowStdDev(v => !v)}> - Show Zone Std. Dev. - */} setColorBg(v => !v)}>

Color Cells By Values

- {/* {colorByStdDev && ( - - {Object.entries(stdDevColors) - .sort((a, b) => +a[0] - +b[0]) - .map(([stdev, backgroundColor], i) => ( - - {+stdev > 0 ? `+${stdev}`: stdev} - - ))} - - )} */} @@ -250,30 +196,6 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf - {/* {!!(averages && showAverages) && ( - - - Zone Averages - - {columnConfig.map((f, i) => ( - - {formatNumber(averages[columnGetter(f.column)], numberFormat)} - - ))} - - )} - {!!(stdDevs && showStdDev) && ( - - - Zone Std. Dev. - - {columnConfig.map((f, i) => ( - - {formatNumber(stdDevs[columnGetter(f.column)], numberFormat)} - - ))} - - )} */} {rows .sort((a, b) => a.zone - b.zone) .map(row => { diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 3b475309c..1f4828633 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -171,6 +171,7 @@ export interface P1ZoneSummaryStats { nhpi_pop: number; black_pop: number; white_pop: number; + two_or_more_races_pop: number; } export type P1TotPopSummaryStats = Omit; @@ -181,6 +182,7 @@ export const P1ZoneSummaryStatsKeys = [ 'nhpi_pop', 'black_pop', 'white_pop', + 'two_or_more_races_pop', ] as const; export const CleanedP1ZoneSummaryStatsKeys = [ @@ -192,6 +194,7 @@ export const CleanedP1ZoneSummaryStatsKeys = [ 'nhpi_pop_pct', 'black_pop_pct', 'white_pop_pct', + 'two_or_more_races_pop_pct', ] as const; export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { @@ -202,6 +205,7 @@ export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { nhpi_pop_pct: number; black_pop_pct: number; white_pop_pct: number; + two_or_more_races_pop_pct: number; } /** diff --git a/backend/app/alembic/versions/d90c9a1a246b_missing_two_or_more_races.py b/backend/app/alembic/versions/d90c9a1a246b_missing_two_or_more_races.py new file mode 100644 index 000000000..204c8eaeb --- /dev/null +++ b/backend/app/alembic/versions/d90c9a1a246b_missing_two_or_more_races.py @@ -0,0 +1,60 @@ +"""missing two or more races + +Revision ID: d90c9a1a246b +Revises: 2494caf34886 +Create Date: 2024-11-18 23:56:24.881723 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from app.constants import SQL_DIR +from pathlib import Path + + +# revision identifiers, used by Alembic. +revision: str = "d90c9a1a246b" +down_revision: Union[str, None] = "2494caf34886" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + for udf in [ + "available_summary_stat_udf.sql", + "summary_stats_p1.sql", + "summary_stats_p1_totals.sql", + "summary_stats_p4.sql", + "summary_stats_p4_totals.sql", + ]: + with Path(SQL_DIR, udf).open() as f: + sql = f.read() + op.execute(sql) + + op.execute( + sa.text( + """ + UPDATE districtrmap d + SET available_summary_stats = ( + SELECT + CASE WHEN d.child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer)) + END + ) + """ + ) + ) + + +def downgrade() -> None: + # Since the previous migraiton touching this logic was buggy, not going to + # to write a downgrade for it. + pass diff --git a/backend/app/main.py b/backend/app/main.py index d207d0d66..f7313a370 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -107,9 +107,7 @@ async def create_document( DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore - DistrictrMap.available_summary_stats.label( - "available_summary_stats" - ), # pyright: ignore + DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore ) .where(Document.document_id == document_id) .join( @@ -261,7 +259,6 @@ async def get_assignments(document_id: str, session: Session = Depends(get_sessi @app.get("/api/document/{document_id}", response_model=DocumentPublic) async def get_document(document_id: str, session: Session = Depends(get_session)): - stmt = ( select( Document.document_id, @@ -273,9 +270,7 @@ async def get_document(document_id: str, session: Session = Depends(get_session) DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore - DistrictrMap.available_summary_stats.label( - "available_summary_stats" - ), # pyright: ignore + DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore ) # pyright: ignore .where(Document.document_id == document_id) .join( diff --git a/backend/app/models.py b/backend/app/models.py index b2f46d5c0..0a84c0938 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -218,6 +218,7 @@ class PopulationStatsP1(BaseModel): nhpi_pop: int black_pop: int white_pop: int + two_or_more_races_pop: int class SummaryStatsP1(PopulationStatsP1): @@ -233,6 +234,7 @@ class PopulationStatsP4(BaseModel): non_hispanic_black_vap: int non_hispanic_white_vap: int non_hispanic_other_vap: int + non_hispanic_two_or_more_races_vap: int class SummaryStatsP4(PopulationStatsP4): diff --git a/backend/app/sql/available_summary_stat_udf.sql b/backend/app/sql/available_summary_stat_udf.sql index 7992c03e5..b5e4b2ec3 100644 --- a/backend/app/sql/available_summary_stat_udf.sql +++ b/backend/app/sql/available_summary_stat_udf.sql @@ -6,7 +6,7 @@ DECLARE p3 BOOLEAN; p4 BOOLEAN; BEGIN - SELECT count(column_name) = 6 INTO p1 + SELECT count(column_name) = 7 INTO p1 FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = gerrydb_table_name AND table_schema = 'gerrydb' @@ -15,10 +15,11 @@ BEGIN 'amin_pop', 'nhpi_pop', 'black_pop', - 'white_pop') + 'white_pop', + 'two_or_more_races_pop') ; - SELECT count(column_name) = 6 INTO p3 + SELECT count(column_name) = 7 INTO p3 FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = gerrydb_table_name AND table_schema = 'gerrydb' @@ -27,10 +28,11 @@ BEGIN 'amin_vap', 'nhpi_vap', 'black_vap', - 'white_vap') + 'white_vap', + 'two_or_more_races_vap') ; - SELECT count(column_name) = 7 INTO p2 + SELECT count(column_name) = 8 INTO p2 FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = gerrydb_table_name AND table_schema = 'gerrydb' @@ -40,11 +42,11 @@ BEGIN 'non_hispanic_nhpi_pop', 'non_hispanic_black_pop', 'non_hispanic_white_pop', - 'non_hispanic_other_pop' - ) + 'non_hispanic_other_pop', + 'non_hispanic_two_or_more_races_pop') ; - SELECT count(column_name) = 7 INTO p4 + SELECT count(column_name) = 8 INTO p4 FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = gerrydb_table_name AND table_schema = 'gerrydb' @@ -54,8 +56,8 @@ BEGIN 'non_hispanic_nhpi_vap', 'non_hispanic_black_vap', 'non_hispanic_white_vap', - 'non_hispanic_other_vap' - ) + 'non_hispanic_other_vap', + 'non_hispanic_two_or_more_races_vap') ; RETURN QUERY diff --git a/backend/app/sql/summary_stats_p1.sql b/backend/app/sql/summary_stats_p1.sql index bc9fa994e..2bd3d0067 100644 --- a/backend/app/sql/summary_stats_p1.sql +++ b/backend/app/sql/summary_stats_p1.sql @@ -1,4 +1,6 @@ -CREATE OR REPLACE FUNCTION get_summary_stats_p1(document_id UUID) +DROP FUNCTION IF EXISTS get_summary_stats_p1(uuid); + +CREATE FUNCTION get_summary_stats_p1(document_id UUID) RETURNS TABLE ( zone TEXT, other_pop BIGINT, @@ -6,7 +8,8 @@ RETURNS TABLE ( amin_pop BIGINT, nhpi_pop BIGINT, black_pop BIGINT, - white_pop BIGINT + white_pop BIGINT, + two_or_more_races_pop BIGINT ) AS $$ DECLARE doc_districtrmap RECORD; @@ -30,7 +33,8 @@ BEGIN SUM(COALESCE(blocks.amin_pop, 0))::BIGINT AS amin_pop, SUM(COALESCE(blocks.nhpi_pop, 0))::BIGINT AS nhpi_pop, SUM(COALESCE(blocks.black_pop, 0))::BIGINT AS black_pop, - SUM(COALESCE(blocks.white_pop, 0))::BIGINT AS white_pop + SUM(COALESCE(blocks.white_pop, 0))::BIGINT AS white_pop, + SUM(COALESCE(blocks.two_or_more_races_pop, 0))::BIGINT AS two_or_more_races_pop FROM document.assignments LEFT JOIN gerrydb.%I blocks ON blocks.path = assignments.geo_id diff --git a/backend/app/sql/summary_stats_p1_totals.sql b/backend/app/sql/summary_stats_p1_totals.sql index 9b65c9ad9..591d37932 100644 --- a/backend/app/sql/summary_stats_p1_totals.sql +++ b/backend/app/sql/summary_stats_p1_totals.sql @@ -1,11 +1,14 @@ -CREATE OR REPLACE FUNCTION get_summary_p1_totals(gerrydb_table TEXT) +DROP FUNCTION IF EXISTS get_summary_p1_totals(TEXT); + +CREATE FUNCTION get_summary_p1_totals(gerrydb_table TEXT) RETURNS TABLE ( other_pop BIGINT, asian_pop BIGINT, amin_pop BIGINT, nhpi_pop BIGINT, black_pop BIGINT, - white_pop BIGINT + white_pop BIGINT, + two_or_more_races_pop BIGINT ) AS $$ DECLARE table_exists BOOLEAN; @@ -31,7 +34,8 @@ BEGIN SUM(COALESCE(amin_pop, 0))::BIGINT AS amin_pop, SUM(COALESCE(nhpi_pop, 0))::BIGINT AS nhpi_pop, SUM(COALESCE(black_pop, 0))::BIGINT AS black_pop, - SUM(COALESCE(white_pop, 0))::BIGINT AS white_pop + SUM(COALESCE(white_pop, 0))::BIGINT AS white_pop, + SUM(COALESCE(two_or_more_races_pop, 0))::BIGINT AS two_or_more_races_pop FROM gerrydb.%I ', $1); RETURN QUERY EXECUTE sql_query; diff --git a/backend/app/sql/summary_stats_p4.sql b/backend/app/sql/summary_stats_p4.sql index 1c75bfbcf..abfebc87e 100644 --- a/backend/app/sql/summary_stats_p4.sql +++ b/backend/app/sql/summary_stats_p4.sql @@ -1,4 +1,6 @@ -CREATE OR REPLACE FUNCTION get_summary_stats_p4(document_id UUID) +DROP FUNCTION IF EXISTS get_summary_stats_p4(uuid); + +CREATE FUNCTION get_summary_stats_p4(document_id UUID) RETURNS TABLE ( zone TEXT, hispanic_vap BIGINT, @@ -7,7 +9,8 @@ RETURNS TABLE ( non_hispanic_nhpi_vap BIGINT, non_hispanic_black_vap BIGINT, non_hispanic_white_vap BIGINT, - non_hispanic_other_vap BIGINT + non_hispanic_other_vap BIGINT, + non_hispanic_two_or_more_races_vap BIGINT ) AS $$ DECLARE doc_districtrmap RECORD; @@ -32,7 +35,8 @@ BEGIN SUM(COALESCE(blocks.non_hispanic_nhpi_vap, 0))::BIGINT AS non_hispanic_nhpi_vap, SUM(COALESCE(blocks.non_hispanic_black_vap, 0))::BIGINT AS non_hispanic_black_vap, SUM(COALESCE(blocks.non_hispanic_white_vap, 0))::BIGINT AS non_hispanic_white_vap, - SUM(COALESCE(blocks.non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap + SUM(COALESCE(blocks.non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap, + SUM(COALESCE(blocks.non_hispanic_two_or_more_races_vap, 0))::BIGINT AS non_hispanic_two_or_more_races_vap FROM document.assignments LEFT JOIN gerrydb.%I blocks ON blocks.path = assignments.geo_id diff --git a/backend/app/sql/summary_stats_p4_totals.sql b/backend/app/sql/summary_stats_p4_totals.sql index a9c57f12d..6ab9cfa56 100644 --- a/backend/app/sql/summary_stats_p4_totals.sql +++ b/backend/app/sql/summary_stats_p4_totals.sql @@ -1,4 +1,6 @@ -CREATE OR REPLACE FUNCTION get_summary_p4_totals(gerrydb_table TEXT) +DROP FUNCTION IF EXISTS get_summary_p4_totals(TEXT); + +CREATE FUNCTION get_summary_p4_totals(gerrydb_table TEXT) RETURNS TABLE ( hispanic_vap BIGINT, non_hispanic_asian_vap BIGINT, @@ -6,7 +8,8 @@ RETURNS TABLE ( non_hispanic_nhpi_vap BIGINT, non_hispanic_black_vap BIGINT, non_hispanic_white_vap BIGINT, - non_hispanic_other_vap BIGINT + non_hispanic_other_vap BIGINT, + non_hispanic_two_or_more_races_vap BIGINT ) AS $$ DECLARE table_exists BOOLEAN; @@ -33,7 +36,8 @@ BEGIN SUM(COALESCE(non_hispanic_nhpi_vap, 0))::BIGINT AS non_hispanic_nhpi_vap, SUM(COALESCE(non_hispanic_black_vap, 0))::BIGINT AS non_hispanic_black_vap, SUM(COALESCE(non_hispanic_white_vap, 0))::BIGINT AS non_hispanic_white_vap, - SUM(COALESCE(non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap + SUM(COALESCE(non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap, + SUM(COALESCE(non_hispanic_two_or_more_races_vap, 0))::BIGINT AS non_hispanic_two_or_more_races_vap FROM gerrydb.%I ', $1); RETURN QUERY EXECUTE sql_query; diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson index 572cefc2e..e6ecf9d23 100644 --- a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson @@ -3,15 +3,15 @@ "name": "SELECT", "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, "features": [ -{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "other_pop": 12, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "other_pop": 1, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "other_pop": 5, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "other_pop": 24, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "other_pop": 32, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 12, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 1, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 5, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 24, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 32, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "two_or_more_races_pop": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } ] } diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson index d84d67435..f35f9fc30 100644 --- a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson @@ -3,15 +3,15 @@ "name": "SELECT", "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, "features": [ -{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 12, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 1, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 5, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 24, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 32, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, -{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 12, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 1, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 5, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 24, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 32, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "non_hispanic_two_or_more_races_vap": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } ] } diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 0e092126f..94b70ff8e 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -537,9 +537,10 @@ def ks_demo_view_census_blocks_summary_stats(session: Session): parent_layer_name=layer, gerrydb_table_name=layer, ) - add_available_summary_stats_to_districtrmap( + summary_stats = add_available_summary_stats_to_districtrmap( session=session, districtr_map_uuid=districtr_map_uuid ) + assert summary_stats == ["P1"], f"Expected P1 to be available, got {summary_stats}" session.commit() From 515e6de6601dd3107aee717246bfb12921c24e46 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 20 Nov 2024 11:30:22 -0500 Subject: [PATCH 44/50] Add P4 Summary (#198) --- app/src/app/components/sidebar/Evaluation.tsx | 69 +++++++++--- app/src/app/store/mapStore.ts | 3 +- app/src/app/utils/api/apiHandlers.ts | 100 +++++++++++++++--- app/src/app/utils/api/queries.ts | 41 ++----- app/src/app/utils/summaryStats.ts | 9 +- 5 files changed, 160 insertions(+), 62 deletions(-) diff --git a/app/src/app/components/sidebar/Evaluation.tsx b/app/src/app/components/sidebar/Evaluation.tsx index c39cebf27..3704b9cd2 100644 --- a/app/src/app/components/sidebar/Evaluation.tsx +++ b/app/src/app/components/sidebar/Evaluation.tsx @@ -3,9 +3,12 @@ import {useMapStore} from '@/app/store/mapStore'; import {useQuery} from '@tanstack/react-query'; import { CleanedP1ZoneSummaryStats, - getP1SummaryStats, + CleanedP4ZoneSummaryStats, + getSummaryStats, P1ZoneSummaryStats, P1ZoneSummaryStatsKeys, + P4ZoneSummaryStats, + P4ZoneSummaryStatsKeys, } from '@/app/utils/api/apiHandlers'; import {Button, CheckboxGroup} from '@radix-ui/themes'; import {Flex, Spinner, Text} from '@radix-ui/themes'; @@ -17,11 +20,8 @@ import {interpolateGreys} from 'd3-scale-chromatic'; type EvalModes = 'share' | 'count' | 'totpop'; type ColumnConfiguration> = Array<{label: string; column: keyof T}>; -type EvaluationProps = { - columnConfig?: ColumnConfiguration; -}; -const defaultColumnConfig: ColumnConfiguration = [ +const p1ColumnConfig: ColumnConfiguration = [ { label: 'White', column: 'white_pop', @@ -52,6 +52,17 @@ const defaultColumnConfig: ColumnConfiguration = [ }, ]; +const p4ColumnConfig: ColumnConfiguration = [ + {column: 'hispanic_vap', label: 'Hispanic'}, + {column: 'non_hispanic_asian_vap', label: 'Non-hispanic Asian'}, + {column: 'non_hispanic_amin_vap', label: 'Non-hispanic Amin.'}, + {column: 'non_hispanic_nhpi_vap', label: 'Non-hispanic NHPI'}, + {column: 'non_hispanic_black_vap', label: 'Non-hispanic Black'}, + {column: 'non_hispanic_white_vap', label: 'Non-hispanic White'}, + {column: 'non_hispanic_other_vap', label: 'Non-hispanic Other'}, + {column: 'non_hispanic_two_or_more_races_vap', label: 'Non-hispanic 2+ Races'}, +]; + const modeButtonConfig: Array<{label: string; value: EvalModes}> = [ { label: 'Population by Share', @@ -72,13 +83,14 @@ const numberFormats: Record = { const getColConfig = (evalMode: EvalModes) => { switch (evalMode) { case 'share': - return (col: keyof P1ZoneSummaryStats) => `${col}_pct` as keyof CleanedP1ZoneSummaryStats; + return (col: keyof P1ZoneSummaryStats | keyof P4ZoneSummaryStats) => + `${col}_pct` as keyof CleanedP1ZoneSummaryStats | keyof CleanedP4ZoneSummaryStats; default: - return (col: keyof P1ZoneSummaryStats) => col; + return (col: keyof P1ZoneSummaryStats | keyof P4ZoneSummaryStats) => col; } }; -const Evaluation: React.FC = ({columnConfig = defaultColumnConfig}) => { +const Evaluation: React.FC = () => { const [evalMode, setEvalMode] = useState('share'); const [colorBg, setColorBg] = useState(true); const [showUnassigned, setShowUnassigned] = useState(true); @@ -89,10 +101,37 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf const mapDocument = useMapStore(state => state.mapDocument); const assignmentsHash = useMapStore(state => state.assignmentsHash); + const columnConfig = useMemo(() => { + const summaryType = mapDocument?.available_summary_stats?.[0]; + + switch (summaryType) { + case 'P1': + return p1ColumnConfig; + case 'P4': + return p4ColumnConfig; + default: + return []; + } + }, [mapDocument]); + + const ZoneSummaryStatsKeys = useMemo(() => { + const summaryType = mapDocument?.available_summary_stats?.[0]; + + switch (summaryType) { + case 'P1': + return P1ZoneSummaryStatsKeys; + case 'P4': + return P4ZoneSummaryStatsKeys; + default: + return []; + } + }, [mapDocument]); + const {data, error, isLoading} = useQuery( { - queryKey: ['p1SummaryStats', mapDocument, assignmentsHash], - queryFn: () => mapDocument && getP1SummaryStats(mapDocument), + queryKey: ['SummaryStats', mapDocument, assignmentsHash], + queryFn: () => + mapDocument && getSummaryStats(mapDocument, mapDocument.available_summary_stats?.[0]), enabled: !!mapDocument, staleTime: 0, placeholderData: previousData => previousData, @@ -116,11 +155,13 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf zone: -999, total: getEntryTotal(totPop), }; - P1ZoneSummaryStatsKeys.forEach(key => { + ZoneSummaryStatsKeys.forEach(key => { let total = unassigned[key]; maxValues[key] = -Math.pow(10, 12); data.results.forEach(row => { + // @ts-ignore total -= row[key]; + // @ts-ignore maxValues[key] = Math.max(row[key], maxValues[key]); }); unassigned[`${key}_pct`] = total / unassigned[key]; @@ -215,6 +256,7 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf {columnConfig.map((f, i) => { const column = columnGetter(f.column); const colorValue = + // @ts-ignore evalMode === 'count' ? row[column] / maxValues[column] : row[column]; const backgroundColor = colorBg && !isUnassigned @@ -230,7 +272,10 @@ const Evaluation: React.FC = ({columnConfig = defaultColumnConf }} key={i} > - {formatNumber(row[column], numberFormat)} + { + // @ts-ignore; + formatNumber(row[column], numberFormat) + } ); })} diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index d1864251b..dadff7b8f 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -9,6 +9,7 @@ import { DistrictrMap, DocumentObject, P1TotPopSummaryStats, + P4TotPopSummaryStats, ShatterResult, ZonePopulation, } from '../utils/api/apiHandlers'; @@ -80,7 +81,7 @@ export interface MapStore { setMapDocument: (mapDocument: DocumentObject) => void; summaryStats: { totpop?: { - data: P1TotPopSummaryStats; + data: P1TotPopSummaryStats | P4TotPopSummaryStats; }; idealpop?: { data: number; diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 1f4828633..66db4a3ff 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -208,23 +208,92 @@ export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { two_or_more_races_pop_pct: number; } + /** - * Get P1 zone stats from the server. + * P4ZoneSummaryStats + * + * @interface + * @property {number} zone - The zone. + * @property {number} total_pop - The total population. + */ +export interface P4ZoneSummaryStats { + zone: number; + hispanic_vap: number, + non_hispanic_asian_vap: number, + non_hispanic_amin_vap: number, + non_hispanic_nhpi_vap: number, + non_hispanic_black_vap: number, + non_hispanic_white_vap: number, + non_hispanic_other_vap: number, + non_hispanic_two_or_more_races_vap: number +} +export type P4TotPopSummaryStats = Omit; + +export const P4ZoneSummaryStatsKeys = [ + 'hispanic_vap', + 'non_hispanic_asian_vap', + 'non_hispanic_amin_vap', + 'non_hispanic_nhpi_vap', + 'non_hispanic_black_vap', + 'non_hispanic_white_vap', + 'non_hispanic_other_vap', + 'non_hispanic_two_or_more_races_vap' +] as const; + +export const CleanedP4ZoneSummaryStatsKeys = [ + ...P4ZoneSummaryStatsKeys, + 'total', + 'hispanic_vap', + 'non_hispanic_asian_vap', + 'non_hispanic_amin_vap', + 'non_hispanic_nhpi_vap', + 'non_hispanic_black_vap', + 'non_hispanic_white_vap', + 'non_hispanic_other_vap', + 'non_hispanic_two_or_more_races_vap' +] as const; + +export interface CleanedP4ZoneSummaryStats extends P4ZoneSummaryStats { + total: number; + hispanic_vap: number, + non_hispanic_asian_vap: number, + non_hispanic_amin_vap: number, + non_hispanic_nhpi_vap: number, + non_hispanic_black_vap: number, + non_hispanic_white_vap: number, + non_hispanic_other_vap: number, + non_hispanic_two_or_more_races_vap: number +} + +/** + * Get zone stats from the server. * @param mapDocument - DocumentObject, the document object - * @returns Promise + * @param summaryType - string, the summary type + * @returns Promise */ -export const getP1SummaryStats: ( - mapDocument: DocumentObject -) => Promise> = async mapDocument => { - if (mapDocument) { +export const getSummaryStats: ( + mapDocument: DocumentObject, + summaryType: string | null | undefined +) => Promise> = async (mapDocument, summaryType) => { + if (mapDocument && summaryType) { return await axios .get< - SummaryStatsResult - >(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/P1`) + SummaryStatsResult + >(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/${summaryType}`) .then(res => { const results = res.data.results.map(row => { const total = getEntryTotal(row); - return P1ZoneSummaryStatsKeys.reduce( + + const zoneSummaryStatsKeys = (() => { + switch(summaryType) { + case "P1": return P1ZoneSummaryStatsKeys; + case "P4": return P4ZoneSummaryStatsKeys; + default: throw new Error('Invalid summary type'); + } + })(); + + + return zoneSummaryStatsKeys.reduce( (acc, key) => { acc[`${key}_pct`] = acc[key] / total; return acc; @@ -250,14 +319,15 @@ export const getP1SummaryStats: ( * @param mapDocument - DocumentObject, the document object * @returns Promise */ -export const getP1TotPopSummaryStats: ( - mapDocument: DocumentObject | null -) => Promise> = async mapDocument => { - if (mapDocument) { +export const getTotPopSummaryStats: ( + mapDocument: DocumentObject | null, + summaryType: string | null | undefined +) => Promise> = async (mapDocument, summaryType) => { + if (mapDocument && summaryType) { return await axios .get< - SummaryStatsResult - >(`${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/P1/${mapDocument.parent_layer}`) + SummaryStatsResult + >(`${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/${summaryType}/${mapDocument.parent_layer}`) .then(res => res.data); } else { throw new Error('No document provided'); diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index f4bbed4e6..949d71cd8 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -10,35 +10,16 @@ import { getZonePopulations, ZonePopulation, SummaryStatsResult, - getP1TotPopSummaryStats, + getTotPopSummaryStats, P1TotPopSummaryStats, + P4TotPopSummaryStats, } from './apiHandlers'; import {getEntryTotal} from '@/app/utils/summaryStats'; -import {MapStore, useMapStore} from '@/app/store/mapStore'; +import {useMapStore} from '@/app/store/mapStore'; const INITIAL_VIEW_LIMIT = 30; const INITIAL_VIEW_OFFSET = 0; -/** - * A utility function that returns a query function based on a nullable parameter. - * - * @param callback - A function that takes a parameter of type ParamT and returns a Promise of type ResultT. - * @param nullableParam - An optional parameter of type ParamT. If this parameter is not provided or is falsy, the function will return a function that returns null. - * - * @returns A function that, when called, will either return null (if nullableParam is not provided) - * or call the callback function with the nullableParam and return its result. - * - * @template ParamT - The type of the parameter that the callback function accepts. - * @template ResultT - The type of the result that the callback function returns. - */ -const getNullableParamQuery = ( - callback: (param: ParamT) => Promise, - nullableParam?: ParamT -) => { - if (!nullableParam) return () => null; - return async () => await callback(nullableParam); -}; - export const mapMetrics = new QueryObserver(queryClient, { queryKey: ['_zonePopulations'], queryFn: skipToken, @@ -120,12 +101,12 @@ updateDocumentFromId.subscribe(mapDocument => { export const fetchAssignments = new QueryObserver(queryClient, { queryKey: ['assignments'], - queryFn: getNullableParamQuery(getAssignments), + queryFn: () => getAssignments(useMapStore.getState().mapDocument), }); export const updateAssignments = (mapDocument: DocumentObject) => { fetchAssignments.setOptions({ - queryFn: getNullableParamQuery(getAssignments, mapDocument), + queryFn: () => getAssignments(mapDocument), queryKey: ['assignments', performance.now()], }); }; @@ -136,20 +117,20 @@ fetchAssignments.subscribe(assignments => { } }); -export const fetchTotPop = new QueryObserver | null>( +export const fetchTotPop = new QueryObserver | null>( queryClient, { queryKey: ['gerrydb_tot_pop'], - queryFn: getNullableParamQuery< - MapStore['mapDocument'], - SummaryStatsResult - >(getP1TotPopSummaryStats), + queryFn: () => getTotPopSummaryStats(useMapStore.getState().mapDocument, useMapStore.getState().mapDocument?.available_summary_stats?.[0]), } ); export const updateTotPop = (mapDocument: DocumentObject | null) => { fetchTotPop.setOptions({ - queryFn: getNullableParamQuery(getP1TotPopSummaryStats, mapDocument), + queryFn: () => getTotPopSummaryStats(mapDocument, mapDocument?.available_summary_stats?.[0]), queryKey: ['gerrydb_tot_pop', mapDocument?.gerrydb_table], }); }; + + +// getNullableParamQuery(, mapDocument, mapDocument?.available_summary_stats?.[0]), diff --git a/app/src/app/utils/summaryStats.ts b/app/src/app/utils/summaryStats.ts index 5b663967a..e9322bd9a 100644 --- a/app/src/app/utils/summaryStats.ts +++ b/app/src/app/utils/summaryStats.ts @@ -1,8 +1,9 @@ -import { P1ZoneSummaryStats } from "./api/apiHandlers"; +import { P1ZoneSummaryStats, P4ZoneSummaryStats } from "./api/apiHandlers"; -export const getEntryTotal = (entry: Omit) => +export const getEntryTotal = (entry: Omit) => Object.entries(entry).reduce((total, [key, value]) => { if (key !== 'zone') { + // @ts-ignore return total + value; // Sum values of properties except 'zone' } return total; // Return total unchanged for 'zone' @@ -26,6 +27,6 @@ export const stdDevColors = { export const getStdDevColor = (value: number) => { const floorValue = value > 0 ? Math.floor(value) : Math.ceil(value) - const cleanValue= (floorValue < -2 ? -2 : floorValue > 2 ? 2 : floorValue) as keyof typeof stdDevColors + const cleanValue= (floorValue < -2 ? -2 : floorValue > 2 ? 2 : floorValue) as keyof typeof stdDevColors return stdDevColors[cleanValue] || 'none' -} \ No newline at end of file +} From ae7187d35269e008f7a3d7c54d058551cc456ae3 Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 20 Nov 2024 14:21:06 -0600 Subject: [PATCH 45/50] Update line layer widths (#175) --- app/src/app/constants/basemapLayers.ts | 5 +++-- app/src/app/constants/layers.ts | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/app/src/app/constants/basemapLayers.ts b/app/src/app/constants/basemapLayers.ts index 8117df2b7..8444b4945 100644 --- a/app/src/app/constants/basemapLayers.ts +++ b/app/src/app/constants/basemapLayers.ts @@ -1100,8 +1100,9 @@ export const BASEMAP_LAYERS: LayerSpecification[] = [ source: 'counties', 'source-layer': 'tl_2023_us_county', paint: { - 'line-color': '#555', - 'line-width': ['interpolate', ['exponential', 1.6], ['zoom'], 6, 0, 9, 0.75, 18, 1], + 'line-color': '#333', + 'line-opacity': 0.8, + 'line-width': ['interpolate', ['exponential', 1.6], ['zoom'], 6, .625, 9, 1.625, 18, 2.25], }, }, { diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 93461404b..27acb445a 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -20,8 +20,9 @@ export const BLOCK_HOVER_LAYER_ID = `${BLOCK_LAYER_ID}-hover`; export const BLOCK_HOVER_LAYER_ID_CHILD = `${BLOCK_LAYER_ID_CHILD}-hover`; export const INTERACTIVE_LAYERS = [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CHILD]; +export const LINE_LAYERS = [BLOCK_LAYER_ID, BLOCK_LAYER_ID_CHILD] as const -export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; +export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; export const CHILD_LAYERS = [ BLOCK_LAYER_ID_CHILD, @@ -52,6 +53,11 @@ ZONE_ASSIGNMENT_STYLE_DYNAMIC.push('#cecece'); // @ts-ignore export const ZONE_ASSIGNMENT_STYLE: ExpressionSpecification = ZONE_ASSIGNMENT_STYLE_DYNAMIC; +const LAYER_LINE_WIDTHS = { + [BLOCK_LAYER_ID]: 2, + [BLOCK_LAYER_ID_CHILD]: 1 +} + export function getLayerFilter(layerId: string, _shatterIds?: MapStore['shatterIds']) { const shatterIds = _shatterIds || useMapStore.getState().shatterIds; const isChildLayer = CHILD_LAYERS.includes(layerId); @@ -180,10 +186,14 @@ export function getHighlightLayerSpecification( }, }; } + + export function getBlocksLayerSpecification( sourceLayer: string, - layerId: string + layerId: typeof LINE_LAYERS[number] ): LayerSpecification { + const lineWidth = LAYER_LINE_WIDTHS[layerId] + const layerSpec: LayerSpecification = { id: layerId, source: BLOCK_SOURCE_ID, @@ -194,8 +204,9 @@ export function getBlocksLayerSpecification( }, paint: { 'line-opacity': 0.8, - 'line-color': '#cecece', // Default color - 'line-width': 1, // Default width + // 'line-color': '#aaaaaa', // Default color + 'line-color': ['interpolate', ['exponential', 1.6], ['zoom'], 6, '#aaa', 9, '#777', 14, '#333'], + 'line-width': ['interpolate', ['exponential', 1.6], ['zoom'], 6, lineWidth*.125, 9, lineWidth*.35, 14, lineWidth], }, }; if (CHILD_LAYERS.includes(layerId)) { From 343f94c9d976ad304a2c1b7b74bda4df1be97ea5 Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 21 Nov 2024 08:46:27 -0600 Subject: [PATCH 46/50] Various bug fixes (#185) --- app/package-lock.json | 120 ++++++++++++++++++ app/package.json | 3 +- app/src/app/components/ErrorNotification.tsx | 78 ++++++++++++ .../app/components/sidebar/ColorPicker.tsx | 4 +- app/src/app/map/page.tsx | 2 + app/src/app/store/mapRenderSubs.ts | 11 +- app/src/app/store/mapStore.ts | 38 +++++- app/src/app/utils/api/queries.ts | 14 ++ app/src/app/utils/events/mapEvents.ts | 9 +- 9 files changed, 267 insertions(+), 12 deletions(-) create mode 100644 app/src/app/components/ErrorNotification.tsx diff --git a/app/package-lock.json b/app/package-lock.json index 4d1b8d6a9..107d95222 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -12,6 +12,7 @@ "@emotion/styled": "^11.13.0", "@radix-ui/colors": "^3.0.0", "@radix-ui/react-icons": "^1.3.0", + "@radix-ui/react-toast": "^1.2.2", "@radix-ui/themes": "^3.0.5", "@sentry/nextjs": "^8.26.0", "@stitches/react": "^1.2.8", @@ -3280,6 +3281,125 @@ } } }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.2.tgz", + "integrity": "sha512-Z6pqSzmAP/bFJoqMAston4eSNa+ud44NSZTiZUmUen+IOZ5nBY8kzuU5WDBVyFXPtcW6yUalOHsxM/BP6Sv8ww==", + "dependencies": { + "@radix-ui/primitive": "1.1.0", + "@radix-ui/react-collection": "1.1.0", + "@radix-ui/react-compose-refs": "1.1.0", + "@radix-ui/react-context": "1.1.1", + "@radix-ui/react-dismissable-layer": "1.1.1", + "@radix-ui/react-portal": "1.1.2", + "@radix-ui/react-presence": "1.1.1", + "@radix-ui/react-primitive": "2.0.0", + "@radix-ui/react-use-callback-ref": "1.1.0", + "@radix-ui/react-use-controllable-state": "1.1.0", + "@radix-ui/react-use-layout-effect": "1.1.0", + "@radix-ui/react-visually-hidden": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast/node_modules/@radix-ui/react-context": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz", + "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast/node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.1.tgz", + "integrity": "sha512-QSxg29lfr/xcev6kSz7MAlmDnzbP1eI/Dwn3Tp1ip0KT5CUELsxkekFEMVBEoykI3oV39hKT4TKZzBNMbcTZYQ==", + "dependencies": { + "@radix-ui/primitive": "1.1.0", + "@radix-ui/react-compose-refs": "1.1.0", + "@radix-ui/react-primitive": "2.0.0", + "@radix-ui/react-use-callback-ref": "1.1.0", + "@radix-ui/react-use-escape-keydown": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast/node_modules/@radix-ui/react-portal": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.2.tgz", + "integrity": "sha512-WeDYLGPxJb/5EGBoedyJbT0MpoULmwnIPMJMSldkuiMsBAv7N1cRdsTWZWht9vpPOiN3qyiGAtbK2is47/uMFg==", + "dependencies": { + "@radix-ui/react-primitive": "2.0.0", + "@radix-ui/react-use-layout-effect": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast/node_modules/@radix-ui/react-presence": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.1.tgz", + "integrity": "sha512-IeFXVi4YS1K0wVZzXNrbaaUvIJ3qdY+/Ih4eHFhWA9SwGR9UDX7Ck8abvL57C4cv3wwMvUE0OG69Qc3NCcTe/A==", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.0", + "@radix-ui/react-use-layout-effect": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-toggle": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.0.tgz", diff --git a/app/package.json b/app/package.json index 90abcf044..18697ec80 100644 --- a/app/package.json +++ b/app/package.json @@ -14,6 +14,7 @@ "@emotion/styled": "^11.13.0", "@radix-ui/colors": "^3.0.0", "@radix-ui/react-icons": "^1.3.0", + "@radix-ui/react-toast": "^1.2.2", "@radix-ui/themes": "^3.0.5", "@sentry/nextjs": "^8.26.0", "@stitches/react": "^1.2.8", @@ -52,4 +53,4 @@ "tailwindcss": "^3.4.1", "typescript": "^5" } -} \ No newline at end of file +} diff --git a/app/src/app/components/ErrorNotification.tsx b/app/src/app/components/ErrorNotification.tsx new file mode 100644 index 000000000..16a55d9fe --- /dev/null +++ b/app/src/app/components/ErrorNotification.tsx @@ -0,0 +1,78 @@ +"use client" +import {useEffect, useState} from 'react'; +import {useMapStore} from '../store/mapStore'; +import {AlertDialog, Button, Flex, Text} from '@radix-ui/themes'; +import * as Toast from '@radix-ui/react-toast'; + +export const ErrorNotification = () => { + const errorNotification = useMapStore(state => state.errorNotification); + const [errorUiActive, setErrorUiActive] = useState(true); + + useEffect(() => { + errorNotification.message && setErrorUiActive(true); + }, [errorNotification]); + + if (!errorUiActive || !errorNotification.message || !errorNotification.severity) { + return null; + } + const ErrorDescription = () => ( + <> + + {errorNotification.message} + + + {!!errorNotification?.id && Error ID: {errorNotification.id}} + + + {!!errorNotification?.severity && Severity: {errorNotification.severity}} + + + ); + + switch (errorNotification.severity) { + case 1: + return ( + + + Error + + + + + + + + + + + + ); + case 2: + return ( + + + + Error + + + + + + + + + + + ); + default: + return null; + } +}; diff --git a/app/src/app/components/sidebar/ColorPicker.tsx b/app/src/app/components/sidebar/ColorPicker.tsx index 0d048e899..e1e02439c 100644 --- a/app/src/app/components/sidebar/ColorPicker.tsx +++ b/app/src/app/components/sidebar/ColorPicker.tsx @@ -41,7 +41,7 @@ export const ColorPicker = ({ onValueChange(indices, values); }} > - {colorArray.map((color, i) => ( + {!!mapDocument && colorArray.slice(0, mapDocument.num_districts ?? 0).map((color, i) => ( ({ value={value !== undefined ? colorArray[value] : undefined} defaultValue={colorArray[defaultValue]} > - {mapDocument && + {!!mapDocument && colorArray.slice(0, mapDocument.num_districts ?? 0).map((color, i) => ( diff --git a/app/src/app/map/page.tsx b/app/src/app/map/page.tsx index e977bb5c6..3124dbe53 100644 --- a/app/src/app/map/page.tsx +++ b/app/src/app/map/page.tsx @@ -6,6 +6,7 @@ import SidebarComponent from '../components/sidebar/Sidebar'; import MobileTopNav from '../components/sidebar/MobileTopNav'; import {QueryClientProvider} from '@tanstack/react-query'; import {queryClient} from '../utils/api/queryClient'; +import { ErrorNotification } from '../components/ErrorNotification'; export default function Map() { if (queryClient) { @@ -16,6 +17,7 @@ export default function Map() { +
); diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 2a1a98b52..1256da961 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -120,7 +120,8 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { [...PARENT_LAYERS, ...CHILD_LAYERS].forEach(layerId => { const isHover = layerId.includes('hover'); const isParent = PARENT_LAYERS.includes(layerId); - isHover && + + if (isHover && mapRef.getLayer(layerId)) { mapRef.setPaintProperty( layerId, 'fill-opacity', @@ -129,6 +130,7 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { isParent ? shatterIds.parents : undefined ) ); + } }); const [lockPaintedAreas, prevLockPaintedAreas] = [curr[6], prev[6]]; const sameLockedAreas = @@ -267,7 +269,7 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { [...PARENT_LAYERS, ...CHILD_LAYERS].forEach(layerId => { const isHover = layerId.includes('hover'); const isParent = PARENT_LAYERS.includes(layerId); - isHover && + if (isHover && mapRef.getLayer(layerId)) { mapRef.setPaintProperty( layerId, 'fill-opacity', @@ -276,10 +278,13 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { isParent ? shatterIds.parents : undefined ) ); + } }); CHILD_LAYERS.forEach(layerId => { - !layerId.includes('hover') && mapRef.setPaintProperty(layerId, 'line-opacity', 1); + if (!layerId.includes('hover') && mapRef.getLayer(layerId)) { + mapRef.setPaintProperty(layerId, 'line-opacity', 1); + } }); } ); diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index dadff7b8f..531084469 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -62,6 +62,12 @@ export interface MapStore { setMapRef: (map: MutableRefObject) => void; mapLock: boolean; setMapLock: (lock: boolean) => void; + errorNotification: { + message?: string, + severity?: 1 | 2 | 3, // 1: dialog, 2: toast, 3: silent + id?:string + }, + setErrorNotification: (errorNotification: MapStore['errorNotification']) => void; /** * Selects map features and updates the zone assignments accordingly. * Debounced zone updates will be sent to backend after a delay. @@ -324,6 +330,8 @@ export const useMapStore = create( }, mapLock: false, setMapLock: mapLock => set({mapLock}), + errorNotification: {}, + setErrorNotification: (errorNotification) => set({errorNotification}), selectMapFeatures: features => { let { accumulatedGeoids, @@ -384,10 +392,20 @@ export const useMapStore = create( if (currentMapDocument?.document_id === mapDocument.document_id) { return; } - setFreshMap(true); - resetZoneAssignments(); - upsertUserMap({mapDocument}); - + setFreshMap(true) + resetZoneAssignments() + + const upsertMapOnDrawSub = useMapStore.subscribe(state => state.zoneAssignments, + (za) => { + if (useMapStore.getState().mapDocument !== mapDocument || za.size){ + upsertMapOnDrawSub() + } + if (useMapStore.getState().mapDocument === mapDocument && za.size) { + upsertUserMap({mapDocument}) + } + } + ) + set({ mapDocument: mapDocument, mapOptions: { @@ -455,7 +473,17 @@ export const useMapStore = create( document_id, geoids, }); - + if (!shatterResult.children.length){ + const mapDocument = get().mapDocument + set({ + errorNotification: { + severity: 2, + message: `Breaking this geography failed. Please refresh this page and try again. If this error persists, please share the error code below the Districtr team.`, + id: `break-patchShatter-no-children-${mapDocument?.gerrydb_table}-${mapDocument?.document_id}-geoid-${JSON.stringify(geoids)}` + } + }) + return + } // TODO Need to return child edges even if the parent is already shattered // currently returns nothing const shatterIds = get().shatterIds; diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index 949d71cd8..b7b61e63a 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -94,6 +94,20 @@ export const updateGetDocumentFromId = (documentId: string) => { }; updateDocumentFromId.subscribe(mapDocument => { + if (typeof window === 'undefined') return + + const documentId = new URLSearchParams(window.location.search).get('document_id'); + if (mapDocument.error && documentId?.length) { + useMapStore.getState().setErrorNotification({ + severity: 2, + id: 'map-document-not-found', + message: `The requested map id "${documentId}" could not be found. Please make sure the URL is correct or select a different geography.` + }); + // remove current document_id on search params + const url = new URL(window.location.href); + url.searchParams.delete('document_id'); + window.history.replaceState({}, document.title, url.toString()); + } if (mapDocument.data) { useMapStore.getState().setMapDocument(mapDocument.data); } diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index 93b3f3197..3cf46cbd4 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -109,7 +109,14 @@ export const handleMapMouseDown = ( export const handleMapMouseEnter = ( e: MapLayerMouseEvent | MapLayerTouchEvent, map: MapLibreMap | null -) => {}; +) => { + // check if mouse is down + // if so, set is painting true + // @ts-ignore this is the correct behavior but event types are incorrect + if (e.originalEvent?.buttons === 1){ + useMapStore.getState().setIsPainting(true) + } +}; export const handleMapMouseOver = ( e: MapLayerMouseEvent | MapLayerTouchEvent, From 1d825a38682e0e1eaac314eec0ee1d0691e9433b Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 21 Nov 2024 10:29:11 -0600 Subject: [PATCH 47/50] Enhance lock and fix issues (#207) --- app/src/app/components/sidebar/ColorPicker.tsx | 3 +++ app/src/app/components/sidebar/Layers.tsx | 4 ---- app/src/app/components/sidebar/ZoneLockPicker.tsx | 14 ++++++++++++++ app/src/app/store/mapStore.ts | 9 +++++---- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/app/src/app/components/sidebar/ColorPicker.tsx b/app/src/app/components/sidebar/ColorPicker.tsx index e1e02439c..cce46aaf2 100644 --- a/app/src/app/components/sidebar/ColorPicker.tsx +++ b/app/src/app/components/sidebar/ColorPicker.tsx @@ -40,6 +40,9 @@ export const ColorPicker = ({ const indices = values.map(f => colorArray.indexOf(f)); onValueChange(indices, values); }} + style={{ + justifyContent: "flex-start" + }} > {!!mapDocument && colorArray.slice(0, mapDocument.num_districts ?? 0).map((color, i) => ( state.visibleLayerIds); const updateVisibleLayerIds = useMapStore(state => state.updateVisibleLayerIds); const toggleHighlightBrokenDistricts = useMapStore(state => state.toggleHighlightBrokenDistricts); - const toggleLockAllAreas = useMapStore(state => state.toggleLockAllAreas); const parentsAreBroken = useMapStore(state => state.shatterIds.parents.size); const mapOptions = useMapStore(state => state.mapOptions); const setMapOptions = useMapStore(state => state.setMapOptions); @@ -78,9 +77,6 @@ export default function Layers() { })}> Highlight unassigned units - toggleLockAllAreas()}> - Lock All Painted Areas - Boundaries diff --git a/app/src/app/components/sidebar/ZoneLockPicker.tsx b/app/src/app/components/sidebar/ZoneLockPicker.tsx index 582718703..2c6bed558 100644 --- a/app/src/app/components/sidebar/ZoneLockPicker.tsx +++ b/app/src/app/components/sidebar/ZoneLockPicker.tsx @@ -6,9 +6,13 @@ import * as RadioGroup from '@radix-ui/react-radio-group'; import {blackA} from '@radix-ui/colors'; import {useMapStore} from '../../store/mapStore'; import {ColorPicker} from './ColorPicker'; +import { LockClosedIcon, LockOpen2Icon } from '@radix-ui/react-icons'; export function ZoneLockPicker() { const lockedZones = useMapStore(state => state.mapOptions.lockPaintedAreas); + const mapDocument = useMapStore(state => state.mapDocument) + const numDistricts = mapDocument?.num_districts || 40 + const allDistrictsNumbers = new Array(numDistricts-1).fill(null).map((_,i) => i+1) const pickerValue = Array.isArray(lockedZones) ? lockedZones.map(f => (null === f ? 0 : f - 1)) : lockedZones === true @@ -21,6 +25,8 @@ export function ZoneLockPicker() { const zoneValues = indices.map(v => v + 1); setLockedZones(zoneValues); }; + const lockAll = () => setLockedZones(allDistrictsNumbers) + const unlockAll = () => setLockedZones([]) return (
@@ -31,6 +37,14 @@ export function ZoneLockPicker() { value={pickerValue} multiple /> + +
); } diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 531084469..89b96a0cf 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -458,7 +458,7 @@ export const useMapStore = create( const geoids = features.map(f => f.id?.toString()).filter(Boolean) as string[]; - const shatterMappings = get().shatterMappings; + const {shatterIds, shatterMappings, lockedFeatures} = get(); const isAlreadyShattered = geoids.some(id => shatterMappings.hasOwnProperty(id)); const shatterResult: ShatterResult = isAlreadyShattered ? ({ @@ -473,6 +473,7 @@ export const useMapStore = create( document_id, geoids, }); + if (!shatterResult.children.length){ const mapDocument = get().mapDocument set({ @@ -486,13 +487,12 @@ export const useMapStore = create( } // TODO Need to return child edges even if the parent is already shattered // currently returns nothing - const shatterIds = get().shatterIds; - + const newLockedFeatures = new Set(lockedFeatures) let existingParents = new Set(shatterIds.parents); let existingChildren = new Set(shatterIds.children); const newParent = shatterResult.parents.geoids; const newChildren = new Set(shatterResult.children.map(child => child.geo_id)); - + newChildren.forEach(child => newLockedFeatures.delete(child)) const zoneAssignments = new Map(get().zoneAssignments); const multipleShattered = shatterResult.parents.geoids.length > 1; const featureBbox = features[0].geometry && bbox(features[0].geometry); @@ -519,6 +519,7 @@ export const useMapStore = create( }, mapLock: false, captiveIds: newChildren, + lockedFeatures: newLockedFeatures, focusFeatures: [ { id: features[0].id, From d2ceb72d16f5e7074fcdd4ac4a82a53d238ee57a Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 21 Nov 2024 12:51:29 -0600 Subject: [PATCH 48/50] Paint by county with ID instead of features (#199) --- app/src/app/components/Map.tsx | 3 ++ .../app/components/sidebar/PaintByCounty.tsx | 17 +++++---- app/src/app/constants/layers.ts | 1 + app/src/app/store/idCache.ts | 25 +++++++++++++ app/src/app/store/mapRenderSubs.ts | 23 +++++++++++- app/src/app/store/mapStore.ts | 6 ++-- app/src/app/store/types.ts | 2 ++ app/src/app/utils/events/mapEvents.ts | 36 ++++++++++++++++++- app/src/app/utils/helpers.ts | 28 ++++++++++----- 9 files changed, 121 insertions(+), 20 deletions(-) create mode 100644 app/src/app/store/idCache.ts diff --git a/app/src/app/components/Map.tsx b/app/src/app/components/Map.tsx index 908a70666..211289797 100644 --- a/app/src/app/components/Map.tsx +++ b/app/src/app/components/Map.tsx @@ -8,6 +8,7 @@ import {MAP_OPTIONS} from '../constants/configuration'; import {mapEvents} from '../utils/events/mapEvents'; import {INTERACTIVE_LAYERS} from '../constants/layers'; import {useMapStore} from '../store/mapStore'; +import { parentIdCache } from '../store/idCache'; export const MapComponent: React.FC = () => { const map: MutableRefObject = useRef(null); @@ -45,6 +46,7 @@ export const MapComponent: React.FC = () => { zoom: MAP_OPTIONS.zoom, maxZoom: MAP_OPTIONS.maxZoom, }); + fitMapToBounds(); map.current.scrollZoom.setWheelZoomRate(1 / 300); map.current.scrollZoom.setZoomRate(1 / 300); @@ -86,3 +88,4 @@ export const MapComponent: React.FC = () => { /> ); }; + diff --git a/app/src/app/components/sidebar/PaintByCounty.tsx b/app/src/app/components/sidebar/PaintByCounty.tsx index 88e72afa4..78c72c93d 100644 --- a/app/src/app/components/sidebar/PaintByCounty.tsx +++ b/app/src/app/components/sidebar/PaintByCounty.tsx @@ -8,12 +8,15 @@ export default function PaintByCounty() { const mapRef = useMapStore(state => state.getMapRef()); const addVisibleLayerIds = useMapStore(state => state.addVisibleLayerIds); const setPaintFunction = useMapStore(state => state.setPaintFunction); - const [checked, setChecked] = useState(false); + const paintByCounty = useMapStore(state => state.mapOptions.paintByCounty) + const setMapOptions = useMapStore(state => state.setMapOptions) - useEffect(() => { + const handleToggle = () => { if (!mapRef) return; - - if (checked) { + setMapOptions({ + paintByCounty: !paintByCounty + }) + if (!paintByCounty) { COUNTY_LAYER_IDS.forEach(layerId => { mapRef.setLayoutProperty(layerId, 'visibility', 'visible'); }); @@ -22,16 +25,16 @@ export default function PaintByCounty() { } else { setPaintFunction(getFeaturesInBbox); } - }, [checked, mapRef, addVisibleLayerIds]); + } return ( setChecked(prevIsChecked => !prevIsChecked)} + onClick={handleToggle} /> Paint by County diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index 27acb445a..8c0da06f8 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -23,6 +23,7 @@ export const INTERACTIVE_LAYERS = [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CH export const LINE_LAYERS = [BLOCK_LAYER_ID, BLOCK_LAYER_ID_CHILD] as const export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; +export const COUNTY_LAYERS = ['counties_fill', 'counties_boundary','counties_labels'] export const CHILD_LAYERS = [ BLOCK_LAYER_ID_CHILD, diff --git a/app/src/app/store/idCache.ts b/app/src/app/store/idCache.ts new file mode 100644 index 000000000..e039ae11f --- /dev/null +++ b/app/src/app/store/idCache.ts @@ -0,0 +1,25 @@ +class IdCache { + cachedTileIndices: Set = new Set() + parentIds: Set = new Set() + + hasCached(index: string){ + return this.cachedTileIndices.has(index) + } + + add(index: string, ids: string[]){ + this.cachedTileIndices.add(index) + ids.forEach(id => this.parentIds.add(id)) + } + + clear(){ + this.parentIds.clear() + this.cachedTileIndices.clear() + } + + getFilteredIds(id: string){ + const regex = new RegExp(`^${id}`); + return Array.from(this.parentIds).filter(f => regex.test(f)); + } +} + +export const parentIdCache = new IdCache() diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 1256da961..a90bb7393 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -11,10 +11,13 @@ import { BLOCK_LAYER_ID_HIGHLIGHT, getHighlightLayerSpecification, BLOCK_LAYER_ID_HIGHLIGHT_CHILD, + COUNTY_LAYERS, } from '../constants/layers'; import { ColorZoneAssignmentsState, colorZoneAssignments, + getFeaturesInBbox, + getFeaturesIntersectingCounties, shallowCompareArray, } from '../utils/helpers'; import {useMapStore as _useMapStore, MapStore} from '@store/mapStore'; @@ -55,7 +58,7 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { // Hide broken parents on parent layer // Show broken children on child layer layersToFilter.forEach(layerId => - mapRef.setFilter(layerId, getLayerFilter(layerId, shatterIds)) + mapRef.getLayer(layerId) && mapRef.setFilter(layerId, getLayerFilter(layerId, shatterIds)) ); // remove zone from parents shatterIds.parents.forEach(id => { @@ -224,16 +227,21 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { activeTool => { const mapRef = useMapStore.getState().getMapRef(); if (!mapRef) return; + const mapOptions = useMapStore.getState().mapOptions + const defaultPaintFunction = mapOptions.paintByCounty ? getFeaturesIntersectingCounties : getFeaturesInBbox let cursor; switch (activeTool) { case 'pan': cursor = ''; + useMapStore.getState().setPaintFunction(defaultPaintFunction); break; case 'brush': cursor = 'pointer'; + useMapStore.getState().setPaintFunction(defaultPaintFunction); break; case 'eraser': cursor = 'pointer'; + useMapStore.getState().setPaintFunction(defaultPaintFunction); break; case 'shatter': cursor = 'crosshair'; @@ -308,6 +316,18 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { } } ); + + const filterCountiesSub = useMapStore.subscribe<[string|undefined, MapStore['getMapRef']]>(state => [state.mapOptions.currentStateFp, state.getMapRef], + ([stateFp, getMapRef]) => { + const mapRef = getMapRef() + if (!mapRef) return + const filterExpression = (stateFp ? ["==", "STATEFP", stateFp] : true) as any + COUNTY_LAYERS.forEach(layer => { + mapRef.getLayer(layer) && mapRef.setFilter(layer, ["any", filterExpression]) + }) + } + ) + return [ addLayerSubMapDocument, _shatterMapSideEffectRender, @@ -316,5 +336,6 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { _updateMapCursor, _applyFocusFeatureState, highlightUnassignedSub, + filterCountiesSub ]; }; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 89b96a0cf..c4dc166d3 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -37,6 +37,7 @@ import {BLOCK_SOURCE_ID} from '../constants/layers'; import {DistrictrMapOptions} from './types'; import {onlyUnique} from '../utils/arrays'; import {queryClient} from '../utils/api/queryClient'; +import { parentIdCache } from './idCache'; const combineSetValues = (setRecord: Record>, keys?: string[]) => { const combinedSet = new Set(); // Create a new set to hold combined values @@ -392,8 +393,9 @@ export const useMapStore = create( if (currentMapDocument?.document_id === mapDocument.document_id) { return; } - setFreshMap(true) - resetZoneAssignments() + parentIdCache.clear() + setFreshMap(true); + resetZoneAssignments(); const upsertMapOnDrawSub = useMapStore.subscribe(state => state.zoneAssignments, (za) => { diff --git a/app/src/app/store/types.ts b/app/src/app/store/types.ts index 47ad70c55..1f38e3bd3 100644 --- a/app/src/app/store/types.ts +++ b/app/src/app/store/types.ts @@ -5,4 +5,6 @@ export type DistrictrMapOptions = { higlightUnassigned?: boolean; lockPaintedAreas: boolean | Array; mode: 'default' | 'break'; + paintByCounty?: boolean; + currentStateFp?: string }; diff --git a/app/src/app/utils/events/mapEvents.ts b/app/src/app/utils/events/mapEvents.ts index 3cf46cbd4..8ab3e914d 100644 --- a/app/src/app/utils/events/mapEvents.ts +++ b/app/src/app/utils/events/mapEvents.ts @@ -2,7 +2,7 @@ Port over from map events declared at: https://github.com/uchicago-dsi/districtr-components/blob/2e8f9e5657b9f0fd2419b6f3258efd74ae310f32/src/Districtr/Districtr.tsx#L230 */ 'use client'; -import type {Map as MapLibreMap, MapLayerMouseEvent, MapLayerTouchEvent} from 'maplibre-gl'; +import type {Map as MapLibreMap, MapLayerMouseEvent, MapLayerTouchEvent, MapDataEvent, MapSourceDataEvent} from 'maplibre-gl'; import {useMapStore} from '@/app/store/mapStore'; import { BLOCK_HOVER_LAYER_ID, @@ -11,6 +11,7 @@ import { } from '@/app/constants/layers'; import {ResetMapSelectState} from '@utils/events/handlers'; import {ActiveTool} from '@/app/constants/types'; +import { parentIdCache } from '@/app/store/idCache'; /* MapEvent handling; these functions are called by the event listeners in the MapComponent @@ -234,6 +235,38 @@ export const handleMapContextMenu = ( }); }; +export const handleIdCache = ( + _e: MapLayerMouseEvent | MapLayerTouchEvent, + map: MapLibreMap | null +) => { + const e = _e as unknown as MapSourceDataEvent + const {tiles_s3_path, parent_layer} = useMapStore.getState().mapDocument || {} + + if ( + !tiles_s3_path || + !parent_layer || + e.dataType !== 'source' || + !("url" in e.source) || + !e.source.url?.includes(tiles_s3_path) + ) return + + const tileData = e.tile.latestFeatureIndex; + if (!tileData) return + + const index = `${tileData.x}-${tileData.y}-${tileData.z}` + if (parentIdCache.hasCached(index)) return + const vtLayers = tileData.loadVTLayers() + + const parentLayerData = vtLayers[parent_layer] + const numFeatures = parentLayerData.length + const featureDataArray = parentLayerData._values + const idArray = featureDataArray.slice(-numFeatures,) + parentIdCache.add(index, idArray) + useMapStore.getState().setMapOptions({ + currentStateFp: idArray[0].replace('vtd:','').slice(0,2) + }) +} + export const mapEvents = [ {action: 'click', handler: handleMapClick}, {action: 'mouseup', handler: handleMapMouseUp}, @@ -253,4 +286,5 @@ export const mapEvents = [ {action: 'moveend', handler: handleMapMoveEnd}, {action: 'zoomend', handler: handleMapZoomEnd}, {action: 'contextmenu', handler: handleMapContextMenu}, + {action: 'data', handler: handleIdCache} ]; diff --git a/app/src/app/utils/helpers.ts b/app/src/app/utils/helpers.ts index 3f4b44dd7..a997a649f 100644 --- a/app/src/app/utils/helpers.ts +++ b/app/src/app/utils/helpers.ts @@ -15,6 +15,7 @@ import { } from '@/app/constants/layers'; import {MapStore, useMapStore} from '../store/mapStore'; import {NullableZone} from '../constants/types'; +import {parentIdCache} from '../store/idCache'; /** * PaintEventHandler @@ -133,16 +134,25 @@ export const getFeaturesIntersectingCounties = ( if (!countyFeatures?.length) return; const fips = countyFeatures[0].properties.STATEFP + countyFeatures[0].properties.COUNTYFP; + const {mapDocument, shatterIds} = useMapStore.getState(); + const filterPrefix = mapDocument?.parent_layer.includes("vtd") ? "vtd:" : "" + const cachedParentFeatures = parentIdCache.getFilteredIds(`${filterPrefix}${fips}`).map(id => ({ + id, + source: BLOCK_SOURCE_ID, + sourceLayer: mapDocument?.parent_layer, + })); + + const childFeatures = shatterIds.children.size + ? (Array.from(shatterIds.children).map(id => ({ + id, + source: BLOCK_SOURCE_ID, + sourceLayer: mapDocument?.child_layer, + })) as any) + : []; - const features = map.queryRenderedFeatures(undefined, { - layers, - }); - - return filterFeatures( - features, - true, - [(feature) => Boolean(feature?.id && feature.id.toString().match(/\d{5}/)?.[0] === fips)] - ); + return filterFeatures([...cachedParentFeatures, ...childFeatures], true, [ + feature => Boolean(feature?.id && feature.id.toString().match(/\d{5}/)?.[0] === fips), + ]); }; /** From 1959af9ef2369313953e69ee90ad671e216a548c Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 21 Nov 2024 12:51:38 -0600 Subject: [PATCH 49/50] Chart enhancements (#209) --- .../app/components/sidebar/ZoneLockPicker.tsx | 2 +- .../sidebar/charts/HorizontalBarChart.tsx | 40 +++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/app/src/app/components/sidebar/ZoneLockPicker.tsx b/app/src/app/components/sidebar/ZoneLockPicker.tsx index 2c6bed558..bd9efbcbe 100644 --- a/app/src/app/components/sidebar/ZoneLockPicker.tsx +++ b/app/src/app/components/sidebar/ZoneLockPicker.tsx @@ -12,7 +12,7 @@ export function ZoneLockPicker() { const lockedZones = useMapStore(state => state.mapOptions.lockPaintedAreas); const mapDocument = useMapStore(state => state.mapDocument) const numDistricts = mapDocument?.num_districts || 40 - const allDistrictsNumbers = new Array(numDistricts-1).fill(null).map((_,i) => i+1) + const allDistrictsNumbers = new Array(numDistricts).fill(null).map((_,i) => i+1) const pickerValue = Array.isArray(lockedZones) ? lockedZones.map(f => (null === f ? 0 : f - 1)) : lockedZones === true diff --git a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx index 8fa91d975..2e2a4f43d 100644 --- a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx +++ b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx @@ -1,4 +1,4 @@ -import {useMapStore} from '@/app/store/mapStore'; +import {MapStore, useMapStore} from '@/app/store/mapStore'; import {Card, Flex, Heading, Text} from '@radix-ui/themes'; import { BarChart, @@ -13,12 +13,21 @@ import { } from 'recharts'; import {colorScheme} from '@/app/constants/colors'; import {useState, useMemo} from 'react'; +import { formatNumber } from '@/app/utils/numbers'; type TooltipInput = { active?: boolean; payload?: [{payload: {total_pop: number; zone: number}}]; }; +const calculateMinMaxRange = (data: Array<{zone: number; total_pop: number}>) => { + const totalPops = data.map(item => item.total_pop); + const min = Math.min(...totalPops); + const max = Math.max(...totalPops); + const range = Math.abs(max - min); + return { min, max, range }; +}; + const numberFormat = new Intl.NumberFormat('en-US'); const CustomTooltip = ({active, payload: items}: TooltipInput) => { @@ -38,6 +47,7 @@ export const HorizontalBar = () => { const summaryStats = useMapStore(state => state.summaryStats); const numDistricts = useMapStore(state => state.mapDocument?.num_districts); const idealPopulation = summaryStats?.idealpop?.data; + const lockPaintedAreas = useMapStore(state => state.mapOptions.lockPaintedAreas) const maxNumberOrderedBars = 40; // max number of zones to consider while keeping blank spaces for missing zones const [totalExpectedBars, setTotalExpectedBars] = useState< Array<{zone: number; total_pop: number}> @@ -61,10 +71,13 @@ export const HorizontalBar = () => { } }; - useMemo(() => { + const stats = useMemo(() => { if (mapMetrics) { const chartObject = calculateChartObject(); + const allAreNonZero = chartObject.every(entry => entry.total_pop > 0) + const stats = allAreNonZero ? calculateMinMaxRange(chartObject) : undefined setTotalExpectedBars(chartObject); + return stats } }, [mapMetrics]); @@ -111,7 +124,8 @@ export const HorizontalBar = () => { /> } /> - + {/* @ts-ignore types are wrong, this works */} + {totalExpectedBars && totalExpectedBars .sort((a, b) => a.zone - b.zone) @@ -120,7 +134,7 @@ export const HorizontalBar = () => { ))} - + {stats?.range !== undefined && + Top-to-bottom deviation: {formatNumber(stats.range || 0, 'string')} + } ); }; + + +const getRenderCustomBarLabel: (lockPaintedAreas: MapStore['mapOptions']['lockPaintedAreas']) => React.FC = (lockPaintedAreas) => { + const InnerComponent: React.FC = ({ y, height, index, value }) => { + const entryIsLocked = Array.isArray(lockPaintedAreas) && lockPaintedAreas.indexOf(index+1) !== -1 + + if (!entryIsLocked || !value) { + return null + } + return + + ; + } + return InnerComponent +} \ No newline at end of file From bf57b0dce62e8d3c55e17e5e54674f843fc5d831 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 22 Nov 2024 08:31:38 -0600 Subject: [PATCH 50/50] Text Updates (#211) --- app/src/app/components/sidebar/Evaluation.tsx | 12 ++-- .../components/sidebar/MapModeSelector.jsx | 2 +- .../app/components/sidebar/PaintByCounty.tsx | 3 - .../sidebar/charts/HorizontalBarChart.tsx | 55 +++++++++++-------- 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/app/src/app/components/sidebar/Evaluation.tsx b/app/src/app/components/sidebar/Evaluation.tsx index 3704b9cd2..e3bfe030d 100644 --- a/app/src/app/components/sidebar/Evaluation.tsx +++ b/app/src/app/components/sidebar/Evaluation.tsx @@ -10,7 +10,7 @@ import { P4ZoneSummaryStats, P4ZoneSummaryStatsKeys, } from '@/app/utils/api/apiHandlers'; -import {Button, CheckboxGroup} from '@radix-ui/themes'; +import {Button, CheckboxGroup, Heading} from '@radix-ui/themes'; import {Flex, Spinner, Text} from '@radix-ui/themes'; import {queryClient} from '@utils/api/queryClient'; import {formatNumber, NumberFormats} from '@/app/utils/numbers'; @@ -100,7 +100,6 @@ const Evaluation: React.FC = () => { const totPop = useMapStore(state => state.summaryStats.totpop?.data); const mapDocument = useMapStore(state => state.mapDocument); const assignmentsHash = useMapStore(state => state.assignmentsHash); - const columnConfig = useMemo(() => { const summaryType = mapDocument?.available_summary_stats?.[0]; @@ -139,6 +138,7 @@ const Evaluation: React.FC = () => { queryClient ); + const { unassigned, maxValues, @@ -155,6 +155,7 @@ const Evaluation: React.FC = () => { zone: -999, total: getEntryTotal(totPop), }; + ZoneSummaryStatsKeys.forEach(key => { let total = unassigned[key]; maxValues[key] = -Math.pow(10, 12); @@ -164,7 +165,7 @@ const Evaluation: React.FC = () => { // @ts-ignore maxValues[key] = Math.max(row[key], maxValues[key]); }); - unassigned[`${key}_pct`] = total / unassigned[key]; + unassigned[`${key}_pct`] = total / unassigned['total']; unassigned[key] = total; }); @@ -189,7 +190,10 @@ const Evaluation: React.FC = () => { const rows = unassigned && showUnassigned ? [...data.results, unassigned] : data.results; return (
- + + Voting age population + + {modeButtonConfig.map((mode, i) => (