diff --git a/app/package-lock.json b/app/package-lock.json index 0ffe56b48..4d1b8d6a9 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -19,6 +19,7 @@ "@turf/bbox": "^7.1.0", "@turf/helpers": "^7.1.0", "axios": "^1.7.2", + "d3-scale-chromatic": "^3.1.0", "idb-keyval": "^6.2.1", "lodash": "^4.17.21", "maplibre-gl": "^4.4.1", @@ -35,6 +36,7 @@ "@flydotio/dockerfile": "^0.5.8", "@testing-library/jest-dom": "^6.4.6", "@testing-library/react": "^16.0.0", + "@types/d3-scale-chromatic": "^3.0.3", "@types/lodash": "^4.17.5", "@types/node": "^20", "@types/react": "^18", @@ -4403,7 +4405,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/bbox/-/bbox-7.1.0.tgz", "integrity": "sha512-PdWPz9tW86PD78vSZj2fiRaB8JhUHy6piSa/QXb83lucxPK+HTAdzlDQMTKj5okRCU8Ox/25IR2ep9T8NdopRA==", - "license": "MIT", "dependencies": { "@turf/helpers": "^7.1.0", "@turf/meta": "^7.1.0", @@ -4418,7 +4419,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/helpers/-/helpers-7.1.0.tgz", "integrity": "sha512-dTeILEUVeNbaEeoZUOhxH5auv7WWlOShbx7QSd4s0T4Z0/iz90z9yaVCtZOLbU89umKotwKaJQltBNO9CzVgaQ==", - "license": "MIT", "dependencies": { "@types/geojson": "^7946.0.10", "tslib": "^2.6.2" @@ -4431,7 +4431,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/meta/-/meta-7.1.0.tgz", "integrity": "sha512-ZgGpWWiKz797Fe8lfRj7HKCkGR+nSJ/5aKXMyofCvLSc2PuYJs/qyyifDPWjASQQCzseJ7AlF2Pc/XQ/3XkkuA==", - "license": "MIT", "dependencies": { "@turf/helpers": "^7.1.0", "@types/geojson": "^7946.0.10" @@ -4532,6 +4531,12 @@ "@types/d3-time": "*" } }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.3.tgz", + "integrity": "sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==", + "dev": true + }, "node_modules/@types/d3-shape": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz", @@ -6171,6 +6176,18 @@ "node": ">=12" } }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", diff --git a/app/package.json b/app/package.json index a61167bbb..90abcf044 100644 --- a/app/package.json +++ b/app/package.json @@ -21,6 +21,7 @@ "@turf/bbox": "^7.1.0", "@turf/helpers": "^7.1.0", "axios": "^1.7.2", + "d3-scale-chromatic": "^3.1.0", "idb-keyval": "^6.2.1", "lodash": "^4.17.21", "maplibre-gl": "^4.4.1", @@ -37,6 +38,7 @@ "@flydotio/dockerfile": "^0.5.8", "@testing-library/jest-dom": "^6.4.6", "@testing-library/react": "^16.0.0", + "@types/d3-scale-chromatic": "^3.0.3", "@types/lodash": "^4.17.5", "@types/node": "^20", "@types/react": "^18", @@ -50,4 +52,4 @@ "tailwindcss": "^3.4.1", "typescript": "^5" } -} +} \ No newline at end of file diff --git a/app/src/app/components/sidebar/DataPanels.tsx b/app/src/app/components/sidebar/DataPanels.tsx index ec16ce4ec..f99b7d269 100644 --- a/app/src/app/components/sidebar/DataPanels.tsx +++ b/app/src/app/components/sidebar/DataPanels.tsx @@ -1,11 +1,7 @@ -import {Box, Flex, Heading} from '@radix-ui/themes'; -import {MapModeSelector} from './MapModeSelector'; -import {ColorPicker} from './ColorPicker'; -import {ResetMapButton} from './ResetMapButton'; -import {GerryDBViewSelector} from './GerryDBViewSelector'; +import {Box} from '@radix-ui/themes'; +import Evaluation from '@components/sidebar/Evaluation'; import {HorizontalBar} from './charts/HorizontalBarChart'; -import {useMapStore} from '@/app/store/mapStore'; -import {Tabs, Text} from '@radix-ui/themes'; +import {Tabs} from '@radix-ui/themes'; import Layers from './Layers'; import React from 'react'; @@ -35,7 +31,7 @@ const defaultPanels: DataPanelSpec[] = [ { title: 'evaluation', label: 'Evaluation', - content: Unimplemented , + content: , }, ]; diff --git a/app/src/app/components/sidebar/Evaluation.tsx b/app/src/app/components/sidebar/Evaluation.tsx new file mode 100644 index 000000000..7aa4f5cbd --- /dev/null +++ b/app/src/app/components/sidebar/Evaluation.tsx @@ -0,0 +1,321 @@ +import React, {useMemo, useState} from 'react'; +import {useMapStore} from '@/app/store/mapStore'; +import {useQuery} from '@tanstack/react-query'; +import { + CleanedP1ZoneSummaryStats, + CleanedP1ZoneSummaryStatsKeys, + getP1SummaryStats, + P1ZoneSummaryStats, + P1ZoneSummaryStatsKeys, +} from '@/app/utils/api/apiHandlers'; +import {Button, Checkbox, CheckboxGroup} from '@radix-ui/themes'; +import {Heading, Flex, Spinner, Text} from '@radix-ui/themes'; +import {queryClient} from '@utils/api/queryClient'; +import {formatNumber, NumberFormats} from '@/app/utils/numbers'; +import {colorScheme} from '@/app/constants/colors'; +import { + getEntryTotal, + getStdDevColor, + stdDevArray, + stdDevColors, + sumArray, +} from '@/app/utils/summaryStats'; +import { interpolateBlues, interpolateGreys } from 'd3-scale-chromatic'; + +type EvalModes = 'share' | 'count' | 'totpop'; +type ColumnConfiguration> = Array<{label: string; column: keyof T}>; +type EvaluationProps = { + columnConfig?: ColumnConfiguration; +}; + +// const calculateColumn = ( +// mode: EvalModes, +// entry: P1ZoneSummaryStats, +// totals: P1ZoneSummaryStats, +// column: keyof Omit +// ) => { +// const count = entry[column]; +// switch (mode) { +// case 'count': +// return count; +// case 'pct': +// return count / entry['total']; +// case 'share': +// return count / totals[column]; +// } +// }; + +const defaultColumnConfig: ColumnConfiguration = [ + { + label: 'White', + column: 'white_pop', + }, + { + label: 'Black', + column: 'black_pop', + }, + { + label: 'Asian', + column: 'asian_pop', + }, + { + label: 'Am. Indian', + column: 'amin_pop', + }, + { + label: 'Pacific Isl.', + column: 'nhpi_pop', + }, + { + label: 'Other', + column: 'other_pop', + }, +]; + +const modeButtonConfig: Array<{label: string; value: EvalModes}> = [ + { + label: 'Population by Share', + value: 'share', + }, + { + label: 'Population by Count', + value: 'count', + }, + // { + // label: "Population by Percent of Zone", + // value: 'totpop' + // } +]; + +const numberFormats: Record = { + share: 'percent', + count: 'string', + totpop: 'percent', +}; + +const getColConfig = (evalMode: EvalModes) => { + switch (evalMode) { + case 'share': + return (col: keyof P1ZoneSummaryStats) => `${col}_pct` as keyof CleanedP1ZoneSummaryStats; + default: + return (col: keyof P1ZoneSummaryStats) => col; + } +}; + +const Evaluation: React.FC = ({columnConfig = defaultColumnConfig}) => { + const [evalMode, setEvalMode] = useState('share'); + // const [showAverages, setShowAverages] = useState(true); + // const [showStdDev, setShowStdDev] = useState(false); + const [colorBg, setColorBg] = useState(true); + const [showUnassigned, setShowUnassigned] = useState(true); + + const numberFormat = numberFormats[evalMode]; + const columnGetter = getColConfig(evalMode); + const totPop = useMapStore(state => state.summaryStats.totpop?.data); + const mapDocument = useMapStore(state => state.mapDocument); + const assignmentsHash = useMapStore(state => state.assignmentsHash); + + const {data, error, isLoading} = useQuery( + { + queryKey: ['p1SummaryStats', mapDocument, assignmentsHash], + queryFn: () => mapDocument && getP1SummaryStats(mapDocument), + enabled: !!mapDocument, + staleTime: 0, + placeholderData: previousData => previousData, + }, + queryClient + ); + + const { + unassigned, + maxValues + // averages, + // stdDevs + } = useMemo(() => { + if (!data?.results || !totPop) { + return {}; + } + let maxValues: Record = {} + + let unassigned: Record = { + ...totPop, + zone: -999, + total: getEntryTotal(totPop), + }; + P1ZoneSummaryStatsKeys.forEach(key => { + let total = unassigned[key]; + maxValues[key] = -Math.pow(10,12) + data.results.forEach(row => { + total -= row[key]; + maxValues[key] = Math.max(row[key], maxValues[key]) + }); + unassigned[`${key}_pct`] = total / unassigned[key]; + unassigned[key] = total; + }); + // const averages: Record = {}; + // const stdDevs: Record = {}; + // CleanedP1ZoneSummaryStatsKeys.forEach(key => { + // const values = data.results.map(row => row[key]); + // averages[key] = sumArray(values) / data.results.length; + // stdDevs[key] = stdDevArray(values); + // }); + return { + unassigned, + maxValues + // averages, + // stdDevs + }; + }, [data?.results, totPop]); + + if (!data || !maxValues || (mapDocument && !mapDocument.available_summary_stats)) { + return Summary statistics are not available for this map.; + } + + if (error) { + return ( +
+

Summary Statistics

+

There was an error loading the summary statistics.

+
+ ); + } + const rows = unassigned && showUnassigned ? [...data.results, unassigned] : data.results; + return ( +
+ + {modeButtonConfig.map((mode, i) => ( + + ))} + {isLoading && } + + + + + setShowUnassigned(v => !v)}> + Show Unassigned Population + + {/* setShowAverages(v => !v)}> + Show Zone Averages + + setShowStdDev(v => !v)}> + Show Zone Std. Dev. + */} + setColorBg(v => !v)}> + +

Color Cells By Values

+ {/* {colorByStdDev && ( + + {Object.entries(stdDevColors) + .sort((a, b) => +a[0] - +b[0]) + .map(([stdev, backgroundColor], i) => ( + + {+stdev > 0 ? `+${stdev}`: stdev} + + ))} + + )} */} + + + + +
+ + + + + {columnConfig.map((f, i) => ( + + ))} + + + + {/* {!!(averages && showAverages) && ( + + + {columnConfig.map((f, i) => ( + + ))} + + )} + {!!(stdDevs && showStdDev) && ( + + + {columnConfig.map((f, i) => ( + + ))} + + )} */} + {rows + .sort((a, b) => a.zone - b.zone) + .map(row => { + const isUnassigned = row.zone === -999; + const zoneName = isUnassigned ? 'None' : row.zone; + const backgroundColor = isUnassigned ? '#DDDDDD' : colorScheme[row.zone - 1]; + + return ( + + + {columnConfig.map((f, i) => { + const column = columnGetter(f.column); + const colorValue = evalMode === 'count' ? row[column]/maxValues[column] : row[column] + const backgroundColor = (colorBg && !isUnassigned) + ? interpolateGreys(colorValue).replace('rgb', 'rgba').replace(')', ",0.5)") + : 'initial' + return ( + + ); + })} + + ); + })} + +
Zone + {f.label} +
+ Zone Averages + + {formatNumber(averages[columnGetter(f.column)], numberFormat)} +
+ Zone Std. Dev. + + {formatNumber(stdDevs[columnGetter(f.column)], numberFormat)} +
+ + {zoneName} + + {formatNumber(row[column], numberFormat)} +
+
+
+ ); +}; + +export default Evaluation; diff --git a/app/src/app/components/sidebar/Layers.tsx b/app/src/app/components/sidebar/Layers.tsx index adab03804..4f3620fe2 100644 --- a/app/src/app/components/sidebar/Layers.tsx +++ b/app/src/app/components/sidebar/Layers.tsx @@ -26,6 +26,7 @@ export default function Layers() { const toggleLockAllAreas = useMapStore(state => state.toggleLockAllAreas); const parentsAreBroken = useMapStore(state => state.shatterIds.parents.size); const mapOptions = useMapStore(state => state.mapOptions); + const setMapOptions = useMapStore(state => state.setMapOptions); const toggleLayers = (layerIds: string[]) => { if (!mapRef) return; @@ -45,6 +46,7 @@ export default function Layers() { visibleLayerIds.includes(BLOCK_LAYER_ID) ? '1' : '', parentsAreBroken && mapOptions.showBrokenDistricts ? '3' : '', mapOptions.lockPaintedAreas === true ? '4' : '', + mapOptions.higlightUnassigned === true ? 'higlightUnassigned' : '' ]} > Highlight Broken Voter Districts + setMapOptions({ + higlightUnassigned: !mapOptions.higlightUnassigned + })}> + Highlight Unassigned Districts + toggleLockAllAreas()}> Lock All Painted Areas diff --git a/app/src/app/components/sidebar/ResetMapButton.tsx b/app/src/app/components/sidebar/ResetMapButton.tsx index 37d1e5481..22673fb24 100644 --- a/app/src/app/components/sidebar/ResetMapButton.tsx +++ b/app/src/app/components/sidebar/ResetMapButton.tsx @@ -1,13 +1,35 @@ import {useMapStore} from '@/app/store/mapStore'; -import {Button} from '@radix-ui/themes'; +import {AlertDialog, Button, Flex} from '@radix-ui/themes'; export function ResetMapButton() { const handleClickResetMap = useMapStore(state => state.handleReset); const noZonesAreAssigned = useMapStore(state => !state.zoneAssignments.size); return ( - + + + + + + Reset Map + + Are you sure? This will reset all zone assignments and broken geographies. Resetting your + map cannot be undone. + + + + + + + + + + + + ); } diff --git a/app/src/app/components/sidebar/ZonePicker.tsx b/app/src/app/components/sidebar/ZonePicker.tsx index 0d71aaa92..e9915bc22 100644 --- a/app/src/app/components/sidebar/ZonePicker.tsx +++ b/app/src/app/components/sidebar/ZonePicker.tsx @@ -19,9 +19,9 @@ export function ZonePicker() { const handleRadioChange = (index: number, _color: string) => { const value = index + 1; console.log('setting accumulated geoids to old zone', selectedZone, 'new zone is', value); - setZoneAssignments(selectedZone, accumulatedGeoids); + // setZoneAssignments(selectedZone, accumulatedGeoids); setSelectedZone(value); - resetAccumulatedBlockPopulations(); + // resetAccumulatedBlockPopulations(); }; return ( diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index cc7504ed0..93461404b 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -3,6 +3,7 @@ import { ExpressionSpecification, FilterSpecification, LayerSpecification, + LineLayerSpecification, } from 'maplibre-gl'; import {Map} from 'maplibre-gl'; import {getBlocksSource} from './sources'; @@ -13,6 +14,7 @@ import {colorScheme} from './colors'; export const BLOCK_SOURCE_ID = 'blocks'; export const BLOCK_LAYER_ID = 'blocks'; export const BLOCK_LAYER_ID_HIGHLIGHT = BLOCK_LAYER_ID + '-highlight'; +export const BLOCK_LAYER_ID_HIGHLIGHT_CHILD = BLOCK_LAYER_ID + '-highlight-child'; export const BLOCK_LAYER_ID_CHILD = 'blocks-child'; export const BLOCK_HOVER_LAYER_ID = `${BLOCK_LAYER_ID}-hover`; export const BLOCK_HOVER_LAYER_ID_CHILD = `${BLOCK_LAYER_ID_CHILD}-hover`; @@ -21,7 +23,11 @@ export const INTERACTIVE_LAYERS = [BLOCK_HOVER_LAYER_ID, BLOCK_HOVER_LAYER_ID_CH export const PARENT_LAYERS = [BLOCK_LAYER_ID, BLOCK_HOVER_LAYER_ID]; -export const CHILD_LAYERS = [BLOCK_LAYER_ID_CHILD, BLOCK_HOVER_LAYER_ID_CHILD]; +export const CHILD_LAYERS = [ + BLOCK_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_LAYER_ID_HIGHLIGHT_CHILD, +]; export const DEFAULT_PAINT_STYLE: ExpressionSpecification = [ 'case', @@ -64,8 +70,11 @@ export function getLayerFill( captiveIds?: Set, shatterIds?: Set ): DataDrivenPropertyValueSpecification { - const innerFillSpec = [ + const innerFillSpec = ([ 'case', + // is broken parent + ['boolean', ['feature-state', 'broken'], false], + 0, // geography is locked ['boolean', ['feature-state', 'locked'], false], 0.35, @@ -103,7 +112,7 @@ export function getLayerFill( ['boolean', ['feature-state', 'hover'], false], 0.6, 0.2, - ] as unknown as DataDrivenPropertyValueSpecification; + ] as unknown) as DataDrivenPropertyValueSpecification; if (captiveIds?.size) { return [ 'case', @@ -124,8 +133,9 @@ export function getLayerFill( } export function getHighlightLayerSpecification( sourceLayer: string, - layerId: string -): LayerSpecification { + layerId: string, + highlightUnassigned?: boolean +): LineLayerSpecification { return { id: layerId, source: BLOCK_SOURCE_ID, @@ -143,15 +153,29 @@ export function getHighlightLayerSpecification( '#000000', // Black color when focused ['boolean', ['feature-state', 'highlighted'], false], '#e5ff00', // yellow color when highlighted + ['boolean', ['feature-state', 'highlighted'], false], + '#e5ff00', // yellow color when highlighted + // @ts-ignore right behavior, wrong types + ['==', ['feature-state', 'zone'], null], + '#FF0000', // optionally red color when zone is not assigned '#000000', // Default color ], 'line-width': [ 'case', - ['boolean', ['feature-state', 'focused'], false], - 5, // Width of 5 when focused - ['boolean', ['feature-state', 'highlighted'], false], - 5, // Width of 5 when highlighted - 0, // Default width + [ + 'any', + ['boolean', ['feature-state', 'focused'], false], + ['boolean', ['feature-state', 'highlighted'], false], + [ + 'all', + // @ts-ignore correct logic, wrong types + ['==', ['feature-state', 'zone'], null], + ['boolean', !!highlightUnassigned], + ['!', ['boolean', ['feature-state', 'broken'], false]], + ], + ], + 3.5, + 0, // Default width if none of the conditions are met ], }, }; @@ -229,15 +253,13 @@ const addBlockLayers = (map: Map | null, mapDocument: DocumentObject) => { getBlocksHoverLayerSpecification(mapDocument.child_layer, BLOCK_HOVER_LAYER_ID_CHILD), LABELS_BREAK_LAYER_ID ); + map?.addLayer( + getHighlightLayerSpecification(mapDocument.child_layer, BLOCK_LAYER_ID_HIGHLIGHT_CHILD), + LABELS_BREAK_LAYER_ID + ); } map?.addLayer(getHighlightLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID_HIGHLIGHT)); useMapStore.getState().setMapRenderingState('loaded'); - - // update map bounds based on document extent - useMapStore.getState().setMapOptions({ - bounds: mapDocument.extent as [number, number, number, number], - container: useMapStore.getState().mapOptions.container, - }); }; export function removeBlockLayers(map: Map | null) { @@ -245,24 +267,20 @@ export function removeBlockLayers(map: Map | null) { return; } useMapStore.getState().setMapRenderingState('loading'); - if (map.getLayer(BLOCK_LAYER_ID)) { - map.removeLayer(BLOCK_LAYER_ID); - } - if (map.getLayer(BLOCK_LAYER_ID_HIGHLIGHT)) { - map.removeLayer(BLOCK_LAYER_ID_HIGHLIGHT); - } - if (map.getLayer(BLOCK_HOVER_LAYER_ID)) { - map.removeLayer(BLOCK_HOVER_LAYER_ID); - } - if (map.getLayer(BLOCK_LAYER_ID_CHILD)) { - map.removeLayer(BLOCK_LAYER_ID_CHILD); - } - if (map.getLayer(BLOCK_HOVER_LAYER_ID_CHILD)) { - map.removeLayer(BLOCK_HOVER_LAYER_ID_CHILD); - } - if (map.getSource(BLOCK_SOURCE_ID)) { - map.removeSource(BLOCK_SOURCE_ID); - } + [ + BLOCK_LAYER_ID, + BLOCK_LAYER_ID_HIGHLIGHT, + BLOCK_HOVER_LAYER_ID, + BLOCK_LAYER_ID_CHILD, + BLOCK_HOVER_LAYER_ID_CHILD, + BLOCK_LAYER_ID_HIGHLIGHT_CHILD, + ].forEach(layer => { + map.getLayer(layer) && map.removeLayer(layer); + }); + + [BLOCK_SOURCE_ID].forEach(source => { + map.getSource(source) && map.removeSource(source); + }); } export {addBlockLayers}; diff --git a/app/src/app/store/mapRenderSubs.ts b/app/src/app/store/mapRenderSubs.ts index 09bc00067..2a1a98b52 100644 --- a/app/src/app/store/mapRenderSubs.ts +++ b/app/src/app/store/mapRenderSubs.ts @@ -8,6 +8,9 @@ import { getLayerFilter, getLayerFill, BLOCK_SOURCE_ID, + BLOCK_LAYER_ID_HIGHLIGHT, + getHighlightLayerSpecification, + BLOCK_LAYER_ID_HIGHLIGHT_CHILD, } from '../constants/layers'; import { ColorZoneAssignmentsState, @@ -56,10 +59,12 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ); // remove zone from parents shatterIds.parents.forEach(id => { - mapRef?.removeFeatureState({ + mapRef?.setFeatureState({ source: BLOCK_SOURCE_ID, id, sourceLayer: mapDocument?.parent_layer, + }, { + broken: true }); }); @@ -102,8 +107,14 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ], (curr, prev) => { colorZoneAssignments(curr, prev); - const {captiveIds, shatterIds, getMapRef, setLockedFeatures, mapRenderingState} = - useMapStore.getState(); + const { + captiveIds, + shatterIds, + getMapRef, + setLockedFeatures, + lockedFeatures, + mapRenderingState, + } = useMapStore.getState(); const mapRef = getMapRef(); if (!mapRef || mapRenderingState !== 'loaded') return; [...PARENT_LAYERS, ...CHILD_LAYERS].forEach(layerId => { @@ -120,6 +131,8 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { ); }); const [lockPaintedAreas, prevLockPaintedAreas] = [curr[6], prev[6]]; + const sameLockedAreas = + JSON.stringify(lockPaintedAreas) === JSON.stringify(prevLockPaintedAreas); const zoneAssignments = curr[0]; // if lockPaintedAreas, lock all zones if (lockPaintedAreas === true) { @@ -128,15 +141,29 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { .filter(([key, value]) => value !== null) .map(([key]) => key) ); - setLockedFeatures(new Set(nonNullZones)); + setLockedFeatures(nonNullZones); // now unlocked, was previously locked } else if (Array.isArray(lockPaintedAreas)) { + const previousWasArray = Array.isArray(prevLockPaintedAreas); const nonNullZones = new Set( [...zoneAssignments.entries()] - .filter(([key, value]) => lockPaintedAreas.includes(value)) + .filter( + ([key, value]) => + // locked zones include assignment zone + lockPaintedAreas.includes(value) || + // locked zones are the same, and this individual feature was previously locked + (sameLockedAreas && lockedFeatures.has(key)) || + // locked zones are changed, BUT this individual feature is not in a zone + // that was previously locked + (!sameLockedAreas && + previousWasArray && + !lockPaintedAreas.includes(value) && + !prevLockPaintedAreas.includes(value) && + lockedFeatures.has(key)) + ) .map(([key]) => key) ); - setLockedFeatures(new Set(nonNullZones)); + setLockedFeatures(nonNullZones); } else if (!lockPaintedAreas && prevLockPaintedAreas) { setLockedFeatures(new Set()); } @@ -256,6 +283,26 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { }); } ); + + const highlightUnassignedSub = useMapStore.subscribe( + state => state.mapOptions.higlightUnassigned, + (higlightUnassigned) => { + const {getMapRef, mapDocument} = useMapStore.getState(); + const mapRef = getMapRef(); + if (!mapRef || !mapDocument?.parent_layer) return; + // set the layer BLOCK_LAYER_ID_HIGHLIGHT style to be the return from getHighlightLayerSpecification + const paintStyle = getHighlightLayerSpecification(mapDocument.parent_layer, BLOCK_LAYER_ID_HIGHLIGHT, higlightUnassigned)['paint'] + if (!paintStyle) return + if(mapRef.getLayer(BLOCK_LAYER_ID_HIGHLIGHT)){ + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT, 'line-width', paintStyle['line-width']); + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT, 'line-color', paintStyle['line-color']); + } + if(mapRef.getLayer(BLOCK_LAYER_ID_HIGHLIGHT_CHILD)){ + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT_CHILD, 'line-width', paintStyle['line-width']); + mapRef.setPaintProperty(BLOCK_LAYER_ID_HIGHLIGHT_CHILD, 'line-color', paintStyle['line-color']); + } + } + ); return [ addLayerSubMapDocument, _shatterMapSideEffectRender, @@ -263,5 +310,6 @@ export const getRenderSubscriptions = (useMapStore: typeof _useMapStore) => { _zoneAssignmentMapSideEffectRender, _updateMapCursor, _applyFocusFeatureState, + highlightUnassignedSub, ]; }; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 6806d1931..dccfb99d7 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -8,6 +8,7 @@ import { Assignment, DistrictrMap, DocumentObject, + P1TotPopSummaryStats, ShatterResult, ZonePopulation, } from '../utils/api/apiHandlers'; @@ -22,19 +23,19 @@ import { getFeaturesInBbox, resetZoneColors, setZones, -} from '../utils/helpers'; -import {getRenderSubscriptions} from './mapRenderSubs'; +} from "../utils/helpers"; +import { getRenderSubscriptions } from "./mapRenderSubs"; +import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; +import { getMapMetricsSubs } from "./metricsSubs"; +import { getMapEditSubs } from "./mapEditSubs"; +import { getQueriesResultsSubs } from "../utils/api/queries"; +import { persistOptions } from "./persistConfig"; import {patchReset, patchShatter, patchUnShatter} from '../utils/api/mutations'; -import {getSearchParamsObersver} from '../utils/api/queryParamsListener'; -import {getMapMetricsSubs} from './metricsSubs'; -import {getMapEditSubs} from './mapEditSubs'; import bbox from '@turf/bbox'; import {BLOCK_SOURCE_ID} from '../constants/layers'; -import {getMapViewsSubs} from '../utils/api/queries'; -import {persistOptions} from './persistConfig'; -import {onlyUnique} from '../utils/arrays'; import {DistrictrMapOptions} from './types'; -import {queryClient} from '../utils/api/queryClient'; +import { onlyUnique } from '../utils/arrays'; +import { queryClient } from '../utils/api/queryClient'; const combineSetValues = (setRecord: Record>, keys?: string[]) => { const combinedSet = new Set(); // Create a new set to hold combined values @@ -77,6 +78,15 @@ export interface MapStore { */ mapDocument: DocumentObject | null; setMapDocument: (mapDocument: DocumentObject) => void; + summaryStats: { + totpop?: { + data: P1TotPopSummaryStats + } + }, + setSummaryStat: ( + stat: T, + value: MapStore['summaryStats'][T] + ) => void, // SHATTERING /** * A subset of IDs that a user is working on in a focused view. @@ -222,6 +232,8 @@ export interface MapStore { resetAccumulatedBlockPopulations: () => void; zoneAssignments: Map; // geoid -> zone setZoneAssignments: (zone: NullableZone, gdbPaths: Set) => void; + assignmentsHash: string; + setAssignmentsHash: (hash: string) => void; loadZoneAssignments: (assigments: Assignment[]) => void; resetZoneAssignments: () => void; zonePopulations: Map; @@ -358,20 +370,38 @@ export const useMapStore = create( setMapViews: mapViews => set({mapViews}), mapDocument: null, setMapDocument: mapDocument => { - const currentMapDocument = get().mapDocument; + const { + mapDocument: currentMapDocument, + setFreshMap, + resetZoneAssignments, + upsertUserMap, + mapOptions + } = get(); if (currentMapDocument?.document_id === mapDocument.document_id) { return; } - get().setFreshMap(true); - get().resetZoneAssignments(); - get().upsertUserMap({ - mapDocument, - }); + setFreshMap(true) + resetZoneAssignments() + upsertUserMap({mapDocument}) + set({ mapDocument: mapDocument, + mapOptions: { + ...mapOptions, + bounds: mapDocument.extent + }, shatterIds: {parents: new Set(), children: new Set()}, }); }, + summaryStats: {}, + setSummaryStat: (stat, value) => { + set({ + summaryStats: { + ...get().summaryStats, + [stat]: value + } + }) + }, // TODO: Refactor to something like this // featureStates: { // locked: [], @@ -551,6 +581,13 @@ export const useMapStore = create( delete shatterMappings[parent.parentId]; newShatterIds.parents.delete(parent.parentId); newZoneAssignments.set(parent.parentId, parent.zone!); + mapRef?.setFeatureState({ + source: BLOCK_SOURCE_ID, + id: parent.parentId, + sourceLayer: mapDocument?.parent_layer, + }, { + broken: false + }); }); set({ @@ -596,12 +633,8 @@ export const useMapStore = create( userMaps.splice(i, 1, userMapData); // Replace the map at index i with the new data } else { const urlParams = new URL(window.location.href).searchParams; - urlParams.delete('document_id'); // Remove the document_id parameter - window.history.pushState( - {}, - '', - window.location.pathname + '?' + urlParams.toString() - ); // Update the URL without document_id + urlParams.delete("document_id"); // Remove the document_id parameter + window.history.pushState({}, '', window.location.pathname + '?' + urlParams.toString()); // Update the URL without document_id userMaps.splice(i, 1); } } @@ -750,6 +783,8 @@ export const useMapStore = create( selectedZone: 1, setSelectedZone: zone => set({selectedZone: zone}), zoneAssignments: new Map(), + assignmentsHash: "", + setAssignmentsHash: (hash) => set({ assignmentsHash: hash }), accumulatedGeoids: new Set(), setAccumulatedGeoids: accumulatedGeoids => set({accumulatedGeoids}), setZoneAssignments: (zone, geoids) => { @@ -851,6 +886,6 @@ export const useMapStore = create( // these need to initialize after the map store getRenderSubscriptions(useMapStore); getMapMetricsSubs(useMapStore); -getMapViewsSubs(useMapStore); +getQueriesResultsSubs(useMapStore); getMapEditSubs(useMapStore); getSearchParamsObersver(); diff --git a/app/src/app/store/metricsSubs.ts b/app/src/app/store/metricsSubs.ts index 25a8c7713..7329a6787 100644 --- a/app/src/app/store/metricsSubs.ts +++ b/app/src/app/store/metricsSubs.ts @@ -1,4 +1,4 @@ -import {updateMapMetrics} from '../utils/api/queries'; +import {updateMapMetrics, updateTotPop} from '../utils/api/queries'; import {useMapStore as _useMapStore} from './mapStore'; export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { @@ -7,6 +7,7 @@ export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { mapDocument => { if (mapDocument) { updateMapMetrics(mapDocument); + updateTotPop(mapDocument) } } ); diff --git a/app/src/app/store/types.ts b/app/src/app/store/types.ts index 81e76c018..47ad70c55 100644 --- a/app/src/app/store/types.ts +++ b/app/src/app/store/types.ts @@ -2,6 +2,7 @@ import {NullableZone} from '../constants/types'; export type DistrictrMapOptions = { showBrokenDistricts?: boolean; + higlightUnassigned?: boolean; lockPaintedAreas: boolean | Array; mode: 'default' | 'break'; }; diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 804ee3913..f8fb21533 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import 'maplibre-gl'; import {useMapStore} from '@/app/store/mapStore'; +import {getEntryTotal} from '../summaryStats'; export const FormatAssignments = () => { const assignments = Array.from(useMapStore.getState().zoneAssignments.entries()).map( @@ -61,7 +62,7 @@ export interface DocumentObject { created_at: string; updated_at: string | null; extent: [number, number, number, number]; // [minx, miny, maxx, maxy] - total_population: number; + available_summary_stats: string[]; } /** @@ -106,7 +107,7 @@ export const getDocument: (document_id: string) => Promise = asy }; export const getAssignments: ( - mapDocument: DocumentObject + mapDocument: DocumentObject | null ) => Promise = async mapDocument => { if (mapDocument) { return await axios @@ -150,6 +151,115 @@ export const getZonePopulations: ( } }; +export interface SummaryStatsResult { + summary_stat: string; + results: T; +} + +/** + * P1ZoneSummaryStats + * + * @interface + * @property {number} zone - The zone. + * @property {number} total_pop - The total population. + */ +export interface P1ZoneSummaryStats { + zone: number; + other_pop: number; + asian_pop: number; + amin_pop: number; + nhpi_pop: number; + black_pop: number; + white_pop: number; +} +export type P1TotPopSummaryStats = Omit; + +export const P1ZoneSummaryStatsKeys = [ + 'other_pop', + 'asian_pop', + 'amin_pop', + 'nhpi_pop', + 'black_pop', + 'white_pop', +] as const; + +export const CleanedP1ZoneSummaryStatsKeys = [ + ...P1ZoneSummaryStatsKeys, + 'total', + 'other_pop_pct', + 'asian_pop_pct', + 'amin_pop_pct', + 'nhpi_pop_pct', + 'black_pop_pct', + 'white_pop_pct', +] as const; + +export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { + total: number; + other_pop_pct: number; + asian_pop_pct: number; + amin_pop_pct: number; + nhpi_pop_pct: number; + black_pop_pct: number; + white_pop_pct: number; +} + +/** + * Get P1 zone stats from the server. + * @param mapDocument - DocumentObject, the document object + * @returns Promise + */ +export const getP1SummaryStats: ( + mapDocument: DocumentObject +) => Promise> = async mapDocument => { + if (mapDocument) { + return await axios + .get>( + `${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/P1` + ) + .then(res => { + const results = res.data.results.map(row => { + const total = getEntryTotal(row); + return P1ZoneSummaryStatsKeys.reduce( + (acc, key) => { + acc[`${key}_pct`] = acc[key] / total; + return acc; + }, + { + ...row, + total, + } + ) as CleanedP1ZoneSummaryStats; + }); + return { + ...res.data, + results, + }; + }); + } else { + throw new Error('No document provided'); + } +}; + +/** + * Get P1 zone stats from the server. + * @param mapDocument - DocumentObject, the document object + * @returns Promise + */ +export const getP1TotPopSummaryStats: ( + mapDocument: DocumentObject | null +) => Promise> = async mapDocument => { + if (mapDocument) { + return await axios + .get>( + `${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/P1TOTPOP/${mapDocument.parent_layer}` + ) + .then(res => res.data); + } else { + throw new Error('No document provided'); + } +}; + /** * Get available DistrictrMap views from the server. * @param limit - number, the number of views to return (default 10, max 100) diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts index 30ae99dff..a8dbc0ecc 100644 --- a/app/src/app/utils/api/mutations.ts +++ b/app/src/app/utils/api/mutations.ts @@ -27,6 +27,7 @@ export const patchShatter = new MutationObserver(queryClient, { }, onSuccess: data => { console.log(`Successfully shattered parents into ${data.children.length} children`); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); return data; }, }); @@ -60,6 +61,7 @@ export const patchUpdates = new MutationObserver(queryClient, { }, onSuccess: (data: AssignmentsCreate) => { console.log(`Successfully upserted ${data.assignments_upserted} assignments`); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); mapMetrics.refetch(); // remove trailing shattered features // This needs to happen AFTER the updates are done @@ -96,6 +98,7 @@ export const document = new MutationObserver(queryClient, { }, onSuccess: data => { useMapStore.getState().setMapDocument(data); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); useMapStore.getState().setAppLoadingState('loaded'); const documentUrl = new URL(window.location.toString()); documentUrl.searchParams.set('document_id', data.document_id); diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index e50f7996c..fb6dd0064 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -9,12 +9,32 @@ import { getDocument, getZonePopulations, ZonePopulation, + SummaryStatsResult, + getP1TotPopSummaryStats, + P1TotPopSummaryStats, } from './apiHandlers'; import {MapStore, useMapStore} from '@/app/store/mapStore'; const INITIAL_VIEW_LIMIT = 30; const INITIAL_VIEW_OFFSET = 0; +/** + * A utility function that returns a query function based on a nullable parameter. + * + * @param callback - A function that takes a parameter of type ParamT and returns a Promise of type ResultT. + * @param nullableParam - An optional parameter of type ParamT. If this parameter is not provided or is falsy, the function will return a function that returns null. + * + * @returns A function that, when called, will either return null (if nullableParam is not provided) + * or call the callback function with the nullableParam and return its result. + * + * @template ParamT - The type of the parameter that the callback function accepts. + * @template ResultT - The type of the result that the callback function returns. + */ +const getNullableParamQuery = (callback: (param: ParamT) => Promise, nullableParam?: ParamT) => { + if (!nullableParam) return () => null; + return async () => await callback(nullableParam); +}; + export const mapMetrics = new QueryObserver(queryClient, { queryKey: ['_zonePopulations'], queryFn: skipToken, @@ -43,12 +63,19 @@ export const updateMapViews = (limit: number, offset: number) => { }); }; -export const getMapViewsSubs = (_useMapStore: typeof useMapStore) => { +export const getQueriesResultsSubs = (_useMapStore: typeof useMapStore) => { mapViewsQuery.subscribe(result => { if (result) { _useMapStore.getState().setMapViews(result); } }); + fetchTotPop.subscribe(response => { + if (response?.data?.results?.length) { + useMapStore.getState().setSummaryStat('totpop', { data: response.data.results[0]}); + } else { + useMapStore.getState().setSummaryStat('totpop', undefined) + } + }); }; const getDocumentFunction = (documentId?: string) => { @@ -81,19 +108,14 @@ updateDocumentFromId.subscribe(mapDocument => { } }); -const getFetchAssignmentsQuery = (mapDocument?: MapStore['mapDocument']) => { - if (!mapDocument) return () => null; - return async () => await getAssignments(mapDocument); -}; - export const fetchAssignments = new QueryObserver(queryClient, { queryKey: ['assignments'], - queryFn: getFetchAssignmentsQuery(), + queryFn: getNullableParamQuery(getAssignments) }); export const updateAssignments = (mapDocument: DocumentObject) => { fetchAssignments.setOptions({ - queryFn: getFetchAssignmentsQuery(mapDocument), + queryFn: getNullableParamQuery(getAssignments, mapDocument), queryKey: ['assignments', performance.now()], }); }; @@ -103,3 +125,16 @@ fetchAssignments.subscribe(assignments => { useMapStore.getState().loadZoneAssignments(assignments.data); } }); + +export const fetchTotPop = new QueryObserver | null>(queryClient, { + queryKey: ['gerrydb_tot_pop'], + queryFn: getNullableParamQuery>(getP1TotPopSummaryStats), +}); + +export const updateTotPop = (mapDocument: DocumentObject | null) => { + fetchTotPop.setOptions({ + queryFn: getNullableParamQuery(getP1TotPopSummaryStats, mapDocument), + queryKey: ['gerrydb_tot_pop', mapDocument?.gerrydb_table], + }); +}; + diff --git a/app/src/app/utils/numbers.ts b/app/src/app/utils/numbers.ts new file mode 100644 index 000000000..fb6443070 --- /dev/null +++ b/app/src/app/utils/numbers.ts @@ -0,0 +1,33 @@ +const percentFormatter = new Intl.NumberFormat('en-US', { + style: 'percent', + minimumFractionDigits: 0, + maximumFractionDigits: 1, +}) +const compactFormatter = new Intl.NumberFormat('en-US', { + notation: 'compact', + compactDisplay: 'short', +}) + +const stringFormatter = (n: number) => (Math.round(n)).toLocaleString() + +export type NumberFormats = 'percent' | 'string' | 'compact' +export const formatNumber = ( + value: number | undefined, + format: NumberFormats +) => { + if (value === undefined) { + return value + } + switch(format){ + case 'percent': + return percentFormatter.format(value) + case 'string': // Added case for 'string' + return stringFormatter(value) // Format as string + case 'compact': // Added case for 'compact' + return compactFormatter.format(value) // Format as compact + default: + const exhaustiveCheck: never = format; + throw new Error(`Unhandled format case: ${exhaustiveCheck}`); + + } +} \ No newline at end of file diff --git a/app/src/app/utils/summaryStats.ts b/app/src/app/utils/summaryStats.ts new file mode 100644 index 000000000..5b663967a --- /dev/null +++ b/app/src/app/utils/summaryStats.ts @@ -0,0 +1,31 @@ +import { P1ZoneSummaryStats } from "./api/apiHandlers"; + +export const getEntryTotal = (entry: Omit) => + Object.entries(entry).reduce((total, [key, value]) => { + if (key !== 'zone') { + return total + value; // Sum values of properties except 'zone' + } + return total; // Return total unchanged for 'zone' + }, 0); + + +export const sumArray = (arr: number[]) => arr.reduce((total, value) => total + value, 0); +export const stdDevArray = (arr: number[]) => { + const mean = sumArray(arr) / arr.length; // Calculate mean + const variance = arr.reduce((total, value) => total + Math.pow(value - mean, 2), 0) / arr.length; // Calculate variance + return Math.sqrt(variance); // Return standard deviation +} + +export const stdDevColors = { + [-2]: '#5e3c9977', + [-1]: '#b2abd277', + [0]: "#ffffff", + [1]: '#fdb86377', + [2]: '#e6610177' +} as const + +export const getStdDevColor = (value: number) => { + const floorValue = value > 0 ? Math.floor(value) : Math.ceil(value) + const cleanValue= (floorValue < -2 ? -2 : floorValue > 2 ? 2 : floorValue) as keyof typeof stdDevColors + return stdDevColors[cleanValue] || 'none' +} \ No newline at end of file diff --git a/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py b/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py new file mode 100644 index 000000000..4fc8ca915 --- /dev/null +++ b/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py @@ -0,0 +1,30 @@ +"""create summary stat metadata endpoint + +Revision ID: 5d9f7335f98a +Revises: 65a4fc0a727d +Create Date: 2024-09-11 10:15:07.929311 + +""" + +from typing import Sequence, Union + +from alembic import op +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "5d9f7335f98a" +down_revision: Union[str, None] = "65a4fc0a727d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with open(SQL_DIR / "available_summary_stat_udf.sql", "r") as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + sql = "DROP FUNCTION IF EXISTS get_available_summary_stats;" + op.execute(sql) diff --git a/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py new file mode 100644 index 000000000..091aa3bf3 --- /dev/null +++ b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py @@ -0,0 +1,50 @@ +"""add available stats to districtrmap + +Revision ID: c3541f016d35 +Revises: 5d9f7335f98a +Create Date: 2024-11-10 12:56:36.766141 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c3541f016d35" +down_revision: Union[str, None] = "5d9f7335f98a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "districtrmap", + sa.Column("available_summary_stats", sa.ARRAY(sa.TEXT()), nullable=True), + ) + + op.execute( + sa.text(""" + UPDATE districtrmap d + SET available_summary_stats = ( + SELECT + CASE WHEN d.child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer)) + END + ) + """) + ) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("districtrmap", "available_summary_stats") + # ### end Alembic commands ### diff --git a/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py b/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py new file mode 100644 index 000000000..ef65c474a --- /dev/null +++ b/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py @@ -0,0 +1,32 @@ +"""summary stat udfs + +Revision ID: f86991e63a62 +Revises: c3541f016d35 +Create Date: 2024-11-10 14:17:46.753393 + +""" + +from typing import Sequence, Union + +from alembic import op +from pathlib import Path +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "f86991e63a62" +down_revision: Union[str, None] = "c3541f016d35" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + for udf in ["summary_stats_p1.sql", "summary_stats_pop_totals.sql"]: + with Path(SQL_DIR, udf).open() as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + op.execute("DROP FUNCTION IF EXISTS get_summary_stats_p1") + op.execute("DROP FUNCTION IF EXISTS get_summary_stats_pop_totals") diff --git a/backend/app/main.py b/backend/app/main.py index e6cf29cd2..ef31b6399 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -26,6 +26,10 @@ DistrictrMapPublic, ParentChildEdges, ShatterResult, + SummaryStatisticType, + SummaryStatsP1, + PopulationStatsP1, + GerryDbSummaryStatisticType, ) if settings.ENVIRONMENT == "production": @@ -90,29 +94,6 @@ async def create_document( ) document_id = results.one()[0] # should be only one row, one column of results - # TODO: make this whole thing a function like get_total_population() - column_name_result = session.execute( - text( - """ - SELECT column_name - FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = :table_name - AND table_schema = 'gerrydb' - AND column_name IN ('total_pop', 'total_vap') - ORDER BY column_name ASC - LIMIT 1; - """ - ), - {"table_name": data.gerrydb_table}, - ).scalar() - if column_name_result: - population_subquery = select( - func.sum(text(column_name_result)).label("total_population") - ).select_from(text(f"gerrydb.{data.gerrydb_table}")) - else: - # neither column exists, this should never happen - population_subquery = select(text("NULL").label("total_population")) - stmt = ( select( Document.document_id, @@ -125,8 +106,9 @@ async def create_document( DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore - # get total population based on parent layer - population_subquery.as_scalar().label("total_population"), + DistrictrMap.available_summary_stats.label( + "available_summary_stats" + ), # pyright: ignore ) .where(Document.document_id == document_id) .join( @@ -279,40 +261,6 @@ async def get_assignments(document_id: str, session: Session = Depends(get_sessi @app.get("/api/document/{document_id}", response_model=DocumentPublic) async def get_document(document_id: str, session: Session = Depends(get_session)): - # TODO: make this whole thing a function like get_total_population() - # get gerrydb name based on doc id - gerrydb_table_name = session.execute( - text( - """ - SELECT gerrydb_table - FROM document.document - WHERE document_id = :document_id - """ - ), - {"document_id": document_id}, - ).scalar() - - column_name_result = session.execute( - text( - """ - SELECT column_name - FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = :table_name - AND table_schema = 'gerrydb' - AND column_name IN ('total_pop', 'total_vap') - ORDER BY column_name ASC - LIMIT 1; - """ - ), - {"table_name": gerrydb_table_name}, - ).scalar() - if column_name_result: - population_subquery = select( - func.sum(text(column_name_result)).label("total_population") - ).select_from(text(f"gerrydb.{gerrydb_table_name}")) - else: - # neither column exists, this should never happen - population_subquery = select(text("NULL").label("total_population")) stmt = ( select( Document.document_id, @@ -324,8 +272,9 @@ async def get_document(document_id: str, session: Session = Depends(get_session) DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore - # get total population based on parent layer - population_subquery.as_scalar().label("total_population"), + DistrictrMap.available_summary_stats.label( + "available_summary_stats" + ), # pyright: ignore ) # pyright: ignore .where(Document.document_id == document_id) .join( @@ -367,6 +316,97 @@ async def get_total_population( ) +@app.get("/api/document/{document_id}/{summary_stat}") +async def get_summary_stat( + document_id: str, summary_stat: str, session: Session = Depends(get_session) +): + try: + _summary_stat = SummaryStatisticType[summary_stat] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid summary_stat: {summary_stat}", + ) + + try: + stmt, SummaryStatsModel = { + "P1": ( + text( + "SELECT * from get_summary_stats_p1(:document_id) WHERE zone is not null" + ), + SummaryStatsP1, + ), + "P1TOTPOP": { + text("SELECT * from get_summary_stats_pop_totals(:document_id)"), + PopulationStatsP1, + }, + }[summary_stat] + except KeyError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Summary stats not implemented for {summary_stat}", + ) + + try: + results = session.execute(stmt, {"document_id": document_id}).fetchall() + return { + "summary_stat": _summary_stat.value, + "results": [SummaryStatsModel.from_orm(row) for row in results], + } + except ProgrammingError as e: + logger.error(e) + error_text = str(e) + if f"Table name not found for document_id: {document_id}" in error_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Document with ID {document_id} not found", + ) + + +@app.get("/api/districtrmap/summary_stats/{summary_stat}/{gerrydb_table}") +async def get_gerrydb_summary_stat( + summary_stat: str, gerrydb_table: str, session: Session = Depends(get_session) +): + try: + _summary_stat = GerryDbSummaryStatisticType[summary_stat] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid summary_stat: {summary_stat}", + ) + + try: + summary_stat_udf, SummaryStatsModel = { + "P1TOTPOP": ("get_summary_stats_pop_totals", PopulationStatsP1), + }[summary_stat] + except KeyError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Summary stats not implemented for {summary_stat}", + ) + + stmt = text( + f"""SELECT * + FROM {summary_stat_udf}(:gerrydb_table)""" + ).bindparams( + bindparam(key="gerrydb_table", type_=String), + ) + try: + results = session.execute(stmt, {"gerrydb_table": gerrydb_table}).fetchall() + return { + "summary_stat": _summary_stat.value, + "results": [SummaryStatsModel.from_orm(row) for row in results], + } + except ProgrammingError as e: + logger.error(e) + error_text = str(e) + if f"Table {gerrydb_table} does not exist in gerrydb schema" in error_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Gerrydb Table with ID {gerrydb_table} not found", + ) + + @app.get("/api/gerrydb/views", response_model=list[DistrictrMapPublic]) async def get_projects( *, diff --git a/backend/app/models.py b/backend/app/models.py index 54838e340..e37a1d08d 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,6 +1,6 @@ from datetime import datetime from typing import Optional -from pydantic import UUID4, BaseModel +from pydantic import UUID4, BaseModel, ConfigDict from sqlmodel import ( Field, ForeignKey, @@ -13,9 +13,11 @@ MetaData, String, ) -from sqlalchemy.types import ARRAY +from sqlalchemy.types import ARRAY, TEXT from sqlalchemy import Float from app.constants import DOCUMENT_SCHEMA +from enum import Enum +from typing import Any class UUIDType(UUID): @@ -44,6 +46,17 @@ class TimeStampMixin(SQLModel): ) +class SummaryStatisticType(Enum): + P1 = "Population by Race" + P2 = "Hispanic or Latino, and Not Hispanic or Latino by Race" + P3 = "Voting Age Population by Race" + P4 = "Hispanic or Latino, and Not Hispanic or Latino by Race Voting Age Population" + + +class GerryDbSummaryStatisticType(Enum): + P1TOTPOP = "Total Population by Race" + + class DistrictrMap(TimeStampMixin, SQLModel, table=True): uuid: str = Field(sa_column=Column(UUIDType, unique=True, primary_key=True)) name: str = Field(nullable=False) @@ -64,12 +77,15 @@ class DistrictrMap(TimeStampMixin, SQLModel, table=True): String, ForeignKey("gerrydbtable.name"), default=None, nullable=True ) ) - extent: list[float] = Field(sa_column=Column(ARRAY(Float), nullable=True)) + extent: list[float] | None = Field(sa_column=Column(ARRAY(Float), nullable=True)) districtr_place_id: str | None = Field(nullable=True) # schema? will need to contrain the schema # where does this go? # when you create the view, pull the columns that you need # we'll want discrete management steps + available_summary_stats: list[SummaryStatisticType] | None = Field( + sa_column=Column(ARRAY(TEXT), nullable=True, default=[]) + ) class DistrictrMapPublic(BaseModel): @@ -79,6 +95,7 @@ class DistrictrMapPublic(BaseModel): child_layer: str | None = None tiles_s3_path: str | None = None num_districts: int | None = None + available_summary_stats: list[str] | None = None class GerryDBTable(TimeStampMixin, SQLModel, table=True): @@ -131,7 +148,7 @@ class DocumentPublic(BaseModel): created_at: datetime updated_at: datetime extent: list[float] | None = None - total_population: int | None = None + available_summary_stats: list[str] | None = None class AssignmentsBase(SQLModel): @@ -220,3 +237,22 @@ class DistrictrPlace(BaseModel): "id", "place_type", "state", name="unique_id_place_type_state" ), ) + + +class SummaryStats(BaseModel): + summary_stat: SummaryStatisticType + results: list[Any] + + +class PopulationStatsP1(BaseModel): + model_config = ConfigDict(from_attributes=True) + other_pop: int + asian_pop: int + amin_pop: int + nhpi_pop: int + black_pop: int + white_pop: int + + +class SummaryStatsP1(PopulationStatsP1): + zone: int diff --git a/backend/app/sql/available_summary_stat_udf.sql b/backend/app/sql/available_summary_stat_udf.sql new file mode 100644 index 000000000..6fde7c4dd --- /dev/null +++ b/backend/app/sql/available_summary_stat_udf.sql @@ -0,0 +1,82 @@ +CREATE OR REPLACE FUNCTION get_available_summary_stats(gerrydb_table_name TEXT) +RETURNS TABLE (summary_stat TEXT) AS $$ +DECLARE + p1 BOOLEAN; + p2 BOOLEAN; + p3 BOOLEAN; + p4 BOOLEAN; +BEGIN + SELECT count(column_name) = 6 INTO p1 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('other_pop', + 'asian_pop', + 'amin_pop', + 'nhpi_pop', + 'black_pop', + 'white_pop') + ; + + SELECT count(column_name) = 6 INTO p3 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('other_vap', + 'asian_vap', + 'amin_vap', + 'nhpi_vap', + 'black_vap', + 'white_vap') + ; + + SELECT count(column_name) = 7 INTO p2 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('hispanic_pop', + 'non_hispanic_asian_pop', + 'non_hispanic_amin_pop', + 'non_hispanic_nhpi_pop', + 'non_hispanic_black_pop', + 'non_hispanic_white_pop', + 'non_hispanic_other_pop' + ) + ; + + SELECT count(column_name) = 7 INTO p4 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('hispanic_pop', + 'non_hispanic_asian_vap', + 'non_hispanic_amin_vap', + 'non_hispanic_nhpi_vap', + 'non_hispanic_black_vap', + 'non_hispanic_white_vap', + 'non_hispanic_other_vap' + ) + ; + + RETURN QUERY + + SELECT 'P1' as summary_stat + WHERE p1 + + UNION + + SELECT 'P2' + WHERE p2 + + UNION + + SELECT 'P3' + WHERE p3 + + UNION + + SELECT 'P4' + WHERE p4; + +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p1.sql b/backend/app/sql/summary_stats_p1.sql new file mode 100644 index 000000000..bc9fa994e --- /dev/null +++ b/backend/app/sql/summary_stats_p1.sql @@ -0,0 +1,42 @@ +CREATE OR REPLACE FUNCTION get_summary_stats_p1(document_id UUID) +RETURNS TABLE ( + zone TEXT, + other_pop BIGINT, + asian_pop BIGINT, + amin_pop BIGINT, + nhpi_pop BIGINT, + black_pop BIGINT, + white_pop BIGINT +) AS $$ +DECLARE + doc_districtrmap RECORD; + sql_query TEXT; +BEGIN + SELECT districtrmap.* INTO doc_districtrmap + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF doc_districtrmap.gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.other_pop, 0))::BIGINT AS other_pop, + SUM(COALESCE(blocks.asian_pop, 0))::BIGINT AS asian_pop, + SUM(COALESCE(blocks.amin_pop, 0))::BIGINT AS amin_pop, + SUM(COALESCE(blocks.nhpi_pop, 0))::BIGINT AS nhpi_pop, + SUM(COALESCE(blocks.black_pop, 0))::BIGINT AS black_pop, + SUM(COALESCE(blocks.white_pop, 0))::BIGINT AS white_pop + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', doc_districtrmap.gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_pop_totals.sql b/backend/app/sql/summary_stats_pop_totals.sql new file mode 100644 index 000000000..196e8ae1d --- /dev/null +++ b/backend/app/sql/summary_stats_pop_totals.sql @@ -0,0 +1,40 @@ +DROP FUNCTION IF EXISTS get_summary_stats_pop_totals; +CREATE OR REPLACE FUNCTION get_summary_stats_pop_totals(gerrydb_table TEXT) +RETURNS TABLE ( + other_pop BIGINT, + asian_pop BIGINT, + amin_pop BIGINT, + nhpi_pop BIGINT, + black_pop BIGINT, + white_pop BIGINT +) AS $$ +DECLARE + table_exists BOOLEAN; + sql_query TEXT; + +BEGIN + -- Check if the table exists + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'gerrydb' + AND table_name = $1 + ) INTO table_exists; + + IF NOT table_exists THEN + RAISE EXCEPTION 'Table % does not exist in gerrydb schema', $1; + END IF; + + sql_query := format(' + SELECT + SUM(COALESCE(other_pop, 0))::BIGINT AS other_pop, + SUM(COALESCE(asian_pop, 0))::BIGINT AS asian_pop, + SUM(COALESCE(amin_pop, 0))::BIGINT AS amin_pop, + SUM(COALESCE(nhpi_pop, 0))::BIGINT AS nhpi_pop, + SUM(COALESCE(black_pop, 0))::BIGINT AS black_pop, + SUM(COALESCE(white_pop, 0))::BIGINT AS white_pop + FROM gerrydb.%I + ', $1); + RETURN QUERY EXECUTE sql_query; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/backend/app/utils.py b/backend/app/utils.py index 007e3a3e2..adb9a8eb5 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -5,7 +5,7 @@ import logging -from app.models import UUIDType +from app.models import SummaryStatisticType, UUIDType logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -202,3 +202,69 @@ def add_extent_to_districtrmap( END $$; """) session.execute(stmt) + + +def get_available_summary_stats(session: Session, gerrydb_table_name: str): + """ + Get the available summary statistics for a given gerrydb table. + + Args: + session: The database session. + gerrydb_table_name: The name of the gerrydb table. + """ + stmt = text("SELECT * FROM get_available_summary_stats(:gerrydb_table_name)") + return session.execute( + stmt, + { + "gerrydb_table_name": gerrydb_table_name, + }, + ).all() + + +def add_available_summary_stats_to_districtrmap( + session: Session, districtr_map_uuid: str, summary_stats: list[str] | None = None +) -> list[SummaryStatisticType] | None: + """ + Add the available summary statistics to the districtr map. + + Args: + session: The database session. + districtr_map_uuid: The UUID of the districtr map. + summary_stats: The summary statistics to add. + """ + if summary_stats is not None: + raise NotImplementedError( + "Manually adding summary stats to a districtr map is not yet implemented." + ) + + stmt = text( + """ + UPDATE districtrmap + SET available_summary_stats = + CASE WHEN child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(parent_layer)) + END + WHERE uuid = :districtr_map_uuid + RETURNING available_summary_stats + """ + ).bindparams( + bindparam(key="districtr_map_uuid", type_=UUIDType), + ) + result = session.execute( + stmt, + { + "districtr_map_uuid": districtr_map_uuid, + "summary_stats": summary_stats, + }, + ) + (available_summary_stats,) = result.one() + logger.info( + f"Updated available summary stats for districtr map {districtr_map_uuid} to {available_summary_stats}" + ) + return available_summary_stats diff --git a/backend/cli.py b/backend/cli.py index 11b592109..5aab2a6fa 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -13,6 +13,7 @@ create_shatterable_gerrydb_view as _create_shatterable_gerrydb_view, create_parent_child_edges as _create_parent_child_edges, add_extent_to_districtrmap as _add_extent_to_districtrmap, + add_available_summary_stats_to_districtrmap as _add_available_summary_stats_to_districtrmap, ) logger = logging.getLogger(__name__) @@ -233,6 +234,10 @@ def create_districtr_map( session=session, districtr_map_uuid=districtr_map_uuid, bounds=bounds ) + _add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + session.commit() logger.info(f"Districtr map created successfully {districtr_map_uuid}") @@ -292,5 +297,26 @@ def add_extent_to_districtr_map(districtr_map: str, bounds: list[float] | None = session.close() +@cli.command("add-available-summary-stats-to-districtr-map") +@click.option("--districtr-map", "-d", help="Districtr map name", required=True) +def add_available_summary_stats_to_districtr_map(districtr_map: str): + session = next(get_session()) + stmt = text( + "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" + ) + (districtr_map_uuid,) = session.execute( + stmt, params={"districtrmap_name": districtr_map} + ).one() + print(f"Found districtmap uuid: {districtr_map_uuid}") + + _add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + + session.commit() + logger.info("Updated available summary stats successfully.") + session.close() + + if __name__ == "__main__": cli() diff --git a/backend/load_data.py b/backend/load_data.py index 2e9faf981..bc1ac99e8 100755 --- a/backend/load_data.py +++ b/backend/load_data.py @@ -76,8 +76,8 @@ def load_sample_data(config): ) result = session.execute(exists_query).scalar() - if result > 0: - print(f"###\Districtr map {name} already exists.\n###") + if result is not None and result > 0: + print(f"Districtr map {name} already exists.") else: subprocess.run( [ diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson new file mode 100644 index 000000000..572cefc2e --- /dev/null +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson @@ -0,0 +1,17 @@ +{ +"type": "FeatureCollection", +"name": "SELECT", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "other_pop": 12, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "other_pop": 1, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "other_pop": 5, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "other_pop": 24, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "other_pop": 32, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +] +} diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 530a1dd1b..1598136cd 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -11,7 +11,7 @@ OGR2OGR_PG_CONNECTION_STRING, FIXTURES_PATH, ) -from app.utils import create_districtr_map +from app.utils import create_districtr_map, add_available_summary_stats_to_districtrmap def test_read_main(client): @@ -30,6 +30,7 @@ def test_get_session(): GERRY_DB_FIXTURE_NAME = "ks_demo_view_census_blocks" GERRY_DB_TOTAL_VAP_FIXTURE_NAME = "ks_demo_view_census_blocks_total_vap" GERRY_DB_NO_POP_FIXTURE_NAME = "ks_demo_view_census_blocks_no_pop" +GERRY_DB_P1_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats" ## Test DB @@ -450,3 +451,89 @@ def test_list_gerydb_views_offset_and_limit(client, districtr_maps): assert response.status_code == 200 data = response.json() assert len(data) == 1 + assert data[0]["name"] == "Districtr map ks_demo_view_census_blocks" + + +@pytest.fixture(name=GERRY_DB_P1_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats(session: Session): + layer = GERRY_DB_P1_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.execute(upsert_query, {"name": layer}) + + (districtr_map_uuid,) = create_districtr_map( + session=session, + name="DistrictMap with P1 view", + parent_layer_name=layer, + gerrydb_table_name=layer, + ) + add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + + session.commit() + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="document_id_p1_summary_stats") +def document_summary_stats_fixture(client, ks_demo_view_census_blocks_summary_stats): + response = client.post( + "/api/create_document", + json={ + "gerrydb_table": GERRY_DB_P1_FIXTURE_NAME, + }, + ) + document_id = response.json()["document_id"] + return document_id + + +def test_get_p1_summary_stats(client, document_id_p1_summary_stats): + # Set up assignments + document_id = document_id_p1_summary_stats + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": 2}, + ] + }, + ) + + summary_stat = "P1" + response = client.get(f"/api/document/{document_id}/{summary_stat}") + data = response.json() + assert response.status_code == 200 + assert data.get("summary_stat") == "Population by Race" + results = data.get("results") + assert results is not None + assert len(results) == 2 + record_1, record_2 = data.get("results") + assert record_1.get("zone") == 1 + assert record_2.get("zone") == 2 + assert record_1.get("other_pop") == 13 + assert record_2.get("other_pop") == 24 diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 883fcc631..7e03b154a 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -5,6 +5,7 @@ create_shatterable_gerrydb_view, create_parent_child_edges, add_extent_to_districtrmap, + get_available_summary_stats, ) from sqlmodel import Session import subprocess @@ -126,6 +127,42 @@ def districtr_map_fixture( return inserted_districtr_map +GERRY_DB_P1_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats" + + +@pytest.fixture(name=GERRY_DB_P1_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats(session: Session): + layer = GERRY_DB_P1_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.begin() + session.execute(upsert_query, {"name": GERRY_DB_P1_FIXTURE_NAME}) + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + # FOR THE TESTS BELOW I NEED TO ADD ACTUAL ASSERTIONS @@ -240,6 +277,18 @@ def test_shattering(client, session: Session, document_id): assert all(d["zone"] == 1 for d in data["children"]) +def test_get_available_summary_stats( + session: Session, ks_demo_view_census_blocks_summary_stats +): + result = get_available_summary_stats(session, GERRY_DB_P1_FIXTURE_NAME) + assert len(result) == 1 + (summary_stats_available,) = result + assert summary_stats_available + assert len(summary_stats_available) == 1 + (summary_stat,) = summary_stats_available + assert summary_stat == "P1" + + def test_unshatter_process(client, document_id): response = client.patch( "/api/update_assignments",