diff --git a/app/package-lock.json b/app/package-lock.json index 0ffe56b48..4d1b8d6a9 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -19,6 +19,7 @@ "@turf/bbox": "^7.1.0", "@turf/helpers": "^7.1.0", "axios": "^1.7.2", + "d3-scale-chromatic": "^3.1.0", "idb-keyval": "^6.2.1", "lodash": "^4.17.21", "maplibre-gl": "^4.4.1", @@ -35,6 +36,7 @@ "@flydotio/dockerfile": "^0.5.8", "@testing-library/jest-dom": "^6.4.6", "@testing-library/react": "^16.0.0", + "@types/d3-scale-chromatic": "^3.0.3", "@types/lodash": "^4.17.5", "@types/node": "^20", "@types/react": "^18", @@ -4403,7 +4405,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/bbox/-/bbox-7.1.0.tgz", "integrity": "sha512-PdWPz9tW86PD78vSZj2fiRaB8JhUHy6piSa/QXb83lucxPK+HTAdzlDQMTKj5okRCU8Ox/25IR2ep9T8NdopRA==", - "license": "MIT", "dependencies": { "@turf/helpers": "^7.1.0", "@turf/meta": "^7.1.0", @@ -4418,7 +4419,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/helpers/-/helpers-7.1.0.tgz", "integrity": "sha512-dTeILEUVeNbaEeoZUOhxH5auv7WWlOShbx7QSd4s0T4Z0/iz90z9yaVCtZOLbU89umKotwKaJQltBNO9CzVgaQ==", - "license": "MIT", "dependencies": { "@types/geojson": "^7946.0.10", "tslib": "^2.6.2" @@ -4431,7 +4431,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@turf/meta/-/meta-7.1.0.tgz", "integrity": "sha512-ZgGpWWiKz797Fe8lfRj7HKCkGR+nSJ/5aKXMyofCvLSc2PuYJs/qyyifDPWjASQQCzseJ7AlF2Pc/XQ/3XkkuA==", - "license": "MIT", "dependencies": { "@turf/helpers": "^7.1.0", "@types/geojson": "^7946.0.10" @@ -4532,6 +4531,12 @@ "@types/d3-time": "*" } }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.3.tgz", + "integrity": "sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==", + "dev": true + }, "node_modules/@types/d3-shape": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz", @@ -6171,6 +6176,18 @@ "node": ">=12" } }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", diff --git a/app/package.json b/app/package.json index a61167bbb..90abcf044 100644 --- a/app/package.json +++ b/app/package.json @@ -21,6 +21,7 @@ "@turf/bbox": "^7.1.0", "@turf/helpers": "^7.1.0", "axios": "^1.7.2", + "d3-scale-chromatic": "^3.1.0", "idb-keyval": "^6.2.1", "lodash": "^4.17.21", "maplibre-gl": "^4.4.1", @@ -37,6 +38,7 @@ "@flydotio/dockerfile": "^0.5.8", "@testing-library/jest-dom": "^6.4.6", "@testing-library/react": "^16.0.0", + "@types/d3-scale-chromatic": "^3.0.3", "@types/lodash": "^4.17.5", "@types/node": "^20", "@types/react": "^18", @@ -50,4 +52,4 @@ "tailwindcss": "^3.4.1", "typescript": "^5" } -} +} \ No newline at end of file diff --git a/app/src/app/components/sidebar/DataPanels.tsx b/app/src/app/components/sidebar/DataPanels.tsx index ec16ce4ec..f99b7d269 100644 --- a/app/src/app/components/sidebar/DataPanels.tsx +++ b/app/src/app/components/sidebar/DataPanels.tsx @@ -1,11 +1,7 @@ -import {Box, Flex, Heading} from '@radix-ui/themes'; -import {MapModeSelector} from './MapModeSelector'; -import {ColorPicker} from './ColorPicker'; -import {ResetMapButton} from './ResetMapButton'; -import {GerryDBViewSelector} from './GerryDBViewSelector'; +import {Box} from '@radix-ui/themes'; +import Evaluation from '@components/sidebar/Evaluation'; import {HorizontalBar} from './charts/HorizontalBarChart'; -import {useMapStore} from '@/app/store/mapStore'; -import {Tabs, Text} from '@radix-ui/themes'; +import {Tabs} from '@radix-ui/themes'; import Layers from './Layers'; import React from 'react'; @@ -35,7 +31,7 @@ const defaultPanels: DataPanelSpec[] = [ { title: 'evaluation', label: 'Evaluation', - content: Unimplemented , + content: , }, ]; diff --git a/app/src/app/components/sidebar/Evaluation.tsx b/app/src/app/components/sidebar/Evaluation.tsx new file mode 100644 index 000000000..af31c3a09 --- /dev/null +++ b/app/src/app/components/sidebar/Evaluation.tsx @@ -0,0 +1,325 @@ +import React, {useMemo, useState} from 'react'; +import {useMapStore} from '@/app/store/mapStore'; +import {useQuery} from '@tanstack/react-query'; +import { + CleanedP1ZoneSummaryStats, + CleanedP1ZoneSummaryStatsKeys, + getP1SummaryStats, + P1ZoneSummaryStats, + P1ZoneSummaryStatsKeys, +} from '@/app/utils/api/apiHandlers'; +import {Button, Checkbox, CheckboxGroup} from '@radix-ui/themes'; +import {Heading, Flex, Spinner, Text} from '@radix-ui/themes'; +import {queryClient} from '@utils/api/queryClient'; +import {formatNumber, NumberFormats} from '@/app/utils/numbers'; +import {colorScheme} from '@/app/constants/colors'; +import { + getEntryTotal, + getStdDevColor, + stdDevArray, + stdDevColors, + sumArray, +} from '@utils/summaryStats'; +import {interpolateBlues, interpolateGreys} from 'd3-scale-chromatic'; + +type EvalModes = 'share' | 'count' | 'totpop'; +type ColumnConfiguration> = Array<{label: string; column: keyof T}>; +type EvaluationProps = { + columnConfig?: ColumnConfiguration; +}; + +// const calculateColumn = ( +// mode: EvalModes, +// entry: P1ZoneSummaryStats, +// totals: P1ZoneSummaryStats, +// column: keyof Omit +// ) => { +// const count = entry[column]; +// switch (mode) { +// case 'count': +// return count; +// case 'pct': +// return count / entry['total']; +// case 'share': +// return count / totals[column]; +// } +// }; + +const defaultColumnConfig: ColumnConfiguration = [ + { + label: 'White', + column: 'white_pop', + }, + { + label: 'Black', + column: 'black_pop', + }, + { + label: 'Asian', + column: 'asian_pop', + }, + { + label: 'Am. Indian', + column: 'amin_pop', + }, + { + label: 'Pacific Isl.', + column: 'nhpi_pop', + }, + { + label: 'Other', + column: 'other_pop', + }, +]; + +const modeButtonConfig: Array<{label: string; value: EvalModes}> = [ + { + label: 'Population by Share', + value: 'share', + }, + { + label: 'Population by Count', + value: 'count', + }, + // { + // label: "Population by Percent of Zone", + // value: 'totpop' + // } +]; + +const numberFormats: Record = { + share: 'percent', + count: 'string', + totpop: 'percent', +}; + +const getColConfig = (evalMode: EvalModes) => { + switch (evalMode) { + case 'share': + return (col: keyof P1ZoneSummaryStats) => `${col}_pct` as keyof CleanedP1ZoneSummaryStats; + default: + return (col: keyof P1ZoneSummaryStats) => col; + } +}; + +const Evaluation: React.FC = ({columnConfig = defaultColumnConfig}) => { + const [evalMode, setEvalMode] = useState('share'); + // const [showAverages, setShowAverages] = useState(true); + // const [showStdDev, setShowStdDev] = useState(false); + const [colorBg, setColorBg] = useState(true); + const [showUnassigned, setShowUnassigned] = useState(true); + + const numberFormat = numberFormats[evalMode]; + const columnGetter = getColConfig(evalMode); + const totPop = useMapStore(state => state.summaryStats.totpop?.data); + const mapDocument = useMapStore(state => state.mapDocument); + const assignmentsHash = useMapStore(state => state.assignmentsHash); + + const {data, error, isLoading} = useQuery( + { + queryKey: ['p1SummaryStats', mapDocument, assignmentsHash], + queryFn: () => mapDocument && getP1SummaryStats(mapDocument), + enabled: !!mapDocument, + staleTime: 0, + placeholderData: previousData => previousData, + }, + queryClient + ); + + const { + unassigned, + maxValues, + // averages, + // stdDevs + } = useMemo(() => { + if (!data?.results || !totPop) { + return {}; + } + let maxValues: Record = {}; + + let unassigned: Record = { + ...totPop, + zone: -999, + total: getEntryTotal(totPop), + }; + P1ZoneSummaryStatsKeys.forEach(key => { + let total = unassigned[key]; + maxValues[key] = -Math.pow(10, 12); + data.results.forEach(row => { + total -= row[key]; + maxValues[key] = Math.max(row[key], maxValues[key]); + }); + unassigned[`${key}_pct`] = total / unassigned[key]; + unassigned[key] = total; + }); + // const averages: Record = {}; + // const stdDevs: Record = {}; + // CleanedP1ZoneSummaryStatsKeys.forEach(key => { + // const values = data.results.map(row => row[key]); + // averages[key] = sumArray(values) / data.results.length; + // stdDevs[key] = stdDevArray(values); + // }); + return { + unassigned, + maxValues, + // averages, + // stdDevs + }; + }, [data?.results, totPop]); + + if (!data || !maxValues || (mapDocument && !mapDocument.available_summary_stats)) { + return Summary statistics are not available for this map.; + } + + if (error) { + return ( +
+

Summary Statistics

+

There was an error loading the summary statistics.

+
+ ); + } + const rows = unassigned && showUnassigned ? [...data.results, unassigned] : data.results; + return ( +
+ + {modeButtonConfig.map((mode, i) => ( + + ))} + {isLoading && } + + + + + setShowUnassigned(v => !v)}> + Show Unassigned Population + + {/* setShowAverages(v => !v)}> + Show Zone Averages + + setShowStdDev(v => !v)}> + Show Zone Std. Dev. + */} + setColorBg(v => !v)}> + +

Color Cells By Values

+ {/* {colorByStdDev && ( + + {Object.entries(stdDevColors) + .sort((a, b) => +a[0] - +b[0]) + .map(([stdev, backgroundColor], i) => ( + + {+stdev > 0 ? `+${stdev}`: stdev} + + ))} + + )} */} + + + + +
+ + + + + {columnConfig.map((f, i) => ( + + ))} + + + + {/* {!!(averages && showAverages) && ( + + + {columnConfig.map((f, i) => ( + + ))} + + )} + {!!(stdDevs && showStdDev) && ( + + + {columnConfig.map((f, i) => ( + + ))} + + )} */} + {rows + .sort((a, b) => a.zone - b.zone) + .map(row => { + const isUnassigned = row.zone === -999; + const zoneName = isUnassigned ? 'None' : row.zone; + const backgroundColor = isUnassigned ? '#DDDDDD' : colorScheme[row.zone - 1]; + + return ( + + + {columnConfig.map((f, i) => { + const column = columnGetter(f.column); + const colorValue = + evalMode === 'count' ? row[column] / maxValues[column] : row[column]; + const backgroundColor = + colorBg && !isUnassigned + ? interpolateGreys(colorValue) + .replace('rgb', 'rgba') + .replace(')', ',0.5)') + : 'initial'; + return ( + + ); + })} + + ); + })} + +
Zone + {f.label} +
+ Zone Averages + + {formatNumber(averages[columnGetter(f.column)], numberFormat)} +
+ Zone Std. Dev. + + {formatNumber(stdDevs[columnGetter(f.column)], numberFormat)} +
+ + {zoneName} + + {formatNumber(row[column], numberFormat)} +
+
+
+ ); +}; + +export default Evaluation; diff --git a/app/src/app/store/mapStore.ts b/app/src/app/store/mapStore.ts index 19cabb0ce..dccfb99d7 100644 --- a/app/src/app/store/mapStore.ts +++ b/app/src/app/store/mapStore.ts @@ -8,6 +8,7 @@ import { Assignment, DistrictrMap, DocumentObject, + P1TotPopSummaryStats, ShatterResult, ZonePopulation, } from '../utils/api/apiHandlers'; @@ -22,19 +23,19 @@ import { getFeaturesInBbox, resetZoneColors, setZones, -} from '../utils/helpers'; -import {getRenderSubscriptions} from './mapRenderSubs'; +} from "../utils/helpers"; +import { getRenderSubscriptions } from "./mapRenderSubs"; +import { getSearchParamsObersver } from "../utils/api/queryParamsListener"; +import { getMapMetricsSubs } from "./metricsSubs"; +import { getMapEditSubs } from "./mapEditSubs"; +import { getQueriesResultsSubs } from "../utils/api/queries"; +import { persistOptions } from "./persistConfig"; import {patchReset, patchShatter, patchUnShatter} from '../utils/api/mutations'; -import {getSearchParamsObersver} from '../utils/api/queryParamsListener'; -import {getMapMetricsSubs} from './metricsSubs'; -import {getMapEditSubs} from './mapEditSubs'; import bbox from '@turf/bbox'; import {BLOCK_SOURCE_ID} from '../constants/layers'; -import {getMapViewsSubs} from '../utils/api/queries'; -import {persistOptions} from './persistConfig'; -import {onlyUnique} from '../utils/arrays'; import {DistrictrMapOptions} from './types'; -import {queryClient} from '../utils/api/queryClient'; +import { onlyUnique } from '../utils/arrays'; +import { queryClient } from '../utils/api/queryClient'; const combineSetValues = (setRecord: Record>, keys?: string[]) => { const combinedSet = new Set(); // Create a new set to hold combined values @@ -77,6 +78,15 @@ export interface MapStore { */ mapDocument: DocumentObject | null; setMapDocument: (mapDocument: DocumentObject) => void; + summaryStats: { + totpop?: { + data: P1TotPopSummaryStats + } + }, + setSummaryStat: ( + stat: T, + value: MapStore['summaryStats'][T] + ) => void, // SHATTERING /** * A subset of IDs that a user is working on in a focused view. @@ -222,6 +232,8 @@ export interface MapStore { resetAccumulatedBlockPopulations: () => void; zoneAssignments: Map; // geoid -> zone setZoneAssignments: (zone: NullableZone, gdbPaths: Set) => void; + assignmentsHash: string; + setAssignmentsHash: (hash: string) => void; loadZoneAssignments: (assigments: Assignment[]) => void; resetZoneAssignments: () => void; zonePopulations: Map; @@ -381,6 +393,15 @@ export const useMapStore = create( shatterIds: {parents: new Set(), children: new Set()}, }); }, + summaryStats: {}, + setSummaryStat: (stat, value) => { + set({ + summaryStats: { + ...get().summaryStats, + [stat]: value + } + }) + }, // TODO: Refactor to something like this // featureStates: { // locked: [], @@ -612,12 +633,8 @@ export const useMapStore = create( userMaps.splice(i, 1, userMapData); // Replace the map at index i with the new data } else { const urlParams = new URL(window.location.href).searchParams; - urlParams.delete('document_id'); // Remove the document_id parameter - window.history.pushState( - {}, - '', - window.location.pathname + '?' + urlParams.toString() - ); // Update the URL without document_id + urlParams.delete("document_id"); // Remove the document_id parameter + window.history.pushState({}, '', window.location.pathname + '?' + urlParams.toString()); // Update the URL without document_id userMaps.splice(i, 1); } } @@ -766,6 +783,8 @@ export const useMapStore = create( selectedZone: 1, setSelectedZone: zone => set({selectedZone: zone}), zoneAssignments: new Map(), + assignmentsHash: "", + setAssignmentsHash: (hash) => set({ assignmentsHash: hash }), accumulatedGeoids: new Set(), setAccumulatedGeoids: accumulatedGeoids => set({accumulatedGeoids}), setZoneAssignments: (zone, geoids) => { @@ -867,6 +886,6 @@ export const useMapStore = create( // these need to initialize after the map store getRenderSubscriptions(useMapStore); getMapMetricsSubs(useMapStore); -getMapViewsSubs(useMapStore); +getQueriesResultsSubs(useMapStore); getMapEditSubs(useMapStore); getSearchParamsObersver(); diff --git a/app/src/app/store/metricsSubs.ts b/app/src/app/store/metricsSubs.ts index 25a8c7713..7329a6787 100644 --- a/app/src/app/store/metricsSubs.ts +++ b/app/src/app/store/metricsSubs.ts @@ -1,4 +1,4 @@ -import {updateMapMetrics} from '../utils/api/queries'; +import {updateMapMetrics, updateTotPop} from '../utils/api/queries'; import {useMapStore as _useMapStore} from './mapStore'; export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { @@ -7,6 +7,7 @@ export const getMapMetricsSubs = (useMapStore: typeof _useMapStore) => { mapDocument => { if (mapDocument) { updateMapMetrics(mapDocument); + updateTotPop(mapDocument) } } ); diff --git a/app/src/app/utils/api/apiHandlers.ts b/app/src/app/utils/api/apiHandlers.ts index 4b7736a27..e9886b4ae 100644 --- a/app/src/app/utils/api/apiHandlers.ts +++ b/app/src/app/utils/api/apiHandlers.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import 'maplibre-gl'; import {useMapStore} from '@/app/store/mapStore'; +import { getEntryTotal } from '../summaryStats'; export const FormatAssignments = () => { const assignments = Array.from(useMapStore.getState().zoneAssignments.entries()).map( @@ -61,6 +62,7 @@ export interface DocumentObject { created_at: string; updated_at: string | null; extent: [number, number, number, number]; // [minx, miny, maxx, maxy] + available_summary_stats: string[]; } /** @@ -105,7 +107,7 @@ export const getDocument: (document_id: string) => Promise = asy }; export const getAssignments: ( - mapDocument: DocumentObject + mapDocument: DocumentObject | null ) => Promise = async mapDocument => { if (mapDocument) { return await axios @@ -149,6 +151,108 @@ export const getZonePopulations: ( } }; +export interface SummaryStatsResult { + summary_stat: string; + results: T; +} + +/** + * P1ZoneSummaryStats + * + * @interface + * @property {number} zone - The zone. + * @property {number} total_pop - The total population. + */ +export interface P1ZoneSummaryStats { + zone: number; + other_pop: number; + asian_pop: number; + amin_pop: number; + nhpi_pop: number; + black_pop: number; + white_pop: number; +} +export type P1TotPopSummaryStats = Omit + +export const P1ZoneSummaryStatsKeys = [ + 'other_pop', + 'asian_pop', + 'amin_pop', + 'nhpi_pop', + 'black_pop', + 'white_pop' +] as const + +export const CleanedP1ZoneSummaryStatsKeys = [ + ...P1ZoneSummaryStatsKeys, + 'total', + 'other_pop_pct', + 'asian_pop_pct', + 'amin_pop_pct', + 'nhpi_pop_pct', + 'black_pop_pct', + 'white_pop_pct', +] as const + +export interface CleanedP1ZoneSummaryStats extends P1ZoneSummaryStats { + total: number; + other_pop_pct: number; + asian_pop_pct: number; + amin_pop_pct: number; + nhpi_pop_pct: number; + black_pop_pct: number; + white_pop_pct: number; +} + +/** + * Get P1 zone stats from the server. + * @param mapDocument - DocumentObject, the document object + * @returns Promise + */ +export const getP1SummaryStats: ( + mapDocument: DocumentObject +) => Promise> = async mapDocument => { + if (mapDocument) { + return await axios + .get>(`${process.env.NEXT_PUBLIC_API_URL}/api/document/${mapDocument.document_id}/P1`) + .then(res => { + const results = res.data.results.map(row => { + const total = getEntryTotal(row) + return P1ZoneSummaryStatsKeys.reduce((acc, key) => { + acc[`${key}_pct`] = acc[key] / total; + return acc; + }, { + ...row, + total + }) as CleanedP1ZoneSummaryStats + }) + return { + ...res.data, + results + } + }) + } else { + throw new Error('No document provided'); + } +}; + +/** + * Get P1 zone stats from the server. + * @param mapDocument - DocumentObject, the document object + * @returns Promise + */ +export const getP1TotPopSummaryStats: ( + mapDocument: DocumentObject | null +) => Promise> = async mapDocument => { + if (mapDocument) { + return await axios + .get>(`${process.env.NEXT_PUBLIC_API_URL}/api/districtrmap/summary_stats/P1/${mapDocument.parent_layer}`) + .then(res => res.data) + } else { + throw new Error('No document provided'); + } +}; + /** * Get available DistrictrMap views from the server. * @param limit - number, the number of views to return (default 10, max 100) diff --git a/app/src/app/utils/api/mutations.ts b/app/src/app/utils/api/mutations.ts index 30ae99dff..a8dbc0ecc 100644 --- a/app/src/app/utils/api/mutations.ts +++ b/app/src/app/utils/api/mutations.ts @@ -27,6 +27,7 @@ export const patchShatter = new MutationObserver(queryClient, { }, onSuccess: data => { console.log(`Successfully shattered parents into ${data.children.length} children`); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); return data; }, }); @@ -60,6 +61,7 @@ export const patchUpdates = new MutationObserver(queryClient, { }, onSuccess: (data: AssignmentsCreate) => { console.log(`Successfully upserted ${data.assignments_upserted} assignments`); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); mapMetrics.refetch(); // remove trailing shattered features // This needs to happen AFTER the updates are done @@ -96,6 +98,7 @@ export const document = new MutationObserver(queryClient, { }, onSuccess: data => { useMapStore.getState().setMapDocument(data); + useMapStore.getState().setAssignmentsHash(performance.now().toString()); useMapStore.getState().setAppLoadingState('loaded'); const documentUrl = new URL(window.location.toString()); documentUrl.searchParams.set('document_id', data.document_id); diff --git a/app/src/app/utils/api/queries.ts b/app/src/app/utils/api/queries.ts index e50f7996c..2549b3c0c 100644 --- a/app/src/app/utils/api/queries.ts +++ b/app/src/app/utils/api/queries.ts @@ -9,12 +9,32 @@ import { getDocument, getZonePopulations, ZonePopulation, + SummaryStatsResult, + getP1TotPopSummaryStats, + P1TotPopSummaryStats, } from './apiHandlers'; import {MapStore, useMapStore} from '@/app/store/mapStore'; const INITIAL_VIEW_LIMIT = 30; const INITIAL_VIEW_OFFSET = 0; +/** + * A utility function that returns a query function based on a nullable parameter. + * + * @param callback - A function that takes a parameter of type ParamT and returns a Promise of type ResultT. + * @param nullableParam - An optional parameter of type ParamT. If this parameter is not provided or is falsy, the function will return a function that returns null. + * + * @returns A function that, when called, will either return null (if nullableParam is not provided) + * or call the callback function with the nullableParam and return its result. + * + * @template ParamT - The type of the parameter that the callback function accepts. + * @template ResultT - The type of the result that the callback function returns. + */ +const getNullableParamQuery = (callback: (param: ParamT) => Promise, nullableParam?: ParamT) => { + if (!nullableParam) return () => null; + return async () => await callback(nullableParam); +}; + export const mapMetrics = new QueryObserver(queryClient, { queryKey: ['_zonePopulations'], queryFn: skipToken, @@ -43,12 +63,19 @@ export const updateMapViews = (limit: number, offset: number) => { }); }; -export const getMapViewsSubs = (_useMapStore: typeof useMapStore) => { +export const getQueriesResultsSubs = (_useMapStore: typeof useMapStore) => { mapViewsQuery.subscribe(result => { if (result) { _useMapStore.getState().setMapViews(result); } }); + fetchTotPop.subscribe(response => { + if (response?.data?.results) { + useMapStore.getState().setSummaryStat('totpop', { data: response.data.results}); + } else { + useMapStore.getState().setSummaryStat('totpop', undefined) + } + }); }; const getDocumentFunction = (documentId?: string) => { @@ -81,19 +108,14 @@ updateDocumentFromId.subscribe(mapDocument => { } }); -const getFetchAssignmentsQuery = (mapDocument?: MapStore['mapDocument']) => { - if (!mapDocument) return () => null; - return async () => await getAssignments(mapDocument); -}; - export const fetchAssignments = new QueryObserver(queryClient, { queryKey: ['assignments'], - queryFn: getFetchAssignmentsQuery(), + queryFn: getNullableParamQuery(getAssignments) }); export const updateAssignments = (mapDocument: DocumentObject) => { fetchAssignments.setOptions({ - queryFn: getFetchAssignmentsQuery(mapDocument), + queryFn: getNullableParamQuery(getAssignments, mapDocument), queryKey: ['assignments', performance.now()], }); }; @@ -103,3 +125,16 @@ fetchAssignments.subscribe(assignments => { useMapStore.getState().loadZoneAssignments(assignments.data); } }); + +export const fetchTotPop = new QueryObserver | null>(queryClient, { + queryKey: ['gerrydb_tot_pop'], + queryFn: getNullableParamQuery>(getP1TotPopSummaryStats), +}); + +export const updateTotPop = (mapDocument: DocumentObject | null) => { + fetchTotPop.setOptions({ + queryFn: getNullableParamQuery(getP1TotPopSummaryStats, mapDocument), + queryKey: ['gerrydb_tot_pop', mapDocument?.gerrydb_table], + }); +}; + diff --git a/app/src/app/utils/numbers.ts b/app/src/app/utils/numbers.ts new file mode 100644 index 000000000..fb6443070 --- /dev/null +++ b/app/src/app/utils/numbers.ts @@ -0,0 +1,33 @@ +const percentFormatter = new Intl.NumberFormat('en-US', { + style: 'percent', + minimumFractionDigits: 0, + maximumFractionDigits: 1, +}) +const compactFormatter = new Intl.NumberFormat('en-US', { + notation: 'compact', + compactDisplay: 'short', +}) + +const stringFormatter = (n: number) => (Math.round(n)).toLocaleString() + +export type NumberFormats = 'percent' | 'string' | 'compact' +export const formatNumber = ( + value: number | undefined, + format: NumberFormats +) => { + if (value === undefined) { + return value + } + switch(format){ + case 'percent': + return percentFormatter.format(value) + case 'string': // Added case for 'string' + return stringFormatter(value) // Format as string + case 'compact': // Added case for 'compact' + return compactFormatter.format(value) // Format as compact + default: + const exhaustiveCheck: never = format; + throw new Error(`Unhandled format case: ${exhaustiveCheck}`); + + } +} \ No newline at end of file diff --git a/app/src/app/utils/summaryStats.ts b/app/src/app/utils/summaryStats.ts new file mode 100644 index 000000000..5b663967a --- /dev/null +++ b/app/src/app/utils/summaryStats.ts @@ -0,0 +1,31 @@ +import { P1ZoneSummaryStats } from "./api/apiHandlers"; + +export const getEntryTotal = (entry: Omit) => + Object.entries(entry).reduce((total, [key, value]) => { + if (key !== 'zone') { + return total + value; // Sum values of properties except 'zone' + } + return total; // Return total unchanged for 'zone' + }, 0); + + +export const sumArray = (arr: number[]) => arr.reduce((total, value) => total + value, 0); +export const stdDevArray = (arr: number[]) => { + const mean = sumArray(arr) / arr.length; // Calculate mean + const variance = arr.reduce((total, value) => total + Math.pow(value - mean, 2), 0) / arr.length; // Calculate variance + return Math.sqrt(variance); // Return standard deviation +} + +export const stdDevColors = { + [-2]: '#5e3c9977', + [-1]: '#b2abd277', + [0]: "#ffffff", + [1]: '#fdb86377', + [2]: '#e6610177' +} as const + +export const getStdDevColor = (value: number) => { + const floorValue = value > 0 ? Math.floor(value) : Math.ceil(value) + const cleanValue= (floorValue < -2 ? -2 : floorValue > 2 ? 2 : floorValue) as keyof typeof stdDevColors + return stdDevColors[cleanValue] || 'none' +} \ No newline at end of file diff --git a/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py b/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py new file mode 100644 index 000000000..4fc8ca915 --- /dev/null +++ b/backend/app/alembic/versions/5d9f7335f98a_create_summary_stat_metadata_endpoint.py @@ -0,0 +1,30 @@ +"""create summary stat metadata endpoint + +Revision ID: 5d9f7335f98a +Revises: 65a4fc0a727d +Create Date: 2024-09-11 10:15:07.929311 + +""" + +from typing import Sequence, Union + +from alembic import op +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "5d9f7335f98a" +down_revision: Union[str, None] = "65a4fc0a727d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with open(SQL_DIR / "available_summary_stat_udf.sql", "r") as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + sql = "DROP FUNCTION IF EXISTS get_available_summary_stats;" + op.execute(sql) diff --git a/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py new file mode 100644 index 000000000..091aa3bf3 --- /dev/null +++ b/backend/app/alembic/versions/c3541f016d35_add_available_stats_to_districtrmap.py @@ -0,0 +1,50 @@ +"""add available stats to districtrmap + +Revision ID: c3541f016d35 +Revises: 5d9f7335f98a +Create Date: 2024-11-10 12:56:36.766141 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c3541f016d35" +down_revision: Union[str, None] = "5d9f7335f98a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "districtrmap", + sa.Column("available_summary_stats", sa.ARRAY(sa.TEXT()), nullable=True), + ) + + op.execute( + sa.text(""" + UPDATE districtrmap d + SET available_summary_stats = ( + SELECT + CASE WHEN d.child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(d.parent_layer)) + END + ) + """) + ) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("districtrmap", "available_summary_stats") + # ### end Alembic commands ### diff --git a/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py b/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py new file mode 100644 index 000000000..29c1aa36a --- /dev/null +++ b/backend/app/alembic/versions/f86991e63a62_summary_stat_udfs.py @@ -0,0 +1,39 @@ +"""summary stat udfs + +Revision ID: f86991e63a62 +Revises: c3541f016d35 +Create Date: 2024-11-10 14:17:46.753393 + +""" + +from typing import Sequence, Union + +from alembic import op +from pathlib import Path +from app.constants import SQL_DIR + + +# revision identifiers, used by Alembic. +revision: str = "f86991e63a62" +down_revision: Union[str, None] = "c3541f016d35" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + for udf in [ + "summary_stats_p1.sql", + "summary_stats_p1_totals.sql", + "summary_stats_p4.sql", + "summary_stats_p4_totals.sql", + ]: + with Path(SQL_DIR, udf).open() as f: + sql = f.read() + op.execute(sql) + + +def downgrade() -> None: + op.execute("DROP FUNCTION IF EXISTS get_summary_stats_p1") + op.execute("DROP FUNCTION IF EXISTS get_summary_p1_totals") + op.execute("DROP FUNCTION IF EXISTS get_summary_stats_p4") + op.execute("DROP FUNCTION IF EXISTS get_summary_p4_totals") diff --git a/backend/app/main.py b/backend/app/main.py index 57b6b8c33..72407b4d1 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -26,6 +26,11 @@ DistrictrMapPublic, ParentChildEdges, ShatterResult, + SummaryStatisticType, + SummaryStatsP1, + PopulationStatsP1, + SummaryStatsP4, + PopulationStatsP4, ) if settings.ENVIRONMENT == "production": @@ -101,6 +106,7 @@ async def create_document( DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore + DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore ) .where(Document.document_id == document_id) .join( @@ -260,6 +266,7 @@ async def get_document(document_id: str, session: Session = Depends(get_session) DistrictrMap.tiles_s3_path.label("tiles_s3_path"), # pyright: ignore DistrictrMap.num_districts.label("num_districts"), # pyright: ignore DistrictrMap.extent.label("extent"), # pyright: ignore + DistrictrMap.available_summary_stats.label("available_summary_stats"), # pyright: ignore ) # pyright: ignore .where(Document.document_id == document_id) .join( @@ -301,6 +308,100 @@ async def get_total_population( ) +@app.get("/api/document/{document_id}/{summary_stat}") +async def get_summary_stat( + document_id: str, summary_stat: str, session: Session = Depends(get_session) +): + try: + _summary_stat = SummaryStatisticType[summary_stat] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid summary_stat: {summary_stat}", + ) + + try: + stmt, SummaryStatsModel = { + "P1": ( + text( + "SELECT * from get_summary_stats_p1(:document_id) WHERE zone is not null" + ), + SummaryStatsP1, + ), + "P4": { + text( + "SELECT * from get_summary_stats_p4(:document_id) WHERE zone is not null" + ), + SummaryStatsP4, + }, + }[summary_stat] + except KeyError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Summary stats not implemented for {summary_stat}", + ) + + try: + results = session.execute(stmt, {"document_id": document_id}).fetchall() + return { + "summary_stat": _summary_stat.value, + "results": [SummaryStatsModel.from_orm(row) for row in results], + } + except ProgrammingError as e: + logger.error(e) + error_text = str(e) + if f"Table name not found for document_id: {document_id}" in error_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Document with ID {document_id} not found", + ) + + +@app.get("/api/districtrmap/summary_stats/{summary_stat}/{gerrydb_table}") +async def get_gerrydb_summary_stat( + summary_stat: str, gerrydb_table: str, session: Session = Depends(get_session) +): + try: + _summary_stat = SummaryStatisticType[summary_stat] + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid summary_stat: {summary_stat}", + ) + + try: + summary_stat_udf, SummaryStatsModel = { + "P1": ("get_summary_p1_totals", PopulationStatsP1), + "P4": ("get_summary_p4_totals", PopulationStatsP4), + }[summary_stat] + except KeyError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Summary stats not implemented for {summary_stat}", + ) + + stmt = text( + f"""SELECT * + FROM {summary_stat_udf}(:gerrydb_table)""" + ).bindparams( + bindparam(key="gerrydb_table", type_=String), + ) + try: + results = session.execute(stmt, {"gerrydb_table": gerrydb_table}).fetchone() + return { + "summary_stat": _summary_stat.value, + "results": SummaryStatsModel.from_orm(results), + } + except ProgrammingError as e: + logger.error(e) + error_text = str(e) + if f"Table {gerrydb_table} does not exist in gerrydb schema" in error_text: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Gerrydb Table with ID {gerrydb_table} not found", + ) + + @app.get("/api/gerrydb/views", response_model=list[DistrictrMapPublic]) async def get_projects( *, diff --git a/backend/app/models.py b/backend/app/models.py index 16fff215f..b40c20aee 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,6 +1,6 @@ from datetime import datetime from typing import Optional -from pydantic import UUID4, BaseModel +from pydantic import UUID4, BaseModel, ConfigDict from sqlmodel import ( Field, ForeignKey, @@ -13,9 +13,11 @@ MetaData, String, ) -from sqlalchemy.types import ARRAY +from sqlalchemy.types import ARRAY, TEXT from sqlalchemy import Float from app.constants import DOCUMENT_SCHEMA +from enum import Enum +from typing import Any class UUIDType(UUID): @@ -44,6 +46,13 @@ class TimeStampMixin(SQLModel): ) +class SummaryStatisticType(Enum): + P1 = "Population by Race" + P2 = "Hispanic or Latino, and Not Hispanic or Latino by Race" + P3 = "Voting Age Population by Race" + P4 = "Hispanic or Latino, and Not Hispanic or Latino by Race Voting Age Population" + + class DistrictrMap(TimeStampMixin, SQLModel, table=True): uuid: str = Field(sa_column=Column(UUIDType, unique=True, primary_key=True)) name: str = Field(nullable=False) @@ -64,11 +73,14 @@ class DistrictrMap(TimeStampMixin, SQLModel, table=True): String, ForeignKey("gerrydbtable.name"), default=None, nullable=True ) ) - extent: list[float] = Field(sa_column=Column(ARRAY(Float), nullable=True)) + extent: list[float] | None = Field(sa_column=Column(ARRAY(Float), nullable=True)) # schema? will need to contrain the schema # where does this go? # when you create the view, pull the columns that you need # we'll want discrete management steps + available_summary_stats: list[SummaryStatisticType] | None = Field( + sa_column=Column(ARRAY(TEXT), nullable=True, default=[]) + ) class DistrictrMapPublic(BaseModel): @@ -78,6 +90,7 @@ class DistrictrMapPublic(BaseModel): child_layer: str | None = None tiles_s3_path: str | None = None num_districts: int | None = None + available_summary_stats: list[str] | None = None class GerryDBTable(TimeStampMixin, SQLModel, table=True): @@ -130,6 +143,7 @@ class DocumentPublic(BaseModel): created_at: datetime updated_at: datetime extent: list[float] | None = None + available_summary_stats: list[str] | None = None class AssignmentsBase(SQLModel): @@ -175,3 +189,37 @@ class ShatterResult(BaseModel): class ZonePopulation(BaseModel): zone: int total_pop: int + + +class SummaryStats(BaseModel): + summary_stat: SummaryStatisticType + results: list[Any] + + +class PopulationStatsP1(BaseModel): + model_config = ConfigDict(from_attributes=True) + other_pop: int + asian_pop: int + amin_pop: int + nhpi_pop: int + black_pop: int + white_pop: int + + +class SummaryStatsP1(PopulationStatsP1): + zone: int + + +class PopulationStatsP4(BaseModel): + model_config = ConfigDict(from_attributes=True) + hispanic_vap: int + non_hispanic_asian_vap: int + non_hispanic_amin_vap: int + non_hispanic_nhpi_vap: int + non_hispanic_black_vap: int + non_hispanic_white_vap: int + non_hispanic_other_vap: int + + +class SummaryStatsP4(PopulationStatsP4): + zone: int diff --git a/backend/app/sql/available_summary_stat_udf.sql b/backend/app/sql/available_summary_stat_udf.sql new file mode 100644 index 000000000..7992c03e5 --- /dev/null +++ b/backend/app/sql/available_summary_stat_udf.sql @@ -0,0 +1,82 @@ +CREATE OR REPLACE FUNCTION get_available_summary_stats(gerrydb_table_name TEXT) +RETURNS TABLE (summary_stat TEXT) AS $$ +DECLARE + p1 BOOLEAN; + p2 BOOLEAN; + p3 BOOLEAN; + p4 BOOLEAN; +BEGIN + SELECT count(column_name) = 6 INTO p1 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('other_pop', + 'asian_pop', + 'amin_pop', + 'nhpi_pop', + 'black_pop', + 'white_pop') + ; + + SELECT count(column_name) = 6 INTO p3 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('other_vap', + 'asian_vap', + 'amin_vap', + 'nhpi_vap', + 'black_vap', + 'white_vap') + ; + + SELECT count(column_name) = 7 INTO p2 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('hispanic_pop', + 'non_hispanic_asian_pop', + 'non_hispanic_amin_pop', + 'non_hispanic_nhpi_pop', + 'non_hispanic_black_pop', + 'non_hispanic_white_pop', + 'non_hispanic_other_pop' + ) + ; + + SELECT count(column_name) = 7 INTO p4 + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = gerrydb_table_name + AND table_schema = 'gerrydb' + AND column_name IN ('hispanic_vap', + 'non_hispanic_asian_vap', + 'non_hispanic_amin_vap', + 'non_hispanic_nhpi_vap', + 'non_hispanic_black_vap', + 'non_hispanic_white_vap', + 'non_hispanic_other_vap' + ) + ; + + RETURN QUERY + + SELECT 'P1' as summary_stat + WHERE p1 + + UNION + + SELECT 'P2' + WHERE p2 + + UNION + + SELECT 'P3' + WHERE p3 + + UNION + + SELECT 'P4' + WHERE p4; + +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p1.sql b/backend/app/sql/summary_stats_p1.sql new file mode 100644 index 000000000..bc9fa994e --- /dev/null +++ b/backend/app/sql/summary_stats_p1.sql @@ -0,0 +1,42 @@ +CREATE OR REPLACE FUNCTION get_summary_stats_p1(document_id UUID) +RETURNS TABLE ( + zone TEXT, + other_pop BIGINT, + asian_pop BIGINT, + amin_pop BIGINT, + nhpi_pop BIGINT, + black_pop BIGINT, + white_pop BIGINT +) AS $$ +DECLARE + doc_districtrmap RECORD; + sql_query TEXT; +BEGIN + SELECT districtrmap.* INTO doc_districtrmap + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF doc_districtrmap.gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.other_pop, 0))::BIGINT AS other_pop, + SUM(COALESCE(blocks.asian_pop, 0))::BIGINT AS asian_pop, + SUM(COALESCE(blocks.amin_pop, 0))::BIGINT AS amin_pop, + SUM(COALESCE(blocks.nhpi_pop, 0))::BIGINT AS nhpi_pop, + SUM(COALESCE(blocks.black_pop, 0))::BIGINT AS black_pop, + SUM(COALESCE(blocks.white_pop, 0))::BIGINT AS white_pop + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', doc_districtrmap.gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p1_totals.sql b/backend/app/sql/summary_stats_p1_totals.sql new file mode 100644 index 000000000..9b65c9ad9 --- /dev/null +++ b/backend/app/sql/summary_stats_p1_totals.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE FUNCTION get_summary_p1_totals(gerrydb_table TEXT) +RETURNS TABLE ( + other_pop BIGINT, + asian_pop BIGINT, + amin_pop BIGINT, + nhpi_pop BIGINT, + black_pop BIGINT, + white_pop BIGINT +) AS $$ +DECLARE + table_exists BOOLEAN; + sql_query TEXT; + +BEGIN + -- Check if the table exists + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'gerrydb' + AND table_name = $1 + ) INTO table_exists; + + IF NOT table_exists THEN + RAISE EXCEPTION 'Table % does not exist in gerrydb schema', $1; + END IF; + + sql_query := format(' + SELECT + SUM(COALESCE(other_pop, 0))::BIGINT AS other_pop, + SUM(COALESCE(asian_pop, 0))::BIGINT AS asian_pop, + SUM(COALESCE(amin_pop, 0))::BIGINT AS amin_pop, + SUM(COALESCE(nhpi_pop, 0))::BIGINT AS nhpi_pop, + SUM(COALESCE(black_pop, 0))::BIGINT AS black_pop, + SUM(COALESCE(white_pop, 0))::BIGINT AS white_pop + FROM gerrydb.%I + ', $1); + RETURN QUERY EXECUTE sql_query; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p4.sql b/backend/app/sql/summary_stats_p4.sql new file mode 100644 index 000000000..1c75bfbcf --- /dev/null +++ b/backend/app/sql/summary_stats_p4.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION get_summary_stats_p4(document_id UUID) +RETURNS TABLE ( + zone TEXT, + hispanic_vap BIGINT, + non_hispanic_asian_vap BIGINT, + non_hispanic_amin_vap BIGINT, + non_hispanic_nhpi_vap BIGINT, + non_hispanic_black_vap BIGINT, + non_hispanic_white_vap BIGINT, + non_hispanic_other_vap BIGINT +) AS $$ +DECLARE + doc_districtrmap RECORD; + sql_query TEXT; +BEGIN + SELECT districtrmap.* INTO doc_districtrmap + FROM document.document + LEFT JOIN districtrmap + ON document.gerrydb_table = districtrmap.gerrydb_table_name + WHERE document.document_id = $1; + + IF doc_districtrmap.gerrydb_table_name IS NULL THEN + RAISE EXCEPTION 'Table name not found for document_id: %', $1; + END IF; + + sql_query := format(' + SELECT + assignments.zone::TEXT AS zone, + SUM(COALESCE(blocks.hispanic_vap, 0))::BIGINT AS hispanic_vap, + SUM(COALESCE(blocks.non_hispanic_asian_vap, 0))::BIGINT AS non_hispanic_asian_vap, + SUM(COALESCE(blocks.non_hispanic_amin_vap, 0))::BIGINT AS non_hispanic_amin_vap, + SUM(COALESCE(blocks.non_hispanic_nhpi_vap, 0))::BIGINT AS non_hispanic_nhpi_vap, + SUM(COALESCE(blocks.non_hispanic_black_vap, 0))::BIGINT AS non_hispanic_black_vap, + SUM(COALESCE(blocks.non_hispanic_white_vap, 0))::BIGINT AS non_hispanic_white_vap, + SUM(COALESCE(blocks.non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap + FROM document.assignments + LEFT JOIN gerrydb.%I blocks + ON blocks.path = assignments.geo_id + WHERE assignments.document_id = $1 + GROUP BY assignments.zone + ', doc_districtrmap.gerrydb_table_name); + RETURN QUERY EXECUTE sql_query USING $1; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/sql/summary_stats_p4_totals.sql b/backend/app/sql/summary_stats_p4_totals.sql new file mode 100644 index 000000000..a9c57f12d --- /dev/null +++ b/backend/app/sql/summary_stats_p4_totals.sql @@ -0,0 +1,41 @@ +CREATE OR REPLACE FUNCTION get_summary_p4_totals(gerrydb_table TEXT) +RETURNS TABLE ( + hispanic_vap BIGINT, + non_hispanic_asian_vap BIGINT, + non_hispanic_amin_vap BIGINT, + non_hispanic_nhpi_vap BIGINT, + non_hispanic_black_vap BIGINT, + non_hispanic_white_vap BIGINT, + non_hispanic_other_vap BIGINT +) AS $$ +DECLARE + table_exists BOOLEAN; + sql_query TEXT; + +BEGIN + -- Check if the table exists + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'gerrydb' + AND table_name = $1 + ) INTO table_exists; + + IF NOT table_exists THEN + RAISE EXCEPTION 'Table % does not exist in gerrydb schema', $1; + END IF; + + sql_query := format(' + SELECT + SUM(COALESCE(hispanic_vap, 0))::BIGINT AS hispanic_vap, + SUM(COALESCE(non_hispanic_asian_vap, 0))::BIGINT AS non_hispanic_asian_vap, + SUM(COALESCE(non_hispanic_amin_vap, 0))::BIGINT AS non_hispanic_amin_vap, + SUM(COALESCE(non_hispanic_nhpi_vap, 0))::BIGINT AS non_hispanic_nhpi_vap, + SUM(COALESCE(non_hispanic_black_vap, 0))::BIGINT AS non_hispanic_black_vap, + SUM(COALESCE(non_hispanic_white_vap, 0))::BIGINT AS non_hispanic_white_vap, + SUM(COALESCE(non_hispanic_other_vap, 0))::BIGINT AS non_hispanic_other_vap + FROM gerrydb.%I + ', $1); + RETURN QUERY EXECUTE sql_query; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/app/utils.py b/backend/app/utils.py index 007e3a3e2..adb9a8eb5 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -5,7 +5,7 @@ import logging -from app.models import UUIDType +from app.models import SummaryStatisticType, UUIDType logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -202,3 +202,69 @@ def add_extent_to_districtrmap( END $$; """) session.execute(stmt) + + +def get_available_summary_stats(session: Session, gerrydb_table_name: str): + """ + Get the available summary statistics for a given gerrydb table. + + Args: + session: The database session. + gerrydb_table_name: The name of the gerrydb table. + """ + stmt = text("SELECT * FROM get_available_summary_stats(:gerrydb_table_name)") + return session.execute( + stmt, + { + "gerrydb_table_name": gerrydb_table_name, + }, + ).all() + + +def add_available_summary_stats_to_districtrmap( + session: Session, districtr_map_uuid: str, summary_stats: list[str] | None = None +) -> list[SummaryStatisticType] | None: + """ + Add the available summary statistics to the districtr map. + + Args: + session: The database session. + districtr_map_uuid: The UUID of the districtr map. + summary_stats: The summary statistics to add. + """ + if summary_stats is not None: + raise NotImplementedError( + "Manually adding summary stats to a districtr map is not yet implemented." + ) + + stmt = text( + """ + UPDATE districtrmap + SET available_summary_stats = + CASE WHEN child_layer IS NOT NULL THEN + ( + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(child_layer) + INTERSECT + SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(parent_layer) + ) + ELSE + (SELECT ARRAY_AGG(summary_stat) FROM get_available_summary_stats(parent_layer)) + END + WHERE uuid = :districtr_map_uuid + RETURNING available_summary_stats + """ + ).bindparams( + bindparam(key="districtr_map_uuid", type_=UUIDType), + ) + result = session.execute( + stmt, + { + "districtr_map_uuid": districtr_map_uuid, + "summary_stats": summary_stats, + }, + ) + (available_summary_stats,) = result.one() + logger.info( + f"Updated available summary stats for districtr map {districtr_map_uuid} to {available_summary_stats}" + ) + return available_summary_stats diff --git a/backend/cli.py b/backend/cli.py index 11b592109..5aab2a6fa 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -13,6 +13,7 @@ create_shatterable_gerrydb_view as _create_shatterable_gerrydb_view, create_parent_child_edges as _create_parent_child_edges, add_extent_to_districtrmap as _add_extent_to_districtrmap, + add_available_summary_stats_to_districtrmap as _add_available_summary_stats_to_districtrmap, ) logger = logging.getLogger(__name__) @@ -233,6 +234,10 @@ def create_districtr_map( session=session, districtr_map_uuid=districtr_map_uuid, bounds=bounds ) + _add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + session.commit() logger.info(f"Districtr map created successfully {districtr_map_uuid}") @@ -292,5 +297,26 @@ def add_extent_to_districtr_map(districtr_map: str, bounds: list[float] | None = session.close() +@cli.command("add-available-summary-stats-to-districtr-map") +@click.option("--districtr-map", "-d", help="Districtr map name", required=True) +def add_available_summary_stats_to_districtr_map(districtr_map: str): + session = next(get_session()) + stmt = text( + "SELECT uuid FROM districtrmap WHERE gerrydb_table_name = :districtrmap_name" + ) + (districtr_map_uuid,) = session.execute( + stmt, params={"districtrmap_name": districtr_map} + ).one() + print(f"Found districtmap uuid: {districtr_map_uuid}") + + _add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + + session.commit() + logger.info("Updated available summary stats successfully.") + session.close() + + if __name__ == "__main__": cli() diff --git a/backend/load_data.py b/backend/load_data.py index 2e9faf981..bc1ac99e8 100755 --- a/backend/load_data.py +++ b/backend/load_data.py @@ -76,8 +76,8 @@ def load_sample_data(config): ) result = session.execute(exists_query).scalar() - if result > 0: - print(f"###\Districtr map {name} already exists.\n###") + if result is not None and result > 0: + print(f"Districtr map {name} already exists.") else: subprocess.run( [ diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson new file mode 100644 index 000000000..572cefc2e --- /dev/null +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats.geojson @@ -0,0 +1,17 @@ +{ +"type": "FeatureCollection", +"name": "SELECT", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "other_pop": 12, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "other_pop": 1, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "other_pop": 5, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "other_pop": 24, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "other_pop": 32, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "other_pop": 0, "amin_pop": 0, "asian_pop":0, "black_pop":0,"nhpi_pop":0,"white_pop":0, "total_pop": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +] +} diff --git a/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson new file mode 100644 index 000000000..d84d67435 --- /dev/null +++ b/backend/tests/fixtures/ks_demo_view_census_blocks_summary_stats_p4.geojson @@ -0,0 +1,17 @@ +{ +"type": "FeatureCollection", +"name": "SELECT", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::26914" } }, +"features": [ +{ "type": "Feature", "properties": { "path": "202090416004010", "area_land": 33168, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 12, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 55 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875211.081238693208434, 4337960.930915 ], [ 875265.224863927578554, 4337963.745180247351527 ], [ 875300.303792983642779, 4337971.768200713209808 ], [ 875327.274071992840618, 4337987.972723919898272 ], [ 875343.876800142694265, 4338007.019734212197363 ], [ 875356.197026390116662, 4338032.2054428094998 ], [ 875359.471527923597023, 4338063.413040107116103 ], [ 875355.397699509863742, 4338153.921939895488322 ], [ 875347.256564430193976, 4338195.378358344547451 ], [ 875334.623098340583965, 4338223.820951136760414 ], [ 875387.976528111146763, 4338241.289160016924143 ], [ 875410.516155685996637, 4338221.4454699838534 ], [ 875421.307235877029598, 4338045.005893977358937 ], [ 875432.630792022915557, 4338000.474406631663442 ], [ 875445.411641188431531, 4337979.829491405747831 ], [ 875454.801698453957215, 4337964.809443462640047 ], [ 875472.424312707735226, 4337940.835065085440874 ], [ 875484.035351007943973, 4337917.463120688684285 ], [ 875418.360436515184119, 4337916.432314324192703 ], [ 875375.462586269248277, 4337914.155511460267007 ], [ 875220.291984744369984, 4337906.170478757470846 ], [ 875212.765701709431596, 4337905.810247281566262 ], [ 875211.081238693208434, 4337960.930915 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090416003004", "area_land": 15823, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 1, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 12 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 876029.603093245532364, 4338611.491121606901288 ], [ 876194.572833474143408, 4338619.185103545896709 ], [ 876196.43945312872529, 4338582.102246845141053 ], [ 876199.421181682031602, 4338523.593026914633811 ], [ 876114.734792487695813, 4338519.528731964528561 ], [ 876099.337269587209448, 4338518.789868013001978 ], [ 876033.778705667937174, 4338515.421732313930988 ], [ 876029.603093245532364, 4338611.491121606901288 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090443032011", "area_land": 19257, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 5, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 31 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 869231.309174439054914, 4340396.545289561152458 ], [ 869319.693646978936158, 4340400.600859931670129 ], [ 869326.915629425784573, 4340398.159290821291506 ], [ 869328.812702536699362, 4340394.68766363710165 ], [ 869330.334983806591481, 4340389.974244618788362 ], [ 869330.934905984206125, 4340386.441455170512199 ], [ 869335.664594965986907, 4340326.571299341507256 ], [ 869338.182566312723793, 4340216.18538093008101 ], [ 869339.686451153829694, 4340210.024406461976469 ], [ 869335.697759646456689, 4340206.386548922397196 ], [ 869242.119611647445709, 4340200.305704364553094 ], [ 869237.76920640678145, 4340277.887773043476045 ], [ 869231.309174439054914, 4340396.545289561152458 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090434001003", "area_land": 24816, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 24, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 130 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 875930.37992833124008, 4332349.252949083223939 ], [ 875933.443328902358189, 4332350.512517770752311 ], [ 875936.723918574396521, 4332352.672826421447098 ], [ 875940.119178901193663, 4332356.062846168875694 ], [ 875943.424073546309955, 4332361.340518746525049 ], [ 875944.319405541173182, 4332364.388283201493323 ], [ 875945.113742337096483, 4332369.54577558953315 ], [ 875944.670285670785233, 4332382.434507312253118 ], [ 875945.567135536577553, 4332387.263029311783612 ], [ 875949.17501455033198, 4332398.89889903459698 ], [ 875951.559500945499167, 4332403.464760144241154 ], [ 875956.987613479956053, 4332411.515112683176994 ], [ 875970.784156027017161, 4332424.195230172947049 ], [ 876010.736134674632922, 4332445.139112876728177 ], [ 876044.262139945523813, 4332455.313976712524891 ], [ 876060.651922063203529, 4332457.323036558926105 ], [ 876103.475923568708822, 4332458.149635425768793 ], [ 876110.933183946879581, 4332291.567219044081867 ], [ 876066.784754700609483, 4332280.327024504542351 ], [ 876050.379456249298528, 4332273.19777974113822 ], [ 876027.842756676953286, 4332258.207085125148296 ], [ 875961.742888991255313, 4332328.495757032185793 ], [ 875952.4995389302494, 4332338.514782093465328 ], [ 875941.136727001168765, 4332343.869353833608329 ], [ 875938.680846909992397, 4332344.419547958299518 ], [ 875935.657703495351598, 4332345.944222977384925 ], [ 875924.61156704777386, 4332348.309071445837617 ], [ 875930.37992833124008, 4332349.252949083223939 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202099800001035", "area_land": 151703, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877813.040280948858708, 4342314.444725112058222 ], [ 877809.014222223311663, 4342502.010962888598442 ], [ 877808.392830823198892, 4342556.183347844518721 ], [ 878187.281443994143046, 4342575.815153966657817 ], [ 878188.527672112570144, 4342539.257936611771584 ], [ 878203.946146892383695, 4342177.056350266560912 ], [ 877870.8709020371316, 4342163.087458959780633 ], [ 877820.343733718618751, 4342161.316630367189646 ], [ 877813.040280948858708, 4342314.444725112058222 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "202090429003012", "area_land": 27367, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 32, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 139 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 877171.124916166299954, 4338452.87533472944051 ], [ 877432.574727031751536, 4338466.573728412389755 ], [ 877438.946534773334861, 4338361.259861093014479 ], [ 877177.250717416638508, 4338348.996695580892265 ], [ 877171.124916166299954, 4338452.87533472944051 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201730056003001", "area_land": 40244, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 13 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 647618.257487860973924, 4162037.393264054320753 ], [ 647677.756609870004468, 4162038.897054230328649 ], [ 647719.070099569740705, 4162039.966174147557467 ], [ 647719.63226254703477, 4162018.332780665252358 ], [ 647786.68576369935181, 4161814.082042149733752 ], [ 647814.53967670770362, 4161742.877915579359978 ], [ 647835.253349754726514, 4161715.499313783831894 ], [ 647931.670532181044109, 4161647.849339275155216 ], [ 647941.242692932020873, 4161641.02766244718805 ], [ 647884.676134104607627, 4161638.35352012841031 ], [ 647883.100702673778869, 4161637.54847350390628 ], [ 647881.001446478301659, 4161636.401107432320714 ], [ 647879.341679102857597, 4161635.372573387343436 ], [ 647876.241502788383514, 4161635.872232513967901 ], [ 647868.445927470806055, 4161637.176076213829219 ], [ 647866.150291252415627, 4161637.135130017530173 ], [ 647770.255428230622783, 4161635.869233383797109 ], [ 647766.972603302798234, 4161651.571556095965207 ], [ 647760.328006867086515, 4161672.541566835716367 ], [ 647720.724104659864679, 4161768.731609313283116 ], [ 647664.439331455505453, 4161899.032108286395669 ], [ 647627.416294273571111, 4161989.274946445599198 ], [ 647621.465386576252058, 4162006.039716630242765 ], [ 647618.257487860973924, 4162037.393264054320753 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200610008021023", "area_land": 5630, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 683379.173801723285578, 4339718.274099923670292 ], [ 683388.636045723804273, 4339720.16153553687036 ], [ 683406.247825293801725, 4339226.619054754264653 ], [ 683392.94784223777242, 4339226.196451342664659 ], [ 683381.891119970707223, 4339620.723863031715155 ], [ 683379.173801723285578, 4339718.274099923670292 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "201474751002233", "area_land": 19988, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 489352.375147499202285, 4391401.554798025637865 ], [ 489353.078518448804971, 4391476.024222874082625 ], [ 489437.041688203113154, 4391475.020668931305408 ], [ 489452.736617584130727, 4391474.888162637129426 ], [ 489455.068263905821368, 4391423.943221963010728 ], [ 489469.648946393863298, 4391424.256213800981641 ], [ 489503.608059620310087, 4391421.102274054661393 ], [ 489502.105179723410401, 4391325.324975534342229 ], [ 489455.362240921007469, 4391325.499848640523851 ], [ 489448.500717744231224, 4391325.398266786709428 ], [ 489438.294428156630602, 4391325.41226374451071 ], [ 489387.434208348044194, 4391325.260247093625367 ], [ 489352.443056440912187, 4391326.640378216281533 ], [ 489352.375147499202285, 4391401.554798025637865 ] ] ] } }, +{ "type": "Feature", "properties": { "path": "200834611002251", "area_land": 2577172, "area_water": 0, "non_hispanic_other_vap": 0, "hispanic_vap": 0, "non_hispanic_amin_vap": 0, "non_hispanic_asian_vap":0, "non_hispanic_black_vap":0,"non_hispanic_nhpi_vap":0,"non_hispanic_white_vap":0, "total_vap": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 409731.251194308104459, 4200666.441016187891364 ], [ 409735.139420253282879, 4200955.250511697493494 ], [ 409736.066063234466128, 4201047.233642023056746 ], [ 409740.753175496880431, 4201504.041755408979952 ], [ 409810.145600938529242, 4201502.277895309962332 ], [ 409972.496923764934763, 4201500.4900694899261 ], [ 410110.95236269995803, 4201498.967932443134487 ], [ 410405.217876273440197, 4201492.078675809316337 ], [ 411342.044738269527443, 4201478.989577942527831 ], [ 411328.750386512838304, 4200568.193698559887707 ], [ 411324.006945162313059, 4200243.884643631987274 ], [ 411318.396692238282412, 4199863.879075475037098 ], [ 410818.479345699190162, 4199874.968814136460423 ], [ 410594.067309823876712, 4199878.082232806831598 ], [ 410399.997044122719672, 4199883.310992266982794 ], [ 410315.125831638462842, 4199885.238996434025466 ], [ 410086.93547103140736, 4199888.407714327797294 ], [ 409722.124784207146149, 4199894.641732443124056 ], [ 409731.251194308104459, 4200666.441016187891364 ] ] ] } } +] +} diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 530a1dd1b..170fa242b 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -11,7 +11,7 @@ OGR2OGR_PG_CONNECTION_STRING, FIXTURES_PATH, ) -from app.utils import create_districtr_map +from app.utils import create_districtr_map, add_available_summary_stats_to_districtrmap def test_read_main(client): @@ -30,6 +30,8 @@ def test_get_session(): GERRY_DB_FIXTURE_NAME = "ks_demo_view_census_blocks" GERRY_DB_TOTAL_VAP_FIXTURE_NAME = "ks_demo_view_census_blocks_total_vap" GERRY_DB_NO_POP_FIXTURE_NAME = "ks_demo_view_census_blocks_no_pop" +GERRY_DB_P1_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats" +GERRY_DB_P4_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats_p4" ## Test DB @@ -450,3 +452,180 @@ def test_list_gerydb_views_offset_and_limit(client, districtr_maps): assert response.status_code == 200 data = response.json() assert len(data) == 1 + assert data[0]["name"] == "Districtr map ks_demo_view_census_blocks" + + +@pytest.fixture(name=GERRY_DB_P1_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats(session: Session): + layer = GERRY_DB_P1_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.execute(upsert_query, {"name": layer}) + + (districtr_map_uuid,) = create_districtr_map( + session=session, + name="DistrictMap with P1 view", + parent_layer_name=layer, + gerrydb_table_name=layer, + ) + add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + + session.commit() + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="document_id_p1_summary_stats") +def document_summary_stats_fixture(client, ks_demo_view_census_blocks_summary_stats): + response = client.post( + "/api/create_document", + json={ + "gerrydb_table": GERRY_DB_P1_FIXTURE_NAME, + }, + ) + document_id = response.json()["document_id"] + return document_id + + +def test_get_p1_summary_stats(client, document_id_p1_summary_stats): + # Set up assignments + document_id = document_id_p1_summary_stats + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": 2}, + ] + }, + ) + + summary_stat = "P1" + response = client.get(f"/api/document/{document_id}/{summary_stat}") + data = response.json() + assert response.status_code == 200 + assert data.get("summary_stat") == "Population by Race" + results = data.get("results") + assert results is not None + assert len(results) == 2 + record_1, record_2 = data.get("results") + assert record_1.get("zone") == 1 + assert record_2.get("zone") == 2 + assert record_1.get("other_pop") == 13 + assert record_2.get("other_pop") == 24 + + +@pytest.fixture(name=GERRY_DB_P4_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats_p4(session: Session): + layer = GERRY_DB_P4_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.execute(upsert_query, {"name": layer}) + + (districtr_map_uuid,) = create_districtr_map( + session=session, + name="DistrictMap with P4 view", + parent_layer_name=layer, + gerrydb_table_name=layer, + ) + summary_stats = add_available_summary_stats_to_districtrmap( + session=session, districtr_map_uuid=districtr_map_uuid + ) + assert summary_stats == ["P4"], f"Expected P4 to be available, got {summary_stats}" + + session.commit() + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + +@pytest.fixture(name="document_id_p4_summary_stats") +def document_p4_summary_stats_fixture( + client, ks_demo_view_census_blocks_summary_stats_p4 +): + response = client.post( + "/api/create_document", + json={ + "gerrydb_table": GERRY_DB_P4_FIXTURE_NAME, + }, + ) + document_id = response.json()["document_id"] + return document_id + + +def test_get_p4_summary_stats(client, document_id_p4_summary_stats): + # Set up assignments + document_id = str(document_id_p4_summary_stats) + response = client.patch( + "/api/update_assignments", + json={ + "assignments": [ + {"document_id": document_id, "geo_id": "202090416004010", "zone": 1}, + {"document_id": document_id, "geo_id": "202090416003004", "zone": 1}, + {"document_id": document_id, "geo_id": "202090434001003", "zone": 2}, + ] + }, + ) + + summary_stat = "P4" + response = client.get(f"/api/document/{document_id}/{summary_stat}") + data = response.json() + assert response.status_code == 200 + assert ( + data.get("summary_stat") + == "Hispanic or Latino, and Not Hispanic or Latino by Race Voting Age Population" + ) + results = data.get("results") + assert results is not None + assert len(results) == 2 + record_1, record_2 = data.get("results") + assert record_1.get("zone") == 1 + assert record_2.get("zone") == 2 + assert record_1.get("hispanic_vap") == 13 + assert record_2.get("hispanic_vap") == 24 diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 883fcc631..7e03b154a 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -5,6 +5,7 @@ create_shatterable_gerrydb_view, create_parent_child_edges, add_extent_to_districtrmap, + get_available_summary_stats, ) from sqlmodel import Session import subprocess @@ -126,6 +127,42 @@ def districtr_map_fixture( return inserted_districtr_map +GERRY_DB_P1_FIXTURE_NAME = "ks_demo_view_census_blocks_summary_stats" + + +@pytest.fixture(name=GERRY_DB_P1_FIXTURE_NAME) +def ks_demo_view_census_blocks_summary_stats(session: Session): + layer = GERRY_DB_P1_FIXTURE_NAME + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + OGR2OGR_PG_CONNECTION_STRING, + os.path.join(FIXTURES_PATH, f"{layer}.geojson"), + "-lco", + "OVERWRITE=yes", + "-nln", + f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema + ], + ) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (gen_random_uuid(), :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + session.begin() + session.execute(upsert_query, {"name": GERRY_DB_P1_FIXTURE_NAME}) + + if result.returncode != 0: + print(f"ogr2ogr failed. Got {result}") + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + # FOR THE TESTS BELOW I NEED TO ADD ACTUAL ASSERTIONS @@ -240,6 +277,18 @@ def test_shattering(client, session: Session, document_id): assert all(d["zone"] == 1 for d in data["children"]) +def test_get_available_summary_stats( + session: Session, ks_demo_view_census_blocks_summary_stats +): + result = get_available_summary_stats(session, GERRY_DB_P1_FIXTURE_NAME) + assert len(result) == 1 + (summary_stats_available,) = result + assert summary_stats_available + assert len(summary_stats_available) == 1 + (summary_stat,) = summary_stats_available + assert summary_stat == "P1" + + def test_unshatter_process(client, document_id): response = client.patch( "/api/update_assignments",