From 2a39ed67d77ada7a9187fc8761a99359305291d7 Mon Sep 17 00:00:00 2001 From: Matthew Wigginton Conway Date: Thu, 1 Apr 2021 11:51:00 -0400 Subject: [PATCH 001/187] fix(HttpApi): make cors headers configurable fixes #717 and supersedes #683 and conveyal/analysis-ui#1457. --- analysis.properties.template | 10 +++++++ .../com/conveyal/analysis/BackendConfig.java | 3 ++ .../conveyal/analysis/components/HttpApi.java | 30 ++++++++++++------- .../PointToPointRouterServer.java | 20 ------------- 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/analysis.properties.template b/analysis.properties.template index a98f52ebe..9b1b5d3bd 100644 --- a/analysis.properties.template +++ b/analysis.properties.template @@ -31,6 +31,16 @@ aws-region=eu-west-1 # The port on which the server will listen for connections from clients and workers. server-port=7070 +# The origin where the frontend is hosted. Due to the same-origin policy, cross-origin requests are generally blocked. +# However, setting this to an origin will add an Access-Control-Allow-Origin header to allow cross-origin requests from +# that origin. For instance, when running locally, this will generally be http://localhost:3000. It is possible to set +# this to *, but that allows requests from anywhere. While this should be relatively safe when authentication is enabled +# since authentication is not handled through cookies, it does increase the attack surface and better practice is to set +# this to the actual origin where the UI is hosted. This is different from frontend-url, since frontend-url is just where +# the frontend code can be retrieved from, and may not even be a valid origin (e.g. if it has a path name). +# Commenting this property out will result in no Access-Control-Allow-Origin header being sent. +access-control-allow-origin=http://localhost:3000 + # A temporary location to store scratch files. The path can be absolute or relative. # This allows you to locate temporary storage on an extra drive in case your main drive does not have enough space. # local-cache=/home/ec2-user/cache diff --git a/src/main/java/com/conveyal/analysis/BackendConfig.java b/src/main/java/com/conveyal/analysis/BackendConfig.java index 649f20781..f24a5326b 100644 --- a/src/main/java/com/conveyal/analysis/BackendConfig.java +++ b/src/main/java/com/conveyal/analysis/BackendConfig.java @@ -55,6 +55,7 @@ public class BackendConfig implements private final String localCacheDirectory; private final int serverPort; private final boolean offline; + private final String accessControlAllowOrigin; private final String seamlessCensusBucket; private final String seamlessCensusRegion; private final String gridBucket; @@ -109,6 +110,7 @@ public BackendConfig (String filename) { localCacheDirectory = getProperty("local-cache", true); serverPort = Integer.parseInt(getProperty("server-port", true)); offline = Boolean.parseBoolean(getProperty("offline", true)); + accessControlAllowOrigin = getProperty("access-control-allow-origin", false); seamlessCensusBucket = getProperty("seamless-census-bucket", true); seamlessCensusRegion = getProperty("seamless-census-region", true); gridBucket = getProperty("grid-bucket", true); @@ -171,6 +173,7 @@ private String getProperty (String key, boolean require) { @Override public String localCacheDirectory () { return localCacheDirectory;} @Override public String bundleBucket () { return bundleBucket; } @Override public boolean offline () { return offline; } + @Override public String accessControlAllowOrigin () { return accessControlAllowOrigin; } @Override public int maxWorkers () { return maxWorkers; } } diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index bf8852337..0343a1742 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -43,6 +43,7 @@ public class HttpApi implements Component { public interface Config { boolean offline (); int serverPort (); + String accessControlAllowOrigin (); } private final FileStorage fileStorage; @@ -87,8 +88,14 @@ private spark.Service configureSparkService () { // FIXME those internal endpoints should be hidden from the outside world by the reverse proxy. // Or now with non-static Spark we can run two HTTP servers on different ports. - // Set CORS headers, to allow requests to this API server from any page. - res.header("Access-Control-Allow-Origin", "*"); + // Set CORS headers, to allow requests to this API server from a frontend hosted on a different domain + // This used to be hardwired to Access-Control-Allow-Origin: * but that leaves the server open to XSRF + // attacks when authentication is disabled (e.g. when running locally). + if (config.accessControlAllowOrigin() != null) { + res.header("Access-Control-Allow-Origin", config.accessControlAllowOrigin()); + // for caching, signal to the browser that responses may be different based on origin + res.header("Vary", "Origin"); + } // The default MIME type is JSON. This will be overridden by the few controllers that do not return JSON. res.type("application/json"); @@ -120,14 +127,17 @@ private spark.Service configureSparkService () { }); // Handle CORS preflight requests (which are OPTIONS requests). - sparkService.options("/*", (req, res) -> { - res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"); - res.header("Access-Control-Allow-Credentials", "true"); - res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," + - "X-Requested-With,Content-Length,X-Conveyal-Access-Group" - ); - return "OK"; - }); + // See comment above about Access-Control-Allow-Origin + if (config.accessControlAllowOrigin() != null) { + sparkService.options("/*", (req, res) -> { + res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"); + res.header("Access-Control-Allow-Credentials", "true"); + res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," + + "X-Requested-With,Content-Length,X-Conveyal-Access-Group" + ); + return "OK"; + }); + } // Allow client to fetch information about the backend build version. sparkService.get( diff --git a/src/main/java/com/conveyal/r5/point_to_point/PointToPointRouterServer.java b/src/main/java/com/conveyal/r5/point_to_point/PointToPointRouterServer.java index 6dc06c770..4393999e3 100644 --- a/src/main/java/com/conveyal/r5/point_to_point/PointToPointRouterServer.java +++ b/src/main/java/com/conveyal/r5/point_to_point/PointToPointRouterServer.java @@ -154,26 +154,6 @@ private static void run(TransportNetwork transportNetwork) { PointToPointQuery pointToPointQuery = new PointToPointQuery(transportNetwork); ParetoServer paretoServer = new ParetoServer(transportNetwork); - // add cors header - before((req, res) -> res.header("Access-Control-Allow-Origin", "*")); - - options("/*", (request, response) -> { - - String accessControlRequestHeaders = request - .headers("Access-Control-Request-Headers"); - if (accessControlRequestHeaders != null) { - response.header("Access-Control-Allow-Headers", accessControlRequestHeaders); - } - - String accessControlRequestMethod = request - .headers("Access-Control-Request-Method"); - if (accessControlRequestMethod != null) { - response.header("Access-Control-Allow-Methods", accessControlRequestMethod); - } - - return "OK"; - }); - get("/metadata", (request, response) -> { response.header("Content-Type", "application/json"); RouterInfo routerInfo = new RouterInfo(); From a5637400c7aa9ff7c9221040f3eda31320c86b04 Mon Sep 17 00:00:00 2001 From: Matthew Wigginton Conway Date: Fri, 2 Apr 2021 13:10:08 -0400 Subject: [PATCH 002/187] fix(HttpApi): require cors headers --- analysis.properties.template | 1 - .../com/conveyal/analysis/BackendConfig.java | 2 +- .../conveyal/analysis/components/HttpApi.java | 26 ++++++++----------- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/analysis.properties.template b/analysis.properties.template index 9b1b5d3bd..e1c4bb200 100644 --- a/analysis.properties.template +++ b/analysis.properties.template @@ -38,7 +38,6 @@ server-port=7070 # since authentication is not handled through cookies, it does increase the attack surface and better practice is to set # this to the actual origin where the UI is hosted. This is different from frontend-url, since frontend-url is just where # the frontend code can be retrieved from, and may not even be a valid origin (e.g. if it has a path name). -# Commenting this property out will result in no Access-Control-Allow-Origin header being sent. access-control-allow-origin=http://localhost:3000 # A temporary location to store scratch files. The path can be absolute or relative. diff --git a/src/main/java/com/conveyal/analysis/BackendConfig.java b/src/main/java/com/conveyal/analysis/BackendConfig.java index f24a5326b..125690cf4 100644 --- a/src/main/java/com/conveyal/analysis/BackendConfig.java +++ b/src/main/java/com/conveyal/analysis/BackendConfig.java @@ -110,7 +110,7 @@ public BackendConfig (String filename) { localCacheDirectory = getProperty("local-cache", true); serverPort = Integer.parseInt(getProperty("server-port", true)); offline = Boolean.parseBoolean(getProperty("offline", true)); - accessControlAllowOrigin = getProperty("access-control-allow-origin", false); + accessControlAllowOrigin = getProperty("access-control-allow-origin", true); seamlessCensusBucket = getProperty("seamless-census-bucket", true); seamlessCensusRegion = getProperty("seamless-census-region", true); gridBucket = getProperty("grid-bucket", true); diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 0343a1742..bd67b5c81 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -91,11 +91,9 @@ private spark.Service configureSparkService () { // Set CORS headers, to allow requests to this API server from a frontend hosted on a different domain // This used to be hardwired to Access-Control-Allow-Origin: * but that leaves the server open to XSRF // attacks when authentication is disabled (e.g. when running locally). - if (config.accessControlAllowOrigin() != null) { - res.header("Access-Control-Allow-Origin", config.accessControlAllowOrigin()); - // for caching, signal to the browser that responses may be different based on origin - res.header("Vary", "Origin"); - } + res.header("Access-Control-Allow-Origin", config.accessControlAllowOrigin()); + // for caching, signal to the browser that responses may be different based on origin + res.header("Vary", "Origin"); // The default MIME type is JSON. This will be overridden by the few controllers that do not return JSON. res.type("application/json"); @@ -128,16 +126,14 @@ private spark.Service configureSparkService () { // Handle CORS preflight requests (which are OPTIONS requests). // See comment above about Access-Control-Allow-Origin - if (config.accessControlAllowOrigin() != null) { - sparkService.options("/*", (req, res) -> { - res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"); - res.header("Access-Control-Allow-Credentials", "true"); - res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," + - "X-Requested-With,Content-Length,X-Conveyal-Access-Group" - ); - return "OK"; - }); - } + sparkService.options("/*", (req, res) -> { + res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"); + res.header("Access-Control-Allow-Credentials", "true"); + res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," + + "X-Requested-With,Content-Length,X-Conveyal-Access-Group" + ); + return "OK"; + }); // Allow client to fetch information about the backend build version. sparkService.get( From be68afe2112d231ae74ccda899e15b61c0b9e0e3 Mon Sep 17 00:00:00 2001 From: Matthew Wigginton Conway Date: Fri, 2 Apr 2021 13:14:43 -0400 Subject: [PATCH 003/187] fix(HttpApi): reinstate null check to make sure that Access-Control-Allow-Origin: null can never be sent --- src/main/java/com/conveyal/analysis/components/HttpApi.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index bd67b5c81..83bd9156f 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -91,6 +91,12 @@ private spark.Service configureSparkService () { // Set CORS headers, to allow requests to this API server from a frontend hosted on a different domain // This used to be hardwired to Access-Control-Allow-Origin: * but that leaves the server open to XSRF // attacks when authentication is disabled (e.g. when running locally). + if (config.accessControlAllowOrigin() == null || config.accessControlAllowOrigin().equals("null")) { + // Access-Control-Allow-Origin: null opens unintended security holes: + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin + throw new IllegalArgumentException("Access-Control-Allow-Origin should not be null"); + } + res.header("Access-Control-Allow-Origin", config.accessControlAllowOrigin()); // for caching, signal to the browser that responses may be different based on origin res.header("Vary", "Origin"); From 0b69847226e8f74f8e906c5832435ef01032dea2 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 24 May 2021 21:02:45 +0800 Subject: [PATCH 004/187] Add variant index check (#728) Users are seeing index out of bounds exceptions if they request a deleted scenario. This add's a check on the backend to surface a clearer error when this occurs. A corresponding front end change set will attempt to address the issue on the client side also. --- .../java/com/conveyal/analysis/models/AnalysisRequest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index b118a793a..af87aa78c 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -186,6 +186,9 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project List modifications = new ArrayList<>(); String scenarioName; if (variantIndex > -1) { + if (variantIndex >= project.variants.length) { + throw AnalysisServerException.badRequest("Scenario does not exist. Please select a new scenario."); + } modifications = modificationsForProject(project.accessGroup, projectId, variantIndex); scenarioName = project.variants[variantIndex]; } else { From 594172958772080ca07cc7af42650b54ef695f2e Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 24 May 2021 20:31:23 -0400 Subject: [PATCH 005/187] refactor(grid): store reference to a WebMercatorExtents instead of inlining the fields. As suggested in an existing TODO. --- .../analysis/models/AnalysisRequest.java | 23 +-- .../BootstrappingTravelTimeReducer.java | 4 +- .../java/com/conveyal/r5/analyst/Grid.java | 136 ++++++++---------- .../r5/analyst/GridTransformWrapper.java | 10 +- .../r5/analyst/WebMercatorExtents.java | 12 ++ .../com/conveyal/r5/analyst/GridTest.java | 11 +- .../network/GridSinglePointTaskBuilder.java | 10 +- 7 files changed, 96 insertions(+), 110 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index af87aa78c..8749e2e21 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -3,6 +3,7 @@ import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.r5.analyst.Grid; +import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; import com.conveyal.r5.analyst.decay.DecayFunction; import com.conveyal.r5.analyst.decay.StepDecayFunction; @@ -227,12 +228,12 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project // TODO define class with static factory function WebMercatorGridBounds.fromLatLonBounds(). // Also include getIndex(x, y), getX(index), getY(index), totalTasks() - Grid grid = new Grid(zoom, bounds.envelope()); - checkZoom(grid); - task.height = grid.height; - task.north = grid.north; - task.west = grid.west; - task.width = grid.width; + WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(bounds.envelope(), zoom); + checkZoom(extents); + task.height = extents.height; + task.north = extents.north; + task.west = extents.west; + task.width = extents.width; task.zoom = zoom; task.date = date; @@ -286,17 +287,17 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project return task; } - private static void checkZoom(Grid grid) { - if (grid.zoom < MIN_ZOOM || grid.zoom > MAX_ZOOM) { + private static void checkZoom(WebMercatorExtents extents) { + if (extents.zoom < MIN_ZOOM || extents.zoom > MAX_ZOOM) { throw AnalysisServerException.badRequest(String.format( - "Requested zoom (%s) is outside valid range (%s - %s)", grid.zoom, MIN_ZOOM, MAX_ZOOM + "Requested zoom (%s) is outside valid range (%s - %s)", extents.zoom, MIN_ZOOM, MAX_ZOOM )); } - if (grid.height * grid.width > MAX_GRID_CELLS) { + if (extents.height * extents.width > MAX_GRID_CELLS) { throw AnalysisServerException.badRequest(String.format( "Requested number of destinations (%s) exceeds limit (%s). " + "Set smaller custom geographic bounds or a lower zoom level.", - grid.height * grid.width, MAX_GRID_CELLS + extents.height * extents.width, MAX_GRID_CELLS )); } } diff --git a/src/main/java/com/conveyal/r5/analyst/BootstrappingTravelTimeReducer.java b/src/main/java/com/conveyal/r5/analyst/BootstrappingTravelTimeReducer.java index b8a8db924..058644a6e 100644 --- a/src/main/java/com/conveyal/r5/analyst/BootstrappingTravelTimeReducer.java +++ b/src/main/java/com/conveyal/r5/analyst/BootstrappingTravelTimeReducer.java @@ -65,8 +65,8 @@ public void recordTravelTimesForTarget(int target, int[] travelTimesForTarget) { // We use the size of the grid to determine the number of destinations used in the linked point set in // TravelTimeComputer, therefore the target indices are relative to the grid, not the task. // TODO verify that the above is still accurate - int gridx = target % grid.width; - int gridy = target / grid.width; + int gridx = target % grid.extents.width; + int gridy = target / grid.extents.width; double opportunityCountAtTarget = grid.grid[gridx][gridy]; // As an optimization, don't even bother to check whether cells that contain no opportunities are reachable. diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index f741ca692..f91ac0820 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -85,25 +85,7 @@ public class Grid extends PointSet { public static final String COUNT_COLUMN_NAME = "[COUNT]"; - /** The web mercator zoom level for this grid. */ - public final int zoom; - - /* The following fields establish the position of this sub-grid within the full worldwide web mercator grid. */ - - /** - * The pixel number of the northernmost pixel in this grid (smallest y value in web Mercator, - * because y increases from north to south in web Mercator). - */ - public final int north; - - /** The pixel number of the westernmost pixel in this grid (smallest x value). */ - public final int west; - - /** The width of the grid in web Mercator pixels. */ - public final int width; - - /** The height of the grid in web Mercator pixels. */ - public final int height; + public final WebMercatorExtents extents; /** * The data values for each pixel within this grid. Dimension order is (x, y), with range [0, width) and [0, height). @@ -116,20 +98,15 @@ public class Grid extends PointSet { /** Maximum area allowed for features in a shapefile upload */ private static final double MAX_FEATURE_AREA_SQ_DEG = 2; - /** - * Used when reading a saved grid. - */ + /** Used when reading a saved grid. */ public Grid (int zoom, int width, int height, int north, int west) { - this.zoom = zoom; - this.width = width; - this.height = height; - this.north = north; - this.west = west; + this.extents = new WebMercatorExtents(west, north, width, height, zoom); this.grid = new double[width][height]; } public Grid (WebMercatorExtents extents) { - this(extents.zoom, extents.width, extents.height, extents.north, extents.west); + this.extents = extents; + this.grid = new double[extents.width][extents.height]; } /** @@ -137,14 +114,9 @@ public Grid (WebMercatorExtents extents) { * @param wgsEnvelope Envelope of grid, in absolute WGS84 lat/lon coordinates */ public Grid (int zoom, Envelope wgsEnvelope) { - WebMercatorExtents webMercatorExtents = WebMercatorExtents.forWgsEnvelope(wgsEnvelope, zoom); - // TODO actually store a reference to an immutable WebMercatorExtents instead of inlining the fields in Grid. - this.zoom = webMercatorExtents.zoom; - this.west = webMercatorExtents.west; - this.north = webMercatorExtents.north; - this.width = webMercatorExtents.width; - this.height = webMercatorExtents.height; - this.grid = new double[width][height]; + WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(wgsEnvelope, zoom); + this.extents = extents; + this.grid = new double[extents.width][extents.height]; } public static class PixelWeight { @@ -205,21 +177,21 @@ public List getPixelWeights (Geometry geometry, boolean relativeToP Envelope env = geometry.getEnvelopeInternal(); - for (int worldy = latToPixel(env.getMaxY(), zoom); worldy <= latToPixel(env.getMinY(), zoom); worldy++) { + for (int worldy = latToPixel(env.getMaxY(), extents.zoom); worldy <= latToPixel(env.getMinY(), extents.zoom); worldy++) { // NB web mercator Y is reversed relative to latitude. // Iterate over longitude (x) in the inner loop to avoid repeat calculations of pixel areas, which should be // equal at a given latitude (y) double pixelAreaAtLat = -1; //Set to -1 to recalculate pixelArea at each latitude. - for (int worldx = lonToPixel(env.getMinX(), zoom); worldx <= lonToPixel(env.getMaxX(), zoom); worldx++) { + for (int worldx = lonToPixel(env.getMinX(), extents.zoom); worldx <= lonToPixel(env.getMaxX(), extents.zoom); worldx++) { - int x = worldx - west; - int y = worldy - north; + int x = worldx - extents.west; + int y = worldy - extents.north; - if (x < 0 || x >= width || y < 0 || y >= height) continue; // off the grid + if (x < 0 || x >= extents.width || y < 0 || y >= extents.height) continue; // off the grid - Geometry pixel = getPixelGeometry(x + west, y + north, zoom); + Geometry pixel = getPixelGeometry(x , y , extents); if (pixelAreaAtLat == -1) pixelAreaAtLat = pixel.getArea(); //Recalculate for a new latitude. // Pixel completely within feature: @@ -268,11 +240,11 @@ public void incrementFromPixelWeights (List weights, double value) * Burn point data into the grid. */ private void incrementPoint (double lat, double lon, double amount) { - int worldx = lonToPixel(lon, zoom); - int worldy = latToPixel(lat, zoom); - int x = worldx - west; - int y = worldy - north; - if (x >= 0 && x < width && y >= 0 && y < height) { + int worldx = lonToPixel(lon, extents.zoom); + int worldy = latToPixel(lat, extents.zoom); + int x = worldx - extents.west; + int y = worldy - extents.north; + if (x >= 0 && x < extents.width && y >= 0 && y < extents.height) { grid[x][y] += amount; } else { LOG.warn("{} opportunities are outside regional bounds, at {}, {}", amount, lon, lat); @@ -306,19 +278,19 @@ public void write (OutputStream outputStream) throws IOException { // On almost all current hardware this is little-endian. Guava saves us again. LittleEndianDataOutputStream out = new LittleEndianDataOutputStream(outputStream); // A header consisting of six 4-byte integers specifying the zoom level and bounds. - out.writeInt(zoom); - out.writeInt(west); - out.writeInt(north); - out.writeInt(width); - out.writeInt(height); + out.writeInt(extents.zoom); + out.writeInt(extents.west); + out.writeInt(extents.north); + out.writeInt(extents.width); + out.writeInt(extents.height); // The rest of the file is 32-bit integers in row-major order (x changes faster than y), delta-coded. // Delta coding and error diffusion are reset on each row to avoid wrapping. int prev = 0; - for (int y = 0; y < height; y++) { + for (int y = 0; y < extents.height; y++) { // Reset error on each row to avoid diffusing to distant locations. // An alternative is to use serpentine iteration or iterative diffusion. double error = 0; - for (int x = 0; x < width; x++) { + for (int x = 0; x < extents.width; x++) { double val = grid[x][y]; checkState(val >= 0, "Opportunity density should never be negative."); val += error; @@ -336,9 +308,9 @@ public void write (OutputStream outputStream) throws IOException { /** Write this grid out in GeoTIFF format */ public void writeGeotiff (OutputStream out) { try { - float[][] data = new float[height][width]; - for (int x = 0; x < width; x++) { - for (int y = 0; y < height; y++) { + float[][] data = new float[extents.height][extents.width]; + for (int x = 0; x < extents.width; x++) { + for (int y = 0; y < extents.height; y++) { data[y][x] = (float) grid[x][y]; } } @@ -395,11 +367,11 @@ public void writePng(OutputStream outputStream) throws IOException { } } - BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY); + BufferedImage img = new BufferedImage(extents.width, extents.height, BufferedImage.TYPE_BYTE_GRAY); byte[] imgPixels = ((DataBufferByte) img.getRaster().getDataBuffer()).getData(); int p = 0; - for (int y = 0; y < height; y++) { - for (int x = 0; x < width; x++) { + for (int y = 0; y < extents.height; y++) { + for (int x = 0; x < extents.width; x++) { double density = grid[x][y]; imgPixels[p++] = (byte)(density * 255 / maxPixel); } @@ -422,13 +394,13 @@ public void writeShapefile (String fileName, String fieldName) { store.createSchema(gridCell); Transaction transaction = new DefaultTransaction("Save Grid"); FeatureWriter writer = store.getFeatureWriterAppend(transaction); - for (int x = 0; x < width; x++) { - for (int y = 0; y < height; y++) { + for (int x = 0; x < extents.width; x++) { + for (int y = 0; y < extents.height; y++) { try { double value = grid[x][y]; if (value > 0) { SimpleFeature feature = (SimpleFeature) writer.next(); - Polygon pixelPolygon = getPixelGeometry(x + west, y + north, zoom); + Polygon pixelPolygon = getPixelGeometry(x, y, extents); feature.setDefaultGeometry(pixelPolygon); feature.setAttribute(fieldName, value); writer.write(); @@ -447,7 +419,7 @@ public void writeShapefile (String fileName, String fieldName) { } public boolean hasEqualExtents(Grid comparisonGrid){ - return this.zoom == comparisonGrid.zoom && this.west == comparisonGrid.west && this.north == comparisonGrid.north && this.width == comparisonGrid.width && this.height == comparisonGrid.height; + return this.extents.equals(comparisonGrid.extents); } /** @@ -460,8 +432,8 @@ public boolean hasEqualExtents(Grid comparisonGrid){ */ public double getLat(int i) { // Integer division of linear index to find vertical integer intra-grid pixel coordinate - int y = i / width; - return pixelToCenterLat(north + y, zoom); + int y = i / extents.width; + return pixelToCenterLat(extents.north + y, extents.zoom); } /** @@ -472,12 +444,12 @@ public double getLat(int i) { */ public double getLon(int i) { // Remainder of division yields horizontal integer intra-grid pixel coordinate - int x = i % width; - return pixelToCenterLon(west + x, zoom); + int x = i % extents.width; + return pixelToCenterLon(extents.west + x, extents.zoom); } public int featureCount() { - return width * height; + return extents.width * extents.height; } /* functions below from http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Mathematics */ @@ -530,16 +502,20 @@ public static double pixelToLat (double yPixel, int zoom) { } /** - * @param x absolute (world) x pixel number at the given zoom level. - * @param y absolute (world) y pixel number at the given zoom level. - * @return a JTS Polygon in WGS84 coordinates for the given absolute (world) pixel. + * Given a pixel's local coordinates in the supplied WebMercatorExtents, return its geometry in absolute (world) + * coordinates + * @param localX x pixel number within the given extents. + * @param localY y pixel number within the given extents. + * @return a JTS Polygon in WGS84 coordinates for the given pixel. */ - public static Polygon getPixelGeometry (int x, int y, int zoom) { - double minLon = pixelToLon(x, zoom); - double maxLon = pixelToLon(x + 1, zoom); + public static Polygon getPixelGeometry (int localX, int localY, WebMercatorExtents extents) { + int x = localX + extents.west; + int y = localY + extents.north; + double minLon = pixelToLon(x, extents.zoom); + double maxLon = pixelToLon(x + 1, extents.zoom); // The y axis increases from north to south in web Mercator. - double minLat = pixelToLat(y + 1, zoom); - double maxLat = pixelToLat(y, zoom); + double minLat = pixelToLat(y + 1, extents.zoom); + double maxLat = pixelToLat(y, extents.zoom); return GeometryUtils.geometryFactory.createPolygon(new Coordinate[] { new Coordinate(minLon, minLat), new Coordinate(minLon, maxLat), @@ -763,8 +739,8 @@ public double sumTotalOpportunities() { @Override public double getOpportunityCount (int i) { - int x = i % this.width; - int y = i / this.width; + int x = i % extents.width; + int y = i / extents.width; return grid[x][y]; } @@ -793,7 +769,7 @@ public Envelope getWgsEnvelope () { @Override public WebMercatorExtents getWebMercatorExtents () { - return new WebMercatorExtents(this.west, this.north, this.width, this.height, this.zoom); + return extents; } /** diff --git a/src/main/java/com/conveyal/r5/analyst/GridTransformWrapper.java b/src/main/java/com/conveyal/r5/analyst/GridTransformWrapper.java index 8c18549cf..ddc5fcec5 100644 --- a/src/main/java/com/conveyal/r5/analyst/GridTransformWrapper.java +++ b/src/main/java/com/conveyal/r5/analyst/GridTransformWrapper.java @@ -26,7 +26,7 @@ public class GridTransformWrapper extends PointSet { * targetGrid cannot be indexed so are effectively zero for the purpose of accessibility calculations. */ public GridTransformWrapper (WebMercatorExtents targetGridExtents, Grid sourceGrid) { - checkArgument(targetGridExtents.zoom == sourceGrid.zoom, "Zoom levels must be identical."); + checkArgument(targetGridExtents.zoom == sourceGrid.extents.zoom, "Zoom levels must be identical."); // Make a pointset for these extents so we can defer to its methods for lat/lon lookup, size, etc. this.targetGrid = new WebMercatorGridPointSet(targetGridExtents); this.sourceGrid = sourceGrid; @@ -37,13 +37,13 @@ public GridTransformWrapper (WebMercatorExtents targetGridExtents, Grid sourceGr // This could certainly be made more efficient (but complex) by forcing sequential iteration over opportunity counts // and disallowing random access, using a new PointSetIterator class that allows reading lat, lon, and counts. private int transformIndex (int i) { - final int x = (i % targetGrid.width) + targetGrid.west - sourceGrid.west; - final int y = (i / targetGrid.width) + targetGrid.north - sourceGrid.north; - if (x < 0 || x >= sourceGrid.width || y < 0 || y >= sourceGrid.height) { + final int x = (i % targetGrid.width) + targetGrid.west - sourceGrid.extents.west; + final int y = (i / targetGrid.width) + targetGrid.north - sourceGrid.extents.north; + if (x < 0 || x >= sourceGrid.extents.width || y < 0 || y >= sourceGrid.extents.height) { // Point in target grid lies outside source grid, there is no valid index. Return special value. return -1; } - return y * sourceGrid.width + x; + return y * sourceGrid.extents.width + x; } @Override diff --git a/src/main/java/com/conveyal/r5/analyst/WebMercatorExtents.java b/src/main/java/com/conveyal/r5/analyst/WebMercatorExtents.java index 79e8f6f5a..150043b91 100644 --- a/src/main/java/com/conveyal/r5/analyst/WebMercatorExtents.java +++ b/src/main/java/com/conveyal/r5/analyst/WebMercatorExtents.java @@ -26,10 +26,22 @@ */ public class WebMercatorExtents { + /** The pixel number of the westernmost pixel (smallest x value). */ public final int west; + + /** + * The pixel number of the northernmost pixel (smallest y value in web Mercator, because y increases from north to + * south in web Mercator). + */ public final int north; + + /** Width in web Mercator pixels */ public final int width; + + /** Height in web Mercator pixels */ public final int height; + + /** Web mercator zoom level. */ public final int zoom; public WebMercatorExtents (int west, int north, int width, int height, int zoom) { diff --git a/src/test/java/com/conveyal/r5/analyst/GridTest.java b/src/test/java/com/conveyal/r5/analyst/GridTest.java index 27b3afc7d..9a6d051e5 100644 --- a/src/test/java/com/conveyal/r5/analyst/GridTest.java +++ b/src/test/java/com/conveyal/r5/analyst/GridTest.java @@ -13,6 +13,7 @@ import java.util.stream.DoubleStream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests for the Grid class, which holds destination counts in tiled spherical mercator pixels. @@ -107,8 +108,8 @@ private static Grid generateRandomGrid (Random random, boolean wholeNumbersOnly) int height = random.nextInt(MAX_GRID_WIDTH_PIXELS) + 1; Grid grid = new Grid(zoom, width, height, north, west); - for (int y = 0; y < grid.height; y++) { - for (int x = 0; x < grid.width; x++) { + for (int y = 0; y < grid.extents.height; y++) { + for (int x = 0; x < grid.extents.width; x++) { double amount = random.nextDouble() * MAX_AMOUNT; if (wholeNumbersOnly) { amount = Math.round(amount); @@ -121,11 +122,7 @@ private static Grid generateRandomGrid (Random random, boolean wholeNumbersOnly) private static void assertGridSemanticEquals(Grid g1, Grid g2, boolean tolerateRounding) { // Note that the name field is excluded because it does not survive serialization. - assertEquals(g1.zoom, g2.zoom); - assertEquals(g1.north, g2.north); - assertEquals(g1.west, g2.west); - assertEquals(g1.width, g2.width); - assertEquals(g1.height, g2.height); + assertTrue(g1.hasEqualExtents(g2)); assertArrayEquals(g1.grid, g2.grid, tolerateRounding); } diff --git a/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java b/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java index 97dcba40b..ced99afa3 100644 --- a/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java +++ b/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java @@ -102,11 +102,11 @@ public GridSinglePointTaskBuilder uniformOpportunityDensity (double density) { // In a single point task, the grid of destinations is given with these fields, not from the pointset object. // The destination point set (containing the opportunity densities) must then match these same dimensions. - task.zoom = grid.zoom; - task.north = grid.north; - task.west = grid.west; - task.width = grid.width; - task.height = grid.height; + task.zoom = grid.extents.zoom; + task.north = grid.extents.north; + task.west = grid.extents.west; + task.width = grid.extents.width; + task.height = grid.extents.height; return this; } From 993d36caa3cee2c65c5e4d4ae128c17addd27f21 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 25 May 2021 21:55:21 -0400 Subject: [PATCH 006/187] refactor(grid): consolidate constructors --- src/main/java/com/conveyal/r5/analyst/Grid.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index f91ac0820..c3aac1a2e 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -114,9 +114,7 @@ public Grid (WebMercatorExtents extents) { * @param wgsEnvelope Envelope of grid, in absolute WGS84 lat/lon coordinates */ public Grid (int zoom, Envelope wgsEnvelope) { - WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(wgsEnvelope, zoom); - this.extents = extents; - this.grid = new double[extents.width][extents.height]; + this(WebMercatorExtents.forWgsEnvelope(wgsEnvelope, zoom)); } public static class PixelWeight { From 52340684d3e8c949ea9f1a138961b572e310c240 Mon Sep 17 00:00:00 2001 From: ansons Date: Wed, 26 May 2021 20:22:35 -0400 Subject: [PATCH 007/187] fix(grid): check for duplicate attributes and excessive size Addresses #727 (but revised progress/error reporting still pending) --- .../java/com/conveyal/r5/analyst/Grid.java | 42 +++++++++++++------ .../com/conveyal/r5/util/ShapefileReader.java | 17 ++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index c3aac1a2e..5f05380fc 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -34,6 +34,7 @@ import org.opengis.feature.Property; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.feature.type.AttributeDescriptor; import org.opengis.parameter.GeneralParameterValue; import org.opengis.parameter.ParameterValueGroup; import org.opengis.referencing.FactoryException; @@ -98,10 +99,13 @@ public class Grid extends PointSet { /** Maximum area allowed for features in a shapefile upload */ private static final double MAX_FEATURE_AREA_SQ_DEG = 2; + /** Limit on number of pixels, to prevent OOME when multiple large grids are being created (width * height * + * number of layers/attributes) */ + private static final int MAX_PIXELS = 100_000_000 * 10; + /** Used when reading a saved grid. */ public Grid (int zoom, int width, int height, int north, int west) { - this.extents = new WebMercatorExtents(west, north, width, height, zoom); - this.grid = new double[width][height]; + this(new WebMercatorExtents(west, north, width, height, zoom)); } public Grid (WebMercatorExtents extents) { @@ -599,6 +603,8 @@ public static List fromCsv(InputStreamProvider csvInputStreamProvider, throw new IllegalArgumentException("CSV file contained no entirely finite, non-negative numeric columns."); } checkWgsEnvelopeSize(envelope); + WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(envelope, zoom); + checkPixelCount(extents, numericColumns.size()); if (progressListener != null) { progressListener.setTotalItems(total); @@ -607,14 +613,14 @@ public static List fromCsv(InputStreamProvider csvInputStreamProvider, // We now have an envelope and know which columns are numeric. Make a grid for each numeric column. Map grids = new HashMap<>(); for (String columnName : numericColumns) { - Grid grid = new Grid(zoom, envelope); + Grid grid = new Grid(extents); grid.name = columnName; grids.put(grid.name, grid); } // Make one more Grid where every point will have a weight of 1, for counting points rather than opportunities. // This assumes there is no column called "[COUNT]" in the source file, which is validated above. - Grid countGrid = new Grid(zoom, envelope); + Grid countGrid = new Grid(extents); countGrid.name = COUNT_COLUMN_NAME; grids.put(countGrid.name, countGrid); @@ -666,20 +672,24 @@ public static List fromShapefile (File shapefile, int zoom) throws IOExcep public static List fromShapefile (File shapefile, int zoom, ProgressListener progressListener) throws IOException, FactoryException, TransformException { - Map grids = new HashMap<>(); ShapefileReader reader = new ShapefileReader(shapefile); - - Envelope envelope = reader.wgs84Bounds(); - int total = reader.getFeatureCount(); - checkWgsEnvelopeSize(envelope); + WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(envelope, zoom); + List numericAttributes = reader.numericAttributes(); + Set uniqueNumericAttributes = new HashSet<>(numericAttributes); + if (uniqueNumericAttributes.size() != numericAttributes.size()) { + throw new IllegalArgumentException("Shapefile has duplicate numeric attributes"); + } + checkPixelCount(extents, numericAttributes.size()); + int total = reader.getFeatureCount(); if (progressListener != null) { progressListener.setTotalItems(total); } AtomicInteger count = new AtomicInteger(0); + Map grids = new HashMap<>(); reader.wgs84Stream().forEach(feat -> { Geometry geom = (Geometry) feat.getDefaultGeometry(); @@ -692,11 +702,10 @@ public static List fromShapefile (File shapefile, int zoom, ProgressListen if (numericVal == 0) continue; String attributeName = p.getName().getLocalPart(); - - // TODO this is assuming that each attribute name can only exist once. Shapefiles can contain duplicate attribute names. Validate to catch this. + Grid grid = grids.get(attributeName); if (grid == null) { - grid = new Grid(zoom, envelope); + grid = new Grid(extents); grid.name = attributeName; grids.put(attributeName, grid); } @@ -799,4 +808,13 @@ public static void checkWgsEnvelopeSize (Envelope envelope) { } } + public static void checkPixelCount (WebMercatorExtents extents, int layers) { + int pixels = extents.width * extents.height * layers; + if (pixels > MAX_PIXELS) { + throw new IllegalArgumentException("Number of zoom level " + extents.zoom + " pixels (" + pixels + ")" + + "exceeds limit (" + MAX_PIXELS +"). Reduce the zoom level or the file's extents or number of " + + "numeric attributes."); + } + } + } diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index fd7540aa0..fe0171a26 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -14,6 +14,8 @@ import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.feature.type.AttributeDescriptor; +import org.opengis.feature.type.AttributeType; import org.opengis.filter.Filter; import org.opengis.referencing.FactoryException; import org.opengis.referencing.crs.CoordinateReferenceSystem; @@ -22,11 +24,14 @@ import java.io.File; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Spliterator; import java.util.Spliterators; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -89,6 +94,18 @@ public ReferencedEnvelope getBounds () throws IOException { return source.getBounds(); } + public List numericAttributes () { + return features.getSchema() + .getAttributeDescriptors() + .stream() + .filter(d -> { + AttributeType type = d.getType().getSuper(); + return type != null && type.getBinding().equals(Number.class); + }) + .map(d -> d.getLocalName()) + .collect(Collectors.toList()); + } + public double getAreaSqKm () throws IOException, TransformException, FactoryException { CoordinateReferenceSystem webMercatorCRS = CRS.decode("EPSG:3857"); MathTransform webMercatorTransform = CRS.findMathTransform(crs, webMercatorCRS, true); From 686676441096526c939e3f3c7bdf35eb15489687 Mon Sep 17 00:00:00 2001 From: ansons Date: Wed, 26 May 2021 20:27:22 -0400 Subject: [PATCH 008/187] feat(grid): allow CSVs with no numeric attributes We populate a special [COUNT] grid anyways Related: #696 --- src/main/java/com/conveyal/r5/analyst/Grid.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 5f05380fc..363b55a99 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -584,11 +584,11 @@ public static List fromCsv(InputStreamProvider csvInputStreamProvider, for (Iterator it = numericColumns.iterator(); it.hasNext();) { String field = it.next(); String value = reader.get(field); - if (value == null || "".equals(value)) continue; // allow missing data + if (value == null || "".equals(value)) continue; // allow missing data TODO add "N/A" etc.? try { double dv = parseDouble(value); if (!(Double.isFinite(dv) || dv < 0)) { - it.remove(); + it.remove(); // TODO track removed columns and report to UI? } } catch (NumberFormatException e) { it.remove(); @@ -599,9 +599,6 @@ public static List fromCsv(InputStreamProvider csvInputStreamProvider, // This will also close the InputStreams. reader.close(); - if (numericColumns.isEmpty()) { - throw new IllegalArgumentException("CSV file contained no entirely finite, non-negative numeric columns."); - } checkWgsEnvelopeSize(envelope); WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(envelope, zoom); checkPixelCount(extents, numericColumns.size()); From 08af79cbe1a2da8a4ed7682433ced42129e70b67 Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 28 May 2021 00:14:34 -0400 Subject: [PATCH 009/187] fix(grid): ignore .shp.xml files --- .../analysis/controllers/OpportunityDatasetController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 5a6532453..3fe817167 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -415,6 +415,8 @@ private Set extractFileExtensions (List fileItems) { private void verifyBaseNamesSame (List fileItems) { String firstBaseName = null; for (FileItem fileItem : fileItems) { + // Ignore .shp.xml files + if (FilenameUtils.getExtension(fileItem.getName()).equals(".xml")) continue; String baseName = FilenameUtils.getBaseName(fileItem.getName()); if (firstBaseName == null) { firstBaseName = baseName; From 270b42e242c902f1aa22f3a75221fbb63468f130 Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 28 May 2021 00:15:41 -0400 Subject: [PATCH 010/187] fix(db): string equality check --- src/main/java/com/conveyal/analysis/persistence/MongoMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java index 80fd07072..b88bddac4 100644 --- a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java +++ b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java @@ -170,7 +170,7 @@ public V updateByUserIfPermitted(V value, String updatedBy, String accessGroup) } public V put(String key, V value) { - if (key != value._id) throw AnalysisServerException.badRequest("ID does not match"); + if (!key.equals(value._id)) throw AnalysisServerException.badRequest("ID does not match"); return put(value, null); } From d216b4057628c9ed90c6a46cdc1d6b6c56c9d076 Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 28 May 2021 00:18:22 -0400 Subject: [PATCH 011/187] Clean up misc. comments/imports --- src/main/java/com/conveyal/analysis/SelectingGridReducer.java | 1 - .../com/conveyal/analysis/persistence/AnalysisCollection.java | 2 +- src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java | 2 +- src/main/java/com/conveyal/r5/util/ShapefileReader.java | 2 -- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/SelectingGridReducer.java b/src/main/java/com/conveyal/analysis/SelectingGridReducer.java index 38cd73e36..e6b81dd26 100644 --- a/src/main/java/com/conveyal/analysis/SelectingGridReducer.java +++ b/src/main/java/com/conveyal/analysis/SelectingGridReducer.java @@ -20,7 +20,6 @@ * When storing bootstrap replications of travel time, we also store the point estimate (using all Monte Carlo draws * equally weighted) as the first value, so a SelectingGridReducer(0) can be used to retrieve the point estimate. * - * This class is not referenced within R5, but is used by the Analysis front end. */ public class SelectingGridReducer { diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index fe7e43984..9a8754bdb 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -67,7 +67,7 @@ public T create(T newModel, String accessGroup, String creatorEmail) { newModel.createdBy = creatorEmail; newModel.updatedBy = creatorEmail; - // This creates the `_id` automatically + // This creates the `_id` automatically if it is missing collection.insertOne(newModel); return newModel; diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 4c31e75fd..35a3306d3 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -35,7 +35,7 @@ public AnalysisDB (Config config) { database = mongo.getDatabase(config.databaseName()).withCodecRegistry(pojoCodecRegistry); - // Reqeust that the JVM clean up database connections in all cases - exiting cleanly or by being terminated. + // Request that the JVM clean up database connections in all cases - exiting cleanly or by being terminated. // We should probably register such hooks for other components to shut down more cleanly. Runtime.getRuntime().addShutdownHook(new Thread(() -> { Persistence.mongo.close(); diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index fe0171a26..cda14e7b8 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -14,7 +14,6 @@ import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.feature.type.AttributeDescriptor; import org.opengis.feature.type.AttributeType; import org.opengis.filter.Filter; import org.opengis.referencing.FactoryException; @@ -24,7 +23,6 @@ import java.io.File; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; From a32b892e9a42fc3cc39005bfa9137925e680b7ce Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 30 May 2021 22:56:19 -0400 Subject: [PATCH 012/187] refactor(spatial): move upload validation methods to new class --- .../OpportunityDatasetController.java | 102 ++-------------- .../analysis/spatial/SpatialDataset.java | 115 ++++++++++++++++++ 2 files changed, 122 insertions(+), 95 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 5a6532453..1c15e90bd 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -6,6 +6,7 @@ import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.Region; import com.conveyal.analysis.persistence.Persistence; +import com.conveyal.analysis.spatial.SpatialDataset; import com.conveyal.analysis.util.FileItemInputStreamProvider; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; @@ -55,6 +56,8 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; +import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; @@ -334,98 +337,6 @@ private String getFormField(Map> formFields, String field } } - private enum UploadFormat { - SHAPEFILE, GRID, CSV - } - - /** - * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary - * grids. In the process we validate the list of uploaded files, making sure certain preconditions are met. - * Some kinds of uploads must contain multiple files (.shp) or can contain multiple files (.grid) while others - * must have only a single file (.csv). Scan the list of uploaded files to ensure it makes sense before acting. - * @throws AnalysisServerException if the type of the upload can't be detected or preconditions are violated. - * @return the expected type of the uploaded file or files, never null. - */ - private UploadFormat detectUploadFormatAndValidate (List fileItems) { - if (fileItems == null || fileItems.isEmpty()) { - throw AnalysisServerException.fileUpload("You must include some files to create an opportunity dataset."); - } - - Set fileExtensions = extractFileExtensions(fileItems); - - // There was at least one file with an extension, the set must now contain at least one extension. - if (fileExtensions.isEmpty()) { - throw AnalysisServerException.fileUpload("No file extensions seen, cannot detect upload type."); - } - - UploadFormat uploadFormat = null; - - // Check that if upload contains any of the Shapefile sidecar files, it contains all of the required ones. - final Set shapefileExtensions = Sets.newHashSet("SHP", "DBF", "PRJ"); - if ( ! Sets.intersection(fileExtensions, shapefileExtensions).isEmpty()) { - if (fileExtensions.containsAll(shapefileExtensions)) { - uploadFormat = UploadFormat.SHAPEFILE; - verifyBaseNamesSame(fileItems); - // TODO check that any additional file is SHX, and that there are no more than 4 files. - } else { - final String message = "You must multi-select at least SHP, DBF, and PRJ files for shapefile upload."; - throw AnalysisServerException.fileUpload(message); - } - } - - // Even if we've already detected a shapefile, run the other tests to check for a bad mixture of file types. - if (fileExtensions.contains("GRID")) { - if (fileExtensions.size() == 1) { - uploadFormat = UploadFormat.GRID; - } else { - String message = "When uploading grids you may upload multiple files, but they must all be grids."; - throw AnalysisServerException.fileUpload(message); - } - } else if (fileExtensions.contains("CSV")) { - if (fileItems.size() == 1) { - uploadFormat = UploadFormat.CSV; - } else { - String message = "When uploading CSV you may only upload one file at a time."; - throw AnalysisServerException.fileUpload(message); - } - } - - if (uploadFormat == null) { - throw AnalysisServerException.fileUpload("Could not detect format of opportunity dataset upload."); - } - return uploadFormat; - } - - private Set extractFileExtensions (List fileItems) { - - Set fileExtensions = new HashSet<>(); - - for (FileItem fileItem : fileItems) { - String fileName = fileItem.getName(); - String extension = FilenameUtils.getExtension(fileName); - if (extension.isEmpty()) { - throw AnalysisServerException.fileUpload("Filename has no extension: " + fileName); - } - fileExtensions.add(extension.toUpperCase()); - } - - return fileExtensions; - } - - private void verifyBaseNamesSame (List fileItems) { - String firstBaseName = null; - for (FileItem fileItem : fileItems) { - String baseName = FilenameUtils.getBaseName(fileItem.getName()); - if (firstBaseName == null) { - firstBaseName = baseName; - } - if (!firstBaseName.equals(baseName)) { - String message = "In a shapefile upload, all files must have the same base name."; - throw AnalysisServerException.fileUpload(message); - } - } - } - /** * Handle many types of file upload. Returns a OpportunityDatasetUploadStatus which has a handle to request status. * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. @@ -456,6 +367,7 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res final List fileItems; final UploadFormat uploadFormat; + final SpatialDataset.SourceFormat uploadFormat; final Map parameters; try { // Validate inputs and parameters, which will throw an exception if there's anything wrong with them. @@ -475,13 +387,13 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res try { // A place to accumulate all the PointSets created, both FreeForm and Grids. List pointsets = new ArrayList<>(); - if (uploadFormat == UploadFormat.GRID) { + if (uploadFormat == SourceFormat.GRID) { LOG.info("Detected opportunity dataset stored in Conveyal binary format."); pointsets.addAll(createGridsFromBinaryGridFiles(fileItems, status)); - } else if (uploadFormat == UploadFormat.SHAPEFILE) { + } else if (uploadFormat == SourceFormat.SHAPEFILE) { LOG.info("Detected opportunity dataset stored as ESRI shapefile."); pointsets.addAll(createGridsFromShapefile(fileItems, zoom, status)); - } else if (uploadFormat == UploadFormat.CSV) { + } else if (uploadFormat == SourceFormat.CSV) { LOG.info("Detected opportunity dataset stored as CSV"); // Create a grid even when user has requested a freeform pointset so we have something to visualize. FileItem csvFileItem = fileItems.get(0); diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java new file mode 100644 index 000000000..f0cb2b9e1 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java @@ -0,0 +1,115 @@ +package com.conveyal.analysis.spatial; + +import com.conveyal.analysis.AnalysisServerException; +import com.google.common.collect.Sets; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.io.FilenameUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Common methods for validating and processing uploaded spatial data files. + */ +public class SpatialDataset { + + public enum SourceFormat { + SHAPEFILE, CSV, GEOJSON, GRID, SEAMLESS + } + + public enum GeometryType { + LINE, POLYGON, POINT, GRID + } + + /** + * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary + * grids. In the process we validate the list of uploaded files, making sure certain preconditions are met. + * Some kinds of uploads must contain multiple files (.shp) or can contain multiple files (.grid) while others + * must have only a single file (.csv). Scan the list of uploaded files to ensure it makes sense before acting. + * @throws AnalysisServerException if the type of the upload can't be detected or preconditions are violated. + * @return the expected type of the uploaded file or files, never null. + */ + public static SourceFormat detectUploadFormatAndValidate (List fileItems) { + if (fileItems == null || fileItems.isEmpty()) { + throw AnalysisServerException.fileUpload("You must include some files to create an opportunity dataset."); + } + + Set fileExtensions = extractFileExtensions(fileItems); + + // There was at least one file with an extension, the set must now contain at least one extension. + if (fileExtensions.isEmpty()) { + throw AnalysisServerException.fileUpload("No file extensions seen, cannot detect upload type."); + } + + SourceFormat uploadFormat = null; + + // Check that if upload contains any of the Shapefile sidecar files, it contains all of the required ones. + final Set shapefileExtensions = Sets.newHashSet("SHP", "DBF", "PRJ"); + if ( ! Sets.intersection(fileExtensions, shapefileExtensions).isEmpty()) { + if (fileExtensions.containsAll(shapefileExtensions)) { + uploadFormat = SourceFormat.SHAPEFILE; + verifyBaseNamesSame(fileItems); + // TODO check that any additional file is SHX, and that there are no more than 4 files. + } else { + final String message = "You must multi-select at least SHP, DBF, and PRJ files for shapefile upload."; + throw AnalysisServerException.fileUpload(message); + } + } + + // Even if we've already detected a shapefile, run the other tests to check for a bad mixture of file types. + if (fileExtensions.contains("GRID")) { + if (fileExtensions.size() == 1) { + uploadFormat = SourceFormat.GRID; + } else { + String message = "When uploading grids you may upload multiple files, but they must all be grids."; + throw AnalysisServerException.fileUpload(message); + } + } else if (fileExtensions.contains("CSV")) { + if (fileItems.size() == 1) { + uploadFormat = SourceFormat.CSV; + } else { + String message = "When uploading CSV you may only upload one file at a time."; + throw AnalysisServerException.fileUpload(message); + } + } + + if (uploadFormat == null) { + throw AnalysisServerException.fileUpload("Could not detect format of opportunity dataset upload."); + } + return uploadFormat; + } + + private static Set extractFileExtensions (List fileItems) { + + Set fileExtensions = new HashSet<>(); + + for (FileItem fileItem : fileItems) { + String fileName = fileItem.getName(); + String extension = FilenameUtils.getExtension(fileName); + if (extension.isEmpty()) { + throw AnalysisServerException.fileUpload("Filename has no extension: " + fileName); + } + fileExtensions.add(extension.toUpperCase()); + } + + return fileExtensions; + } + + private static void verifyBaseNamesSame (List fileItems) { + String firstBaseName = null; + for (FileItem fileItem : fileItems) { + // Ignore .shp.xml files + if (FilenameUtils.getExtension(fileItem.getName()).equals(".xml")) continue; + String baseName = FilenameUtils.getBaseName(fileItem.getName()); + if (firstBaseName == null) { + firstBaseName = baseName; + } + if (!firstBaseName.equals(baseName)) { + String message = "In a shapefile upload, all files must have the same base name."; + throw AnalysisServerException.fileUpload(message); + } + } + } + +} From 99103b3b3cfe8666999b894e5293c12d422f148e Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 30 May 2021 22:57:53 -0400 Subject: [PATCH 013/187] feat(nonce): remove locking from newer BaseModel --- src/main/java/com/conveyal/analysis/models/BaseModel.java | 3 --- .../conveyal/analysis/persistence/AnalysisCollection.java | 7 ------- 2 files changed, 10 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index d3bfec99d..e894d5031 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -6,9 +6,6 @@ public class BaseModel { // Can retrieve `createdAt` from here public ObjectId _id; - // For version management. ObjectId's contain a timestamp, so can retrieve `updatedAt` from here. - public ObjectId nonce = new ObjectId(); - public String createdBy = null; public String updatedBy = null; diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index fe7e43984..088853aa8 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -78,14 +78,9 @@ public T update(T value) { } public T update(T value, String accessGroup) { - // Store the current nonce for querying and to check later if needed. - ObjectId oldNonce = value.nonce; - - value.nonce = new ObjectId(); UpdateResult result = collection.replaceOne(and( eq("_id", value._id), - eq("nonce", oldNonce), eq("accessGroup", accessGroup) ), value); @@ -94,8 +89,6 @@ public T update(T value, String accessGroup) { T model = findById(value._id); if (model == null) { throw AnalysisServerException.notFound(type.getName() + " was not found."); - } else if (model.nonce != oldNonce) { - throw AnalysisServerException.nonce(); } else if (!model.accessGroup.equals(accessGroup)) { throw invalidAccessGroup(); } else { From 60e2f04f86620163735ea19c15d72d8db08a11e6 Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 30 May 2021 23:04:48 -0400 Subject: [PATCH 014/187] feat(spatial-source): initial work for grouping spatial datasets by their source --- .../OpportunityDatasetController.java | 83 +++++++++---------- .../conveyal/analysis/models/BaseModel.java | 16 ++++ .../analysis/models/SpatialDatasetSource.java | 29 +++++++ .../conveyal/r5/analyst/progress/Task.java | 4 +- .../r5/analyst/progress/WorkProduct.java | 8 +- .../r5/analyst/progress/WorkProductType.java | 11 +-- 6 files changed, 93 insertions(+), 58 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 1c15e90bd..279a21ef1 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -1,10 +1,12 @@ package com.conveyal.analysis.controllers; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.Region; +import com.conveyal.analysis.models.SpatialDatasetSource; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.spatial.SpatialDataset; import com.conveyal.analysis.util.FileItemInputStreamProvider; @@ -15,11 +17,10 @@ import com.conveyal.r5.analyst.FreeFormPointSet; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.PointSet; -import com.conveyal.r5.analyst.WebMercatorExtents; +import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ExceptionUtils; import com.conveyal.r5.util.InputStreamProvider; import com.conveyal.r5.util.ProgressListener; -import com.google.common.collect.Sets; import com.google.common.io.Files; import com.mongodb.QueryBuilder; import org.apache.commons.fileupload.FileItem; @@ -48,14 +49,13 @@ import java.util.Collection; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; @@ -143,29 +143,31 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); - // default - final String accessGroup = req.attribute("accessGroup"); - final String email = req.attribute("email"); - final Region region = Persistence.regions.findByIdIfPermitted(regionId, accessGroup); + UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + final Region region = Persistence.regions.findByIdIfPermitted(regionId, userPermissions.accessGroup); // Common UUID for all LODES datasets created in this download (e.g. so they can be grouped together and // deleted as a batch using deleteSourceSet) - final String downloadBatchId = new ObjectId().toString(); // The bucket name contains the specific lodes data set and year so works as an appropriate name final OpportunityDatasetUploadStatus status = new OpportunityDatasetUploadStatus(regionId, extractor.sourceName); addStatusAndRemoveOldStatuses(status); - taskScheduler.enqueueHeavyTask(() -> { - try { - status.message = "Extracting census data for region"; - List grids = extractor.censusDataForBounds(region.bounds, zoom); - createDatasetsFromPointSets( - email, accessGroup, extractor.sourceName, downloadBatchId, regionId, status, grids - ); - } catch (IOException e) { - status.completeWithError(e); - LOG.error("Exception processing LODES data: " + ExceptionUtils.stackTraceString(e)); - } - }); + SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, extractor.sourceName) + .withRegion(regionId); + + taskScheduler.enqueue(Task.create("Extracting LODES data") + .forUser(userPermissions) + .setHeavy(true) + .withWorkProduct(source) + .withAction((progressListener) -> { + try { + status.message = "Extracting census data for region"; + List grids = extractor.censusDataForBounds(region.bounds, zoom); + updateAndStoreDatasets(source, status, grids); + } catch (IOException e) { + status.completeWithError(e); + LOG.error("Exception processing LODES data: " + ExceptionUtils.stackTraceString(e)); + } + })); return status; } @@ -174,31 +176,21 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) * Given a list of new PointSets, serialize each PointSet and save it to S3, then create a metadata object about * that PointSet and store it in Mongo. */ - private List createDatasetsFromPointSets(String email, - String accessGroup, - String sourceName, - String sourceId, - String regionId, - OpportunityDatasetUploadStatus status, - List pointSets) { - status.status = Status.UPLOADING; - status.totalGrids = pointSets.size(); + private void updateAndStoreDatasets (SpatialDatasetSource source, + OpportunityDatasetUploadStatus status, + List pointSets) { // Create an OpportunityDataset holding some metadata about each PointSet (Grid or FreeForm). final List datasets = new ArrayList<>(); for (PointSet pointSet : pointSets) { - - // Make new PointSet metadata objects. - // Unfortunately we can't pull this step out into a method because there are so many parameters. - // Some of that metadata could be consolidated e.g. user email and access group. OpportunityDataset dataset = new OpportunityDataset(); - dataset.sourceName = sourceName; - dataset.sourceId = sourceId; + dataset.sourceName = source.name; + dataset.sourceId = source._id.toString(); + dataset.createdBy = source.createdBy; + dataset.accessGroup = source.accessGroup; + dataset.regionId = source.regionId; dataset.name = pointSet.name; - dataset.createdBy = email; - dataset.accessGroup = accessGroup; dataset.totalPoints = pointSet.featureCount(); - dataset.regionId = regionId; dataset.totalOpportunities = pointSet.sumTotalOpportunities(); dataset.format = getFormatCode(pointSet); if (dataset.format == FileStorageFormat.FREEFORM) { @@ -235,7 +227,8 @@ private List createDatasetsFromPointSets(String email, fileStorage.moveIntoStorage(dataset.getStorageKey(FileStorageFormat.GRID), gridFile); } else if (pointSet instanceof FreeFormPointSet) { // Upload serialized freeform pointset back to S3 - FileStorageKey fileStorageKey = new FileStorageKey(GRIDS, regionId + "/" + dataset._id + ".pointset"); + FileStorageKey fileStorageKey = new FileStorageKey(GRIDS, source.regionId + "/" + dataset._id + + ".pointset"); File pointsetFile = FileUtils.createScratchFile("pointset"); OutputStream os = new GZIPOutputStream(new FileOutputStream(pointsetFile)); @@ -257,7 +250,6 @@ private List createDatasetsFromPointSets(String email, throw AnalysisServerException.unknown(e); } } - return datasets; } private static FileStorageFormat getFormatCode (PointSet pointSet) { @@ -342,8 +334,7 @@ private String getFormField(Map> formFields, String field * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. */ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Response res) { - final String accessGroup = req.attribute("accessGroup"); - final String email = req.attribute("email"); + final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); final Map> formFields; try { ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); @@ -366,7 +357,6 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res addStatusAndRemoveOldStatuses(status); final List fileItems; - final UploadFormat uploadFormat; final SpatialDataset.SourceFormat uploadFormat; final Map parameters; try { @@ -427,8 +417,9 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res // Create a single unique ID string that will be referenced by all opportunity datasets produced by // this upload. This allows us to group together datasets from the same source and associate them with // the file(s) that produced them. - final String sourceFileId = new ObjectId().toString(); - createDatasetsFromPointSets(email, accessGroup, sourceName, sourceFileId, regionId, status, pointsets); + SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName) + .withRegion(regionId); + updateAndStoreDatasets(source, status, pointsets); } catch (Exception e) { status.completeWithError(e); } diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index e894d5031..8ceff2cf2 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.models; +import com.conveyal.analysis.UserPermissions; import org.bson.types.ObjectId; public class BaseModel { @@ -14,4 +15,19 @@ public class BaseModel { // Everything has a name public String name = null; + + // package private to encourage use of static factory methods + BaseModel (UserPermissions user, String name){ + this._id = new ObjectId(); + this.createdBy = user.email; + this.updatedBy = user.email; + this.accessGroup = user.accessGroup; + this.name = name; + } + + + BaseModel () { + // No-arg + } + } diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java new file mode 100644 index 000000000..5d8e5ad4a --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -0,0 +1,29 @@ +package com.conveyal.analysis.models; + +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.spatial.SpatialDataset.GeometryType; +import com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; + +import java.util.Map; + +public class SpatialDatasetSource extends BaseModel { + public String regionId; + public String description; + public SourceFormat sourceFormat; + public GeometryType geometryType; + public Map attributes; + + private SpatialDatasetSource (UserPermissions userPermissions, String sourceName) { + super(userPermissions, sourceName); + } + + public static SpatialDatasetSource create (UserPermissions userPermissions, String sourceName) { + return new SpatialDatasetSource(userPermissions, sourceName); + } + + public SpatialDatasetSource withRegion (String regionId) { + this.regionId = regionId; + return this; + } + +} diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index 949ad2d22..a321a24cb 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -1,7 +1,7 @@ package com.conveyal.r5.analyst.progress; import com.conveyal.analysis.UserPermissions; -import com.conveyal.analysis.models.Model; +import com.conveyal.analysis.models.BaseModel; import com.conveyal.r5.util.ExceptionUtils; import java.time.Duration; @@ -251,7 +251,7 @@ public Task withAction (TaskAction action) { // We can't return the WorkProduct from TaskAction, that would be disrupted by throwing exceptions. // It is also awkward to make a method to set it on ProgressListener - it's not really progress. // So we set it directly on the task before submitting it. Requires pre-setting (not necessarily storing) Model._id. - public Task withWorkProduct (Model model) { + public Task withWorkProduct (BaseModel model) { this.workProduct = WorkProduct.forModel(model); return this; } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java index beff7e3e6..e9a9f9761 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java @@ -1,7 +1,6 @@ package com.conveyal.r5.analyst.progress; -import com.conveyal.analysis.controllers.UserActivityController; -import com.conveyal.analysis.models.Model; +import com.conveyal.analysis.models.BaseModel; /** * A unique identifier for the final product of a single task action. Currently this serves as both an @@ -21,9 +20,8 @@ public WorkProduct (WorkProductType type, String id, String regionId) { } // FIXME Not all Models have a regionId. Rather than pass that in as a String, refine the programming API. - public static WorkProduct forModel (Model model) { - WorkProduct product = new WorkProduct(WorkProductType.forModel(model), model._id, null); - return product; + public static WorkProduct forModel (BaseModel model) { + return new WorkProduct(WorkProductType.forModel(model), model._id.toString(), null); } } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java index bb71ba955..d3240b203 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java @@ -1,10 +1,10 @@ package com.conveyal.r5.analyst.progress; -import com.conveyal.analysis.controllers.UserActivityController; +import com.conveyal.analysis.models.AggregationArea; import com.conveyal.analysis.models.Bundle; -import com.conveyal.analysis.models.Model; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.RegionalAnalysis; +import com.conveyal.analysis.models.SpatialDatasetSource; /** * There is some implicit and unenforced correspondence between these values and those in FileCategory, as well @@ -13,13 +13,14 @@ */ public enum WorkProductType { - BUNDLE, REGIONAL_ANALYSIS, AGGREGATION_AREA, OPPORTUNITY_DATASET; + BUNDLE, REGIONAL_ANALYSIS, AGGREGATION_AREA, OPPORTUNITY_DATASET, SPATIAL_DATASET_SOURCE; - // Currently we have two base classes for db objects so may need to use Object instead of BaseModel parameter - public static WorkProductType forModel (Model model) { + public static WorkProductType forModel (Object model) { if (model instanceof Bundle) return BUNDLE; if (model instanceof OpportunityDataset) return OPPORTUNITY_DATASET; if (model instanceof RegionalAnalysis) return REGIONAL_ANALYSIS; + if (model instanceof AggregationArea) return AGGREGATION_AREA; + if (model instanceof SpatialDatasetSource) return SPATIAL_DATASET_SOURCE; throw new IllegalArgumentException("Unrecognized work product type."); } } From d7cf39f7b00c41e8b61ad5ccfd5cd06dd62d877a Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 30 May 2021 23:05:24 -0400 Subject: [PATCH 015/187] refactor(spatial): factor out method for setting extents --- .../controllers/OpportunityDatasetController.java | 12 +----------- .../analysis/models/OpportunityDataset.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 279a21ef1..75907b397 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -196,17 +196,7 @@ private void updateAndStoreDatasets (SpatialDatasetSource source, if (dataset.format == FileStorageFormat.FREEFORM) { dataset.name = String.join(" ", pointSet.name, "(freeform)"); } - // These bounds are currently in web Mercator pixels, which are relevant to Grids but are not natural units - // for FreeformPointSets. There are only unique minimal web Mercator bounds for FreeformPointSets because - // the zoom level is fixed in OpportunityDataset (there is not even a field for it). - // Perhaps these metadata bounds should be WGS84 instead, it depends how the UI uses them. - { - WebMercatorExtents webMercatorExtents = pointSet.getWebMercatorExtents(); - dataset.north = webMercatorExtents.north; - dataset.west = webMercatorExtents.west; - dataset.width = webMercatorExtents.width; - dataset.height = webMercatorExtents.height; - } + dataset.setWebMercatorExtents(pointSet); // TODO make origin and destination pointsets reference each other and indicate they are suitable // for one-to-one analyses diff --git a/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java b/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java index dc977b586..77696cc04 100644 --- a/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java +++ b/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java @@ -2,6 +2,7 @@ import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; +import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.WebMercatorExtents; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -103,6 +104,19 @@ public WebMercatorExtents getWebMercatorExtents () { return new WebMercatorExtents(west, north, width, height, DEFAULT_ZOOM); } + @JsonIgnore + public void setWebMercatorExtents (PointSet pointset) { + // These bounds are currently in web Mercator pixels, which are relevant to Grids but are not natural units + // for FreeformPointSets. There are only unique minimal web Mercator bounds for FreeformPointSets if + // the zoom level is fixed in OpportunityDataset (FIXME we may change this soon). + // Perhaps these metadata bounds should be WGS84 instead, it depends how the UI uses them. + WebMercatorExtents extents = pointset.getWebMercatorExtents(); + this.west = extents.west; + this.north = extents.north; + this.width = extents.width; + this.height = extents.height; + } + /** Analysis region this dataset was uploaded in. */ public String regionId; } From 732a7ee95a69f8abbb5fd7d58f946ea56a2f5202 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 31 May 2021 21:15:42 -0400 Subject: [PATCH 016/187] feat(spatial): transition aggregation areas to use new database, base API model, and task tracking --- .../components/BackendComponents.java | 2 +- .../AggregationAreaController.java | 65 ++++++++++++------- .../analysis/models/AggregationArea.java | 21 +++++- .../persistence/AnalysisCollection.java | 4 ++ .../analysis/persistence/Persistence.java | 2 - 5 files changed, 68 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 197f46b22..d711c64da 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -92,7 +92,7 @@ public List standardHttpControllers () { new BundleController(this), new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor), new RegionalAnalysisController(broker, fileStorage), - new AggregationAreaController(fileStorage), + new AggregationAreaController(fileStorage, database, taskScheduler), new TimetableController(), new FileStorageController(fileStorage, database), // This broker controller registers at least one handler at URL paths beginning with /internal, which diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index faca62d73..38d5d5de4 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -1,16 +1,20 @@ package com.conveyal.analysis.controllers; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.models.AggregationArea; -import com.conveyal.analysis.persistence.Persistence; +import com.conveyal.analysis.models.SpatialDatasetSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; import com.conveyal.r5.analyst.Grid; +import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ShapefileReader; import com.google.common.io.Files; -import com.mongodb.QueryBuilder; import org.apache.commons.fileupload.FileItem; import org.json.simple.JSONObject; import org.locationtech.jts.geom.Envelope; @@ -34,9 +38,13 @@ import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; +import static com.conveyal.analysis.components.HttpApi.USER_GROUP_ATTRIBUTE; +import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.mongodb.client.model.Filters.and; +import static com.mongodb.client.model.Filters.eq; /** * Stores vector aggregationAreas (used to define the region of a weighted average accessibility metric). @@ -53,9 +61,18 @@ public class AggregationAreaController implements HttpController { private static int MAX_FEATURES = 100; private final FileStorage fileStorage; + private final TaskScheduler taskScheduler; - public AggregationAreaController (FileStorage fileStorage) { + private final AnalysisCollection aggregationAreaCollection; + + public AggregationAreaController ( + FileStorage fileStorage, + AnalysisDB database, + TaskScheduler taskScheduler + ) { this.fileStorage = fileStorage; + this.taskScheduler = taskScheduler; + this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); } private FileStorageKey getStoragePath (AggregationArea area) { @@ -73,6 +90,12 @@ private FileStorageKey getStoragePath (AggregationArea area) { private List createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); Map> query = HttpUtils.getRequestFiles(req.raw()); + UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + String maskName = query.get("name").get(0).getString("UTF-8"); + String regionId = req.params("regionId"); + + SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, maskName) + .withRegion(regionId); // 1. Extract relevant files: .shp, .prj, .dbf, and .shx. ====================================================== Map filesByName = query.get("files").stream() @@ -90,8 +113,6 @@ private List createAggregationAreas (Request req, Response res) throw AnalysisServerException.fileUpload("Shapefile upload must contain .shp, .prj, and .dbf"); } - String regionId = req.params("regionId"); - File tempDir = Files.createTempDir(); File shpFile = new File(tempDir, "grid.shp"); @@ -119,7 +140,6 @@ private List createAggregationAreas (Request req, Response res) if (reader != null) reader.close(); } - Map areas = new HashMap<>(); boolean unionRequested = Boolean.parseBoolean(query.get("union").get(0).getString()); @@ -136,15 +156,23 @@ private List createAggregationAreas (Request req, Response res) List geometries = features.stream().map(f -> (Geometry) f.getDefaultGeometry()).collect(Collectors.toList()); UnaryUnionOp union = new UnaryUnionOp(geometries); // Name the area using the name in the request directly - String maskName = query.get("name").get(0).getString("UTF-8"); areas.put(maskName, union.union()); } else { // Don't union. Name each area by looking up its value for the name property in the request. String nameProperty = query.get("nameProperty").get(0).getString("UTF-8"); features.forEach(f -> areas.put(readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry())); } + + taskScheduler.enqueue(Task.create("Creating aggregation area") + .forUser(userPermissions) + .setHeavy(true) + .withWorkProduct(source) + // TODO move below into .withAction() + ); + // 3. Convert to raster grids, then store them. ================================================================ areas.forEach((String name, Geometry geometry) -> { + if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); Envelope env = geometry.getEnvelopeInternal(); Grid maskGrid = new Grid(zoom, env); @@ -152,13 +180,8 @@ private List createAggregationAreas (Request req, Response res) List weights = maskGrid.getPixelWeights(geometry, true); weights.forEach(pixel -> maskGrid.grid[pixel.x][pixel.y] = pixel.weight * 100_000); - AggregationArea aggregationArea = new AggregationArea(); - aggregationArea.name = name; - aggregationArea.regionId = regionId; - - // Set `createdBy` and `accessGroup` - aggregationArea.accessGroup = req.attribute("accessGroup"); - aggregationArea.createdBy = req.attribute("email"); + AggregationArea aggregationArea = AggregationArea.create(userPermissions, name) + .withSource(source); try { File gridFile = FileUtils.createScratchFile("grid"); @@ -167,7 +190,7 @@ private List createAggregationAreas (Request req, Response res) os.close(); // Create the aggregation area before generating the S3 key so that the `_id` is generated - Persistence.aggregationAreas.create(aggregationArea); + aggregationAreaCollection.insert(aggregationArea); aggregationAreas.add(aggregationArea); fileStorage.moveIntoStorage(getStoragePath(aggregationArea), gridFile); @@ -192,17 +215,15 @@ private String readProperty (SimpleFeature feature, String propertyName) { } private Collection getAggregationAreas (Request req, Response res) { - return Persistence.aggregationAreas.findPermitted( - QueryBuilder.start("regionId").is(req.params("regionId")).get(), - req.attribute("accessGroup") + return aggregationAreaCollection.findPermitted( + and(eq("regionId", req.queryParams("regionId"))), req.attribute(USER_GROUP_ATTRIBUTE) ); } private Object getAggregationArea (Request req, Response res) { - final String accessGroup = req.attribute("accessGroup"); - final String maskId = req.params("maskId"); - - AggregationArea aggregationArea = Persistence.aggregationAreas.findByIdIfPermitted(maskId, accessGroup); + AggregationArea aggregationArea = (AggregationArea) aggregationAreaCollection.findPermitted( + eq("_id", req.params("maskId")), req.attribute(USER_GROUP_ATTRIBUTE) + ); String url = fileStorage.getURL(getStoragePath(aggregationArea)); JSONObject wrappedUrl = new JSONObject(); diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index 253e0fe3b..565827218 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.models; +import com.conveyal.analysis.UserPermissions; import com.fasterxml.jackson.annotation.JsonIgnore; /** @@ -7,8 +8,26 @@ * It is defined by a geometry that is rasterized and stored as a grid, with pixels with values between 0 and 100,000 * depending on how much of that pixel is overlapped by the mask. */ -public class AggregationArea extends Model { +public class AggregationArea extends BaseModel { public String regionId; + public String sourceId; + + private AggregationArea(UserPermissions user, String name) { + super(user, name); + } + + // FLUENT METHODS FOR CONFIGURING + + /** Call this static factory to begin building a task. */ + public static AggregationArea create (UserPermissions user, String name) { + return new AggregationArea(user, name); + } + + public AggregationArea withSource (SpatialDatasetSource source) { + this.regionId = source.regionId; + this.sourceId = source._id.toString(); + return this; + } @JsonIgnore public String getS3Key () { diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 088853aa8..5fa867918 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -73,6 +73,10 @@ public T create(T newModel, String accessGroup, String creatorEmail) { return newModel; } + public void insert (T model) { + collection.insertOne(model); + } + public T update(T value) { return update(value, value.accessGroup); } diff --git a/src/main/java/com/conveyal/analysis/persistence/Persistence.java b/src/main/java/com/conveyal/analysis/persistence/Persistence.java index 71129c0a9..785694a80 100644 --- a/src/main/java/com/conveyal/analysis/persistence/Persistence.java +++ b/src/main/java/com/conveyal/analysis/persistence/Persistence.java @@ -40,7 +40,6 @@ public class Persistence { public static MongoMap bundles; public static MongoMap regions; public static MongoMap regionalAnalyses; - public static MongoMap aggregationAreas; public static MongoMap opportunityDatasets; // TODO progressively migrate to AnalysisDB which is non-static @@ -59,7 +58,6 @@ public static void initializeStatically (AnalysisDB.Config config) { bundles = getTable("bundles", Bundle.class); regions = getTable("regions", Region.class); regionalAnalyses = getTable("regional-analyses", RegionalAnalysis.class); - aggregationAreas = getTable("aggregationAreas", AggregationArea.class); opportunityDatasets = getTable("opportunityDatasets", OpportunityDataset.class); } From 6bc48a7bd7184cac64d59a43a3194ab15bfaaf17 Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Wed, 2 Jun 2021 17:14:38 -0400 Subject: [PATCH 017/187] Apply suggestions from code review Co-authored-by: Andrew Byrd --- .../analysis/controllers/OpportunityDatasetController.java | 4 ++-- src/main/java/com/conveyal/r5/util/ShapefileReader.java | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 3fe817167..93b25938a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -415,8 +415,8 @@ private Set extractFileExtensions (List fileItems) { private void verifyBaseNamesSame (List fileItems) { String firstBaseName = null; for (FileItem fileItem : fileItems) { - // Ignore .shp.xml files - if (FilenameUtils.getExtension(fileItem.getName()).equals(".xml")) continue; + // Ignore .shp.xml files, which will fail the verifyBaseNamesSame check + if ("xml".equalsIgnoreCase(FilenameUtils.getExtension(fileItem.getName()))) continue; String baseName = FilenameUtils.getBaseName(fileItem.getName()); if (firstBaseName == null) { firstBaseName = baseName; diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index cda14e7b8..d04e6d4bc 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -96,11 +96,8 @@ public List numericAttributes () { return features.getSchema() .getAttributeDescriptors() .stream() - .filter(d -> { - AttributeType type = d.getType().getSuper(); - return type != null && type.getBinding().equals(Number.class); - }) - .map(d -> d.getLocalName()) + .filter(d -> Number.class.isInstance(d.getType().getBinding())) + .map(AttributeDescriptor::getLocalName) .collect(Collectors.toList()); } From 179d742b851cfaa589b4ae8cd39a036b438ceca7 Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Wed, 2 Jun 2021 17:19:49 -0400 Subject: [PATCH 018/187] Apply suggestions from code review Co-authored-by: Andrew Byrd --- src/main/java/com/conveyal/r5/analyst/Grid.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 363b55a99..6b35b859c 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -101,7 +101,7 @@ public class Grid extends PointSet { /** Limit on number of pixels, to prevent OOME when multiple large grids are being created (width * height * * number of layers/attributes) */ - private static final int MAX_PIXELS = 100_000_000 * 10; + private static final int MAX_PIXELS = 10_000 * 10_000 * 10; /** Used when reading a saved grid. */ public Grid (int zoom, int width, int height, int north, int west) { @@ -504,8 +504,8 @@ public static double pixelToLat (double yPixel, int zoom) { } /** - * Given a pixel's local coordinates in the supplied WebMercatorExtents, return its geometry in absolute (world) - * coordinates + * Given a pixel's local grid coordinates within the supplied WebMercatorExtents, return a closed + * polygon of that pixel's outline in WGS84 global geographic coordinates. * @param localX x pixel number within the given extents. * @param localY y pixel number within the given extents. * @return a JTS Polygon in WGS84 coordinates for the given pixel. From a69e3f601d66d48da690fc5b87aeb6d1f516ffba Mon Sep 17 00:00:00 2001 From: ansons Date: Wed, 2 Jun 2021 17:24:01 -0400 Subject: [PATCH 019/187] refactor(grid): revise method signature for consistency with WebMercatorExtents constructor --- .../java/com/conveyal/analysis/SelectingGridReducer.java | 2 +- src/main/java/com/conveyal/r5/analyst/Grid.java | 5 ++--- src/test/java/com/conveyal/r5/analyst/GridTest.java | 8 +++----- .../com/conveyal/r5/analyst/GridTransformWrapperTest.java | 7 +++---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/SelectingGridReducer.java b/src/main/java/com/conveyal/analysis/SelectingGridReducer.java index e6b81dd26..bbff784d1 100644 --- a/src/main/java/com/conveyal/analysis/SelectingGridReducer.java +++ b/src/main/java/com/conveyal/analysis/SelectingGridReducer.java @@ -63,7 +63,7 @@ public Grid compute (InputStream rawInput) throws IOException { // median travel time. int nSamples = input.readInt(); - Grid outputGrid = new Grid(zoom, width, height, north, west); + Grid outputGrid = new Grid(west, north, width, height, zoom); int[] valuesThisOrigin = new int[nSamples]; diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 6b35b859c..6f186f185 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -34,7 +34,6 @@ import org.opengis.feature.Property; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.feature.type.AttributeDescriptor; import org.opengis.parameter.GeneralParameterValue; import org.opengis.parameter.ParameterValueGroup; import org.opengis.referencing.FactoryException; @@ -104,7 +103,7 @@ public class Grid extends PointSet { private static final int MAX_PIXELS = 10_000 * 10_000 * 10; /** Used when reading a saved grid. */ - public Grid (int zoom, int width, int height, int north, int west) { + public Grid (int west, int north, int width, int height, int zoom) { this(new WebMercatorExtents(west, north, width, height, zoom)); } @@ -343,7 +342,7 @@ public static Grid read (InputStream inputStream) throws IOException { int width = data.readInt(); int height = data.readInt(); - Grid grid = new Grid(zoom, width, height, north, west); + Grid grid = new Grid(west, north, width, height, zoom); // loop in row-major order for (int y = 0, value = 0; y < height; y++) { diff --git a/src/test/java/com/conveyal/r5/analyst/GridTest.java b/src/test/java/com/conveyal/r5/analyst/GridTest.java index 9a6d051e5..6798f548e 100644 --- a/src/test/java/com/conveyal/r5/analyst/GridTest.java +++ b/src/test/java/com/conveyal/r5/analyst/GridTest.java @@ -7,10 +7,8 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; -import java.util.Arrays; import java.util.List; import java.util.Random; -import java.util.stream.DoubleStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -32,7 +30,7 @@ public void testGetMercatorEnvelopeMeters() throws Exception { int zoom = 4; int xTile = 14; int yTile = 9; - Grid grid = new Grid(zoom, 256, 256, 256 * yTile, 256 * xTile); + Grid grid = new Grid(256 * xTile, 256 * yTile, 256, 256, zoom); ReferencedEnvelope envelope = grid.getWebMercatorExtents().getMercatorEnvelopeMeters(); assertEquals(15028131.257091936, envelope.getMinX(), 0.1); assertEquals(-5009377.085697312, envelope.getMinY(), 0.1); @@ -43,7 +41,7 @@ public void testGetMercatorEnvelopeMeters() throws Exception { zoom = 5; xTile = 16; yTile = 11; - grid = new Grid(zoom, 256, 256, 256 * yTile, 256 * xTile); + grid = new Grid(256 * xTile, 256 * yTile, 256, 256, zoom); envelope = grid.getWebMercatorExtents().getMercatorEnvelopeMeters(); assertEquals(0, envelope.getMinX(), 0.1); assertEquals(5009377.085697312, envelope.getMinY(), 0.1); @@ -107,7 +105,7 @@ private static Grid generateRandomGrid (Random random, boolean wholeNumbersOnly) int width = random.nextInt(MAX_GRID_WIDTH_PIXELS) + 1; int height = random.nextInt(MAX_GRID_WIDTH_PIXELS) + 1; - Grid grid = new Grid(zoom, width, height, north, west); + Grid grid = new Grid(west, north, width, height, zoom); for (int y = 0; y < grid.extents.height; y++) { for (int x = 0; x < grid.extents.width; x++) { double amount = random.nextDouble() * MAX_AMOUNT; diff --git a/src/test/java/com/conveyal/r5/analyst/GridTransformWrapperTest.java b/src/test/java/com/conveyal/r5/analyst/GridTransformWrapperTest.java index 9e5f33eb0..a74fc3f87 100644 --- a/src/test/java/com/conveyal/r5/analyst/GridTransformWrapperTest.java +++ b/src/test/java/com/conveyal/r5/analyst/GridTransformWrapperTest.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Random; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -18,11 +17,11 @@ class GridTransformWrapperTest { void testTwoAdjacentGrids () { // Two grids side by side, right one bigger than than the left, with top 20 pixels lower - Grid leftGrid = new Grid(10, 200, 300, 1000, 2000); - Grid rightGrid = new Grid(10, 300, 400, 1020, 2200); + Grid leftGrid = new Grid(2000, 1000, 200, 300, 10); + Grid rightGrid = new Grid(2200, 1020, 300, 400, 10); // One minimum bounding grid exactly encompassing the other two. - Grid superGrid = new Grid(10, 500, 400, 1000, 2000); + Grid superGrid = new Grid(2000, 1000, 500, 400, 10); // Make a column of pixel weights 2 pixels wide and 26 pixels high. List weights = new ArrayList<>(); From a91865c976b1ed03e933bcd08eaabb6fbb971199 Mon Sep 17 00:00:00 2001 From: ansons Date: Wed, 2 Jun 2021 17:24:17 -0400 Subject: [PATCH 020/187] refactor(grid): rename method for clarity --- .../java/com/conveyal/analysis/models/AnalysisRequest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 8749e2e21..7eb39b629 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -2,7 +2,6 @@ import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.persistence.Persistence; -import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; import com.conveyal.r5.analyst.decay.DecayFunction; @@ -229,7 +228,7 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project // Also include getIndex(x, y), getX(index), getY(index), totalTasks() WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(bounds.envelope(), zoom); - checkZoom(extents); + checkGridSize(extents); task.height = extents.height; task.north = extents.north; task.west = extents.west; @@ -287,7 +286,7 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project return task; } - private static void checkZoom(WebMercatorExtents extents) { + private static void checkGridSize (WebMercatorExtents extents) { if (extents.zoom < MIN_ZOOM || extents.zoom > MAX_ZOOM) { throw AnalysisServerException.badRequest(String.format( "Requested zoom (%s) is outside valid range (%s - %s)", extents.zoom, MIN_ZOOM, MAX_ZOOM From 3d4bdb476e235faa9d1fffd9881281bf58a1df0b Mon Sep 17 00:00:00 2001 From: ansons Date: Wed, 2 Jun 2021 17:28:01 -0400 Subject: [PATCH 021/187] Resolve missing import --- src/main/java/com/conveyal/r5/util/ShapefileReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index d04e6d4bc..3767f6fb2 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -14,6 +14,7 @@ import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.feature.type.AttributeDescriptor; import org.opengis.feature.type.AttributeType; import org.opengis.filter.Filter; import org.opengis.referencing.FactoryException; From 34f7d9c01d7df9e8ba8c4017c6c81210d60588cb Mon Sep 17 00:00:00 2001 From: ansons Date: Thu, 3 Jun 2021 19:39:39 -0400 Subject: [PATCH 022/187] refactor(spatial-source): copy methods from OpportunityDatasetController copypasta cleanup pending --- .../AggregationAreaController.java | 1 - .../controllers/SpatialDatasetController.java | 608 ++++++++++++++++++ .../analysis/models/SpatialDatasetSource.java | 1 + .../com/conveyal/r5/util/ShapefileReader.java | 20 + 4 files changed, 629 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 38d5d5de4..567d4922a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -62,7 +62,6 @@ public class AggregationAreaController implements HttpController { private final FileStorage fileStorage; private final TaskScheduler taskScheduler; - private final AnalysisCollection aggregationAreaCollection; public AggregationAreaController ( diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java new file mode 100644 index 000000000..a5a84f09b --- /dev/null +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -0,0 +1,608 @@ +package com.conveyal.analysis.controllers; + +import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.components.TaskScheduler; +import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; +import com.conveyal.analysis.models.OpportunityDataset; +import com.conveyal.analysis.models.Region; +import com.conveyal.analysis.models.SpatialDatasetSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.analysis.persistence.AnalysisDB; +import com.conveyal.analysis.persistence.Persistence; +import com.conveyal.analysis.util.FileItemInputStreamProvider; +import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; +import com.conveyal.file.FileUtils; +import com.conveyal.r5.analyst.FreeFormPointSet; +import com.conveyal.r5.analyst.Grid; +import com.conveyal.r5.analyst.PointSet; +import com.conveyal.r5.analyst.WebMercatorExtents; +import com.conveyal.r5.analyst.progress.Task; +import com.conveyal.r5.util.ExceptionUtils; +import com.conveyal.r5.util.InputStreamProvider; +import com.conveyal.r5.util.ShapefileReader; +import com.google.common.io.Files; +import com.mongodb.QueryBuilder; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.FileItemFactory; +import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.disk.DiskFileItemFactory; +import org.apache.commons.fileupload.servlet.ServletFileUpload; +import org.apache.commons.io.FilenameUtils; +import org.json.simple.JSONObject; +import org.locationtech.jts.geom.Envelope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; +import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; +import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; +import static com.conveyal.analysis.util.JsonUtil.toJson; +import static com.conveyal.file.FileCategory.GRIDS; +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; + +/** + * Controller that handles fetching opportunity datasets (grids and other pointset formats). + */ +public class SpatialDatasetController implements HttpController { + + private static final Logger LOG = LoggerFactory.getLogger(SpatialDatasetController.class); + + private static final FileItemFactory fileItemFactory = new DiskFileItemFactory(); + + // Component Dependencies + + private final FileStorage fileStorage; + private final AnalysisCollection spatialSourceCollection; + private final TaskScheduler taskScheduler; + private final SeamlessCensusGridExtractor extractor; + + public SpatialDatasetController ( + FileStorage fileStorage, + AnalysisDB database, + TaskScheduler taskScheduler, + SeamlessCensusGridExtractor extractor + ) { + this.fileStorage = fileStorage; + this.spatialSourceCollection = database.getAnalysisCollection("spatialSources", SpatialDatasetSource.class); + this.taskScheduler = taskScheduler; + this.extractor = extractor; + } + + private JSONObject getJSONURL (FileStorageKey key) { + JSONObject json = new JSONObject(); + String url = fileStorage.getURL(key); + json.put("url", url); + return json; + } + + private Collection getRegionDatasets(Request req, Response res) { + return Persistence.opportunityDatasets.findPermitted( + QueryBuilder.start("regionId").is(req.params("regionId")).get(), + req.attribute("accessGroup") + ); + } + + private Object getOpportunityDataset(Request req, Response res) { + OpportunityDataset dataset = Persistence.opportunityDatasets.findByIdFromRequestIfPermitted(req); + if (dataset.format == FileStorageFormat.GRID) { + return getJSONURL(dataset.getStorageKey()); + } else { + // Currently the UI can only visualize grids, not other kinds of datasets (freeform points). + // We do generate a rasterized grid for each of the freeform pointsets we create, so ideally we'd redirect + // to that grid for display and preview, but the freeform and corresponding grid pointset have different + // IDs and there are no references between them. + LOG.error("We cannot yet visualize freeform pointsets. Returning nothing to the UI."); + return null; + } + } + + private SpatialDatasetSource downloadLODES(Request req, Response res) { + final String regionId = req.params("regionId"); + final int zoom = parseZoom(req.queryParams("zoom")); + + UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + final Region region = Persistence.regions.findByIdIfPermitted(regionId, userPermissions.accessGroup); + // Common UUID for all LODES datasets created in this download (e.g. so they can be grouped together and + // deleted as a batch using deleteSourceSet) + // The bucket name contains the specific lodes data set and year so works as an appropriate name + + SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, extractor.sourceName) + .withRegion(regionId); + + taskScheduler.enqueue(Task.create("Extracting LODES data") + .forUser(userPermissions) + .setHeavy(true) + .withWorkProduct(source) + .withAction((progressListener) -> { + // TODO implement + })); + + return source; + } + + /** + * Given a list of new PointSets, serialize each PointSet and save it to S3, then create a metadata object about + * that PointSet and store it in Mongo. + */ + private void updateAndStoreDatasets (SpatialDatasetSource source, + List pointSets) { + + // Create an OpportunityDataset holding some metadata about each PointSet (Grid or FreeForm). + final List datasets = new ArrayList<>(); + for (PointSet pointSet : pointSets) { + OpportunityDataset dataset = new OpportunityDataset(); + dataset.sourceName = source.name; + dataset.sourceId = source._id.toString(); + dataset.createdBy = source.createdBy; + dataset.accessGroup = source.accessGroup; + dataset.regionId = source.regionId; + dataset.name = pointSet.name; + dataset.totalPoints = pointSet.featureCount(); + dataset.totalOpportunities = pointSet.sumTotalOpportunities(); + dataset.format = getFormatCode(pointSet); + if (dataset.format == FileStorageFormat.FREEFORM) { + dataset.name = String.join(" ", pointSet.name, "(freeform)"); + } + dataset.setWebMercatorExtents(pointSet); + // TODO make origin and destination pointsets reference each other and indicate they are suitable + // for one-to-one analyses + + // Store the PointSet metadata in Mongo and accumulate these objects into the method return list. + Persistence.opportunityDatasets.create(dataset); + datasets.add(dataset); + + // Persist a serialized representation of each PointSet (not the metadata) to S3 or other object storage. + // TODO this should probably be pulled out to another method, and possibly called one frame up. + // Persisting the PointSets to S3 is a separate task than making metadata and storing in Mongo. + try { + if (pointSet instanceof Grid) { + File gridFile = FileUtils.createScratchFile("grid"); + + OutputStream fos = new GZIPOutputStream(new FileOutputStream(gridFile)); + ((Grid)pointSet).write(fos); + + fileStorage.moveIntoStorage(dataset.getStorageKey(FileStorageFormat.GRID), gridFile); + } else if (pointSet instanceof FreeFormPointSet) { + // Upload serialized freeform pointset back to S3 + FileStorageKey fileStorageKey = new FileStorageKey(GRIDS, source.regionId + "/" + dataset._id + + ".pointset"); + File pointsetFile = FileUtils.createScratchFile("pointset"); + + OutputStream os = new GZIPOutputStream(new FileOutputStream(pointsetFile)); + ((FreeFormPointSet)pointSet).write(os); + + fileStorage.moveIntoStorage(fileStorageKey, pointsetFile); + } else { + throw new IllegalArgumentException("Unrecognized PointSet type, cannot persist it."); + } + // TODO task tracking + } catch (NumberFormatException e) { + throw new AnalysisServerException("Error attempting to parse number in uploaded file: " + e.toString()); + } catch (Exception e) { + throw AnalysisServerException.unknown(e); + } + } + } + + private static FileStorageFormat getFormatCode (PointSet pointSet) { + if (pointSet instanceof FreeFormPointSet) { + return FileStorageFormat.FREEFORM; + } else if (pointSet instanceof Grid) { + return FileStorageFormat.GRID; + } else { + throw new RuntimeException("Unknown pointset type."); + } + } + + /** + * Given a CSV file, converts each property (CSV column) into a freeform (non-gridded) pointset. + * + * The provided multipart form data must include latField and lonField. To indicate paired origins and destinations + * (e.g. to use results from an origin-destination survey in a one-to-one regional analysis), the form data should + * include the optional latField2 and lonField2 fields. + * + * This method executes in a blocking (synchronous) manner, but it can take a while so should be called within an + * non-blocking asynchronous task. + */ + private List createFreeFormPointSetsFromCsv(FileItem csvFileItem, Map params) { + + String latField = params.get("latField"); + String lonField = params.get("lonField"); + if (latField == null || lonField == null) { + throw AnalysisServerException.fileUpload("You must specify a latitude and longitude column."); + } + + // The name of the column containing a unique identifier for each row. May be missing (null). + String idField = params.get("idField"); + + // The name of the column containing the opportunity counts at each point. May be missing (null). + String countField = params.get("countField"); + + // Optional secondary latitude, longitude, and count fields. + // This allows you to create two matched parallel pointsets of the same size with the same IDs. + String latField2 = params.get("latField2"); + String lonField2 = params.get("lonField2"); + + try { + List pointSets = new ArrayList<>(); + InputStreamProvider csvStreamProvider = new FileItemInputStreamProvider(csvFileItem); + pointSets.add(FreeFormPointSet.fromCsv(csvStreamProvider, latField, lonField, idField, countField)); + // The second pair of lat and lon fields allow creating two matched pointsets from the same CSV. + // This is used for one-to-one travel times between specific origins/destinations. + if (latField2 != null && lonField2 != null) { + pointSets.add(FreeFormPointSet.fromCsv(csvStreamProvider, latField2, lonField2, idField, countField)); + } + return pointSets; + } catch (Exception e) { + throw AnalysisServerException.fileUpload("Could not convert CSV to Freeform PointSet: " + e.toString()); + } + + } + + /** + * Get the specified field from a map representing a multipart/form-data POST request, as a UTF-8 String. + * FileItems represent any form item that was received within a multipart/form-data POST request, not just files. + */ + private String getFormField(Map> formFields, String fieldName, boolean required) { + try { + List fileItems = formFields.get(fieldName); + if (fileItems == null || fileItems.isEmpty()) { + if (required) { + throw AnalysisServerException.badRequest("Missing required field: " + fieldName); + } else { + return null; + } + } + String value = fileItems.get(0).getString("UTF-8"); + return value; + } catch (UnsupportedEncodingException e) { + throw AnalysisServerException.badRequest(String.format("Multipart form field '%s' had unsupported encoding", + fieldName)); + } + } + + /** + * Handle many types of spatial upload. + * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. + */ + private void handleUpload(Request req, Response res) { + final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + final Map> formFields; + try { + ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); + formFields = sfu.parseParameterMap(req.raw()); + } catch (FileUploadException e) { + // We can't even get enough information to create a status tracking object. Re-throw an exception. + throw AnalysisServerException.fileUpload("Unable to parse uploaded file(s). " + ExceptionUtils.stackTraceString(e)); + } + + // Parse required fields. Will throw a ServerException on failure. + final String sourceName = getFormField(formFields, "Name", true); + final String regionId = getFormField(formFields, "regionId", true); + + SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName).withRegion(regionId); + + // TODO tracking + final List fileItems; + final SourceFormat uploadFormat; + try { + // Validate inputs and parameters, which will throw an exception if there's anything wrong with them. + // Call remove() rather than get() so that subsequent code will see only string parameters, not the files. + fileItems = formFields.remove("files"); + uploadFormat = detectUploadFormatAndValidate(fileItems); + parameters = extractStringParameters(formFields); + } catch (Exception e) { + // TODO tracking + } + + LOG.info("Handling uploaded {} file", uploadFormat); + + if (uploadFormat == SourceFormat.GRID) { + + } else if (uploadFormat == SourceFormat.SHAPEFILE) { + processShapefile(fileItems); + } else if (uploadFormat == SourceFormat.CSV) { + + } else if (uploadFormat == SourceFormat.GEOJSON) { + + } + + // TODO move to storage + } + + /** + * Given pre-parsed multipart POST data containing some text fields, pull those fields out into a simple String + * Map to simplify later use, performing some validation in the process. + * All FileItems are expected to be form fields, not uploaded files, and all items should have only a single subitem + * which can be understood as a UTF-8 String. + */ + private Map extractStringParameters(Map> formFields) { + // All other keys should be for String parameters. + Map parameters = new HashMap<>(); + formFields.forEach((key, items) -> { + if (items.size() != 1) { + LOG.error("In multipart form upload, key '{}' had {} sub-items (expected one).", key, items.size()); + } + FileItem fileItem = items.get(0); + if (fileItem.isFormField()) { + try { + parameters.put(key, fileItem.getString("UTF-8")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + LOG.warn("In multipart form upload, key '{}' was not for a form field.", key); + } + }); + return parameters; + } + + private OpportunityDataset editOpportunityDataset(Request request, Response response) throws IOException { + return Persistence.opportunityDatasets.updateFromJSONRequest(request); + } + + private Collection deleteSourceSet(Request request, Response response) { + String sourceId = request.params("sourceId"); + String accessGroup = request.attribute("accessGroup"); + Collection datasets = Persistence.opportunityDatasets.findPermitted( + QueryBuilder.start("sourceId").is(sourceId).get(), accessGroup); + + datasets.forEach(dataset -> deleteDataset(dataset._id, accessGroup)); + + return datasets; + } + + private OpportunityDataset deleteOpportunityDataset(Request request, Response response) { + String opportunityDatasetId = request.params("_id"); + return deleteDataset(opportunityDatasetId, request.attribute("accessGroup")); + } + + /** + * Delete an Opportunity Dataset from the database and all formats from the file store. + */ + private OpportunityDataset deleteDataset(String id, String accessGroup) { + OpportunityDataset dataset = Persistence.opportunityDatasets.removeIfPermitted(id, accessGroup); + + if (dataset == null) { + throw AnalysisServerException.notFound("Opportunity dataset could not be found."); + } else { + fileStorage.delete(dataset.getStorageKey(FileStorageFormat.GRID)); + fileStorage.delete(dataset.getStorageKey(FileStorageFormat.PNG)); + fileStorage.delete(dataset.getStorageKey(FileStorageFormat.TIFF)); + } + + return dataset; + } + + /** + * Create a grid from WGS 84 points in a CSV file. + * The supplied CSV file will not be deleted - it may be used again to make another (freeform) pointset. + * TODO explain latField2 usage + * @return one or two Grids for each numeric column in the CSV input. + */ + private List createGridsFromCsv(FileItem csvFileItem, + Map> query, + int zoom) throws Exception { + + String latField = getFormField(query, "latField", true); + String lonField = getFormField(query, "lonField", true); + String idField = getFormField(query, "idField", false); + + // Optional fields to run grid construction twice with two different sets of points. + // This is only really useful when creating grids to visualize freeform pointsets for one-to-one analyses. + String latField2 = getFormField(query, "latField2", false); + String lonField2 = getFormField(query, "lonField2", false); + + List ignoreFields = Arrays.asList(idField, latField2, lonField2); + InputStreamProvider csvStreamProvider = new FileItemInputStreamProvider(csvFileItem); + List grids = Grid.fromCsv(csvStreamProvider, latField, lonField, ignoreFields, zoom, status); + // TODO verify correctness of this second pass + if (latField2 != null && lonField2 != null) { + ignoreFields = Arrays.asList(idField, latField, lonField); + grids.addAll(Grid.fromCsv(csvStreamProvider, latField2, lonField2, ignoreFields, zoom, status)); + } + + return grids; + } + + /** + * Create a grid from an input stream containing a binary grid file. + * For those in the know, we can upload manually created binary grid files. + */ + private List createGridsFromBinaryGridFiles(List uploadedFiles, + OpportunityDatasetUploadStatus status) throws Exception { + + List grids = new ArrayList<>(); + status.totalFeatures = uploadedFiles.size(); + for (FileItem fileItem : uploadedFiles) { + Grid grid = Grid.read(fileItem.getInputStream()); + String name = fileItem.getName(); + // Remove ".grid" from the name + if (name.contains(".grid")) name = name.split(".grid")[0]; + grid.name = name; + grids.add(grid); + status.completedFeatures += 1; + } + status.completedFeatures = status.totalFeatures; + return grids; + } + + /** + * Preconditions: fileItems must contain SHP, DBF, and PRJ files, and optionally SHX. All files should have the + * same base name, and should not contain any other files but these three or four. + */ + private SpatialDatasetSource processShapefile(List fileItems) throws Exception { + + // In the caller, we should have already verified that all files have the same base name and have an extension. + // Extract the relevant files: .shp, .prj, .dbf, and .shx. + // We need the SHX even though we're looping over every feature as they might be sparse. + Map filesByExtension = new HashMap<>(); + for (FileItem fileItem : fileItems) { + filesByExtension.put(FilenameUtils.getExtension(fileItem.getName()).toUpperCase(), fileItem); + } + + // Copy the shapefile component files into a temporary directory with a fixed base name. + File tempDir = Files.createTempDir(); + + File shpFile = new File(tempDir, "grid.shp"); + filesByExtension.get("SHP").write(shpFile); + + File prjFile = new File(tempDir, "grid.prj"); + filesByExtension.get("PRJ").write(prjFile); + + File dbfFile = new File(tempDir, "grid.dbf"); + filesByExtension.get("DBF").write(dbfFile); + + // The .shx file is an index. It is optional, and not needed for dense shapefiles. + if (filesByExtension.containsKey("SHX")) { + File shxFile = new File(tempDir, "grid.shx"); + filesByExtension.get("SHX").write(shxFile); + } + + ShapefileReader reader = new ShapefileReader(shpFile); + Envelope envelope = reader.wgs84Bounds(); + checkWgsEnvelopeSize(envelope); + + + reader.getAttributeTypes(); + + if (uniqueNumericAttributes.size() != numericAttributes.size()) { + throw new IllegalArgumentException("Shapefile has duplicate numeric attributes"); + } + checkPixelCount(extents, numericAttributes.size()); + + int total = reader.getFeatureCount(); + if (progressListener != null) { + progressListener.setTotalItems(total); + } + + AtomicInteger count = new AtomicInteger(0); + Map grids = new HashMap<>(); + + reader.wgs84Stream().forEach(feat -> { + Geometry geom = (Geometry) feat.getDefaultGeometry(); + + for (Property p : feat.getProperties()) { + Object val = p.getValue(); + + if (!(val instanceof Number)) continue; + double numericVal = ((Number) val).doubleValue(); + if (numericVal == 0) continue; + + String attributeName = p.getName().getLocalPart(); + + Grid grid = grids.get(attributeName); + if (grid == null) { + grid = new Grid(extents); + grid.name = attributeName; + grids.put(attributeName, grid); + } + + if (geom instanceof Point) { + Point point = (Point) geom; + // already in WGS 84 + grid.incrementPoint(point.getY(), point.getX(), numericVal); + } else if (geom instanceof Polygon || geom instanceof MultiPolygon) { + grid.rasterize(geom, numericVal); + } else { + throw new IllegalArgumentException("Unsupported geometry type"); + } + } + + int currentCount = count.incrementAndGet(); + if (progressListener != null) { + progressListener.setCompletedItems(currentCount); + } + if (currentCount % 10000 == 0) { + LOG.info("{} / {} features read", human(currentCount), human(total)); + } + }); + reader.close(); + return new ArrayList<>(grids.values()); + } + + /** + * Respond to a request with a redirect to a downloadable file. + * @param req should specify regionId, opportunityDatasetId, and an available download format (.tiff or .grid) + */ + private Object downloadOpportunityDataset (Request req, Response res) throws IOException { + FileStorageFormat downloadFormat; + try { + downloadFormat = FileStorageFormat.valueOf(req.params("format").toUpperCase()); + } catch (IllegalArgumentException iae) { + // This code handles the deprecated endpoint for retrieving opportunity datasets + // get("/api/opportunities/:regionId/:gridKey") is the same signature as this endpoint. + String regionId = req.params("_id"); + String gridKey = req.params("format"); + FileStorageKey storageKey = new FileStorageKey(GRIDS, String.format("%s/%s.grid", regionId, gridKey)); + return getJSONURL(storageKey); + } + + if (FileStorageFormat.GRID.equals(downloadFormat)) return getOpportunityDataset(req, res); + + final OpportunityDataset opportunityDataset = Persistence.opportunityDatasets.findByIdFromRequestIfPermitted(req); + + FileStorageKey gridKey = opportunityDataset.getStorageKey(FileStorageFormat.GRID); + FileStorageKey formatKey = opportunityDataset.getStorageKey(downloadFormat); + + // if this grid is not on S3 in the requested format, try to get the .grid format + if (!fileStorage.exists(gridKey)) { + throw AnalysisServerException.notFound("Requested grid does not exist."); + } + + if (!fileStorage.exists(formatKey)) { + // get the grid and convert it to the requested format + File gridFile = fileStorage.getFile(gridKey); + Grid grid = Grid.read(new GZIPInputStream(new FileInputStream(gridFile))); // closes input stream + File localFile = FileUtils.createScratchFile(downloadFormat.toString()); + FileOutputStream fos = new FileOutputStream(localFile); + + if (FileStorageFormat.PNG.equals(downloadFormat)) { + grid.writePng(fos); + } else if (FileStorageFormat.TIFF.equals(downloadFormat)) { + grid.writeGeotiff(fos); + } + + fileStorage.moveIntoStorage(formatKey, localFile); + } + + return getJSONURL(formatKey); + } + + @Override + public void registerEndpoints (spark.Service sparkService) { + sparkService.path("/api/opportunities", () -> { + sparkService.post("", this::createOpportunityDataset, toJson); + sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); + sparkService.get("/region/:regionId/status", this::getRegionUploadStatuses, toJson); + sparkService.delete("/region/:regionId/status/:statusId", this::clearStatus, toJson); + sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); + sparkService.delete("/source/:sourceId", this::deleteSourceSet, toJson); + sparkService.delete("/:_id", this::deleteOpportunityDataset, toJson); + sparkService.get("/:_id", this::getOpportunityDataset, toJson); + sparkService.put("/:_id", this::editOpportunityDataset, toJson); + sparkService.get("/:_id/:format", this::downloadOpportunityDataset, toJson); + }); + } +} diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index 5d8e5ad4a..45dba98a9 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -12,6 +12,7 @@ public class SpatialDatasetSource extends BaseModel { public SourceFormat sourceFormat; public GeometryType geometryType; public Map attributes; + public int featureCount; private SpatialDatasetSource (UserPermissions userPermissions, String sourceName) { super(userPermissions, sourceName); diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 3767f6fb2..284dbbffe 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -1,5 +1,6 @@ package com.conveyal.r5.util; +import com.conveyal.analysis.AnalysisServerException; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; import org.geotools.data.FeatureSource; @@ -25,6 +26,7 @@ import java.io.File; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -138,4 +140,22 @@ public void close () { public int getFeatureCount() throws IOException { return source.getCount(Query.ALL); } + + public Map getAttributeTypes () { + Map attributes = new HashMap<>(); + HashSet uniqueAttributes = new HashSet<>(); + features.getSchema() + .getAttributeDescriptors() + .forEach(d -> { + String attributeName = d.getLocalName(); + AttributeType type = d.getType().getSuper(); + attributes.put(attributeName, type.getBinding()); + uniqueAttributes.add(attributeName); + }); + if (attributes.size() != uniqueAttributes.size()) { + throw new AnalysisServerException("Shapefile has duplicate " + + "attributes."); + } + return attributes; + } } From fbd02ad7aaab2f1e3a0ed589db18efaaba8492fb Mon Sep 17 00:00:00 2001 From: ansons Date: Thu, 3 Jun 2021 19:40:41 -0400 Subject: [PATCH 023/187] refactor(db): add id at instantiation rather than in constructor --- src/main/java/com/conveyal/analysis/models/BaseModel.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index 8ceff2cf2..ef3f219a1 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -5,7 +5,7 @@ public class BaseModel { // Can retrieve `createdAt` from here - public ObjectId _id; + public ObjectId _id = new ObjectId(); public String createdBy = null; public String updatedBy = null; @@ -17,8 +17,7 @@ public class BaseModel { public String name = null; // package private to encourage use of static factory methods - BaseModel (UserPermissions user, String name){ - this._id = new ObjectId(); + BaseModel (UserPermissions user, String name) { this.createdBy = user.email; this.updatedBy = user.email; this.accessGroup = user.accessGroup; From dde5784b774207c48c0d78a9301d49cd0a30dd36 Mon Sep 17 00:00:00 2001 From: ansons Date: Thu, 3 Jun 2021 20:28:29 -0400 Subject: [PATCH 024/187] docs(db): add notice about mongo driver mutating record --- .../com/conveyal/analysis/persistence/AnalysisCollection.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index e891e43bc..7e2e2f089 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -73,6 +73,10 @@ public T create(T newModel, String accessGroup, String creatorEmail) { return newModel; } + /** + * Note that if the supplied model has _id = null, the Mongo insertOne method will overwrite it with a new + * ObjectId(). We consider it good practice to set the _id for any model object ourselves, avoiding this behavior. + */ public void insert (T model) { collection.insertOne(model); } From 7d7ff78ed4661382b7d498ebde92d6823b426b5c Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 4 Jun 2021 11:12:09 -0400 Subject: [PATCH 025/187] fix(shapefile): correct numeric attributes filter https://github.com/conveyal/r5/pull/731/files#r645297401 --- src/main/java/com/conveyal/r5/util/ShapefileReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 284dbbffe..d9302470b 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -99,7 +99,7 @@ public List numericAttributes () { return features.getSchema() .getAttributeDescriptors() .stream() - .filter(d -> Number.class.isInstance(d.getType().getBinding())) + .filter(d -> Number.class.isAssignableFrom(d.getType().getBinding())) .map(AttributeDescriptor::getLocalName) .collect(Collectors.toList()); } From 5e857b79ff79f6116a491018808ff9929b942388 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 8 Jun 2021 08:19:46 -0400 Subject: [PATCH 026/187] docs(files): small javadoc updates --- .../conveyal/analysis/controllers/FileStorageController.java | 2 +- src/main/java/com/conveyal/analysis/models/FileInfo.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java index b41f7eeef..d7724587b 100644 --- a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java +++ b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java @@ -64,7 +64,7 @@ private List findAllForRegion(Request req, Response res) { /** * Create the metadata object used to represent a file in FileStorage. Note: this does not handle the process of - * storing the file itself. See `addFile` for that. + * storing the file itself. See `uploadFile` for that. */ private FileInfo createFileInfo(Request req, Response res) throws IOException { FileInfo fileInfo = fileCollection.create(req, res); diff --git a/src/main/java/com/conveyal/analysis/models/FileInfo.java b/src/main/java/com/conveyal/analysis/models/FileInfo.java index 8c38cbf96..351058899 100644 --- a/src/main/java/com/conveyal/analysis/models/FileInfo.java +++ b/src/main/java/com/conveyal/analysis/models/FileInfo.java @@ -6,6 +6,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import org.bson.types.ObjectId; +/** + * Metadata about files uploaded to Conveyal + */ public class FileInfo extends BaseModel { public String regionId = null; From 8601e08ae38c62cf33f6c1226e458a45f225af82 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 8 Jun 2021 08:20:40 -0400 Subject: [PATCH 027/187] feat(spatial-source): continue cleanup --- .../controllers/SpatialDatasetController.java | 166 +++--------------- .../analysis/models/SpatialDatasetSource.java | 60 ++++++- 2 files changed, 88 insertions(+), 138 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index a5a84f09b..8698a4606 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -18,7 +18,6 @@ import com.conveyal.r5.analyst.FreeFormPointSet; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.PointSet; -import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ExceptionUtils; import com.conveyal.r5.util.InputStreamProvider; @@ -286,7 +285,7 @@ private String getFormField(Map> formFields, String field * Handle many types of spatial upload. * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. */ - private void handleUpload(Request req, Response res) { + private SpatialDatasetSource handleUpload(Request req, Response res) { final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); final Map> formFields; try { @@ -301,9 +300,12 @@ private void handleUpload(Request req, Response res) { final String sourceName = getFormField(formFields, "Name", true); final String regionId = getFormField(formFields, "regionId", true); + // TODO tracking + + // TODO move to storage + SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName).withRegion(regionId); - // TODO tracking final List fileItems; final SourceFormat uploadFormat; try { @@ -311,51 +313,30 @@ private void handleUpload(Request req, Response res) { // Call remove() rather than get() so that subsequent code will see only string parameters, not the files. fileItems = formFields.remove("files"); uploadFormat = detectUploadFormatAndValidate(fileItems); - parameters = extractStringParameters(formFields); + + if (uploadFormat == SourceFormat.GRID) { + // TODO source.fromGrids(fileItems); + } else if (uploadFormat == SourceFormat.SHAPEFILE) { + source.fromShapefile(fileItems); + } else if (uploadFormat == SourceFormat.CSV) { + // TODO source.fromCsv(fileItems); + } else if (uploadFormat == SourceFormat.GEOJSON) { + // TODO source.fromGeojson(fileItems); + } + + spatialSourceCollection.insert(source); + } catch (Exception e) { // TODO tracking + throw new AnalysisServerException("Problem reading files"); } LOG.info("Handling uploaded {} file", uploadFormat); - if (uploadFormat == SourceFormat.GRID) { - - } else if (uploadFormat == SourceFormat.SHAPEFILE) { - processShapefile(fileItems); - } else if (uploadFormat == SourceFormat.CSV) { - } else if (uploadFormat == SourceFormat.GEOJSON) { - } - - // TODO move to storage - } + return source; - /** - * Given pre-parsed multipart POST data containing some text fields, pull those fields out into a simple String - * Map to simplify later use, performing some validation in the process. - * All FileItems are expected to be form fields, not uploaded files, and all items should have only a single subitem - * which can be understood as a UTF-8 String. - */ - private Map extractStringParameters(Map> formFields) { - // All other keys should be for String parameters. - Map parameters = new HashMap<>(); - formFields.forEach((key, items) -> { - if (items.size() != 1) { - LOG.error("In multipart form upload, key '{}' had {} sub-items (expected one).", key, items.size()); - } - FileItem fileItem = items.get(0); - if (fileItem.isFormField()) { - try { - parameters.put(key, fileItem.getString("UTF-8")); - } catch (Exception e) { - throw new RuntimeException(e); - } - } else { - LOG.warn("In multipart form upload, key '{}' was not for a form field.", key); - } - }); - return parameters; } private OpportunityDataset editOpportunityDataset(Request request, Response response) throws IOException { @@ -416,11 +397,11 @@ private List createGridsFromCsv(FileItem csvFileItem, List ignoreFields = Arrays.asList(idField, latField2, lonField2); InputStreamProvider csvStreamProvider = new FileItemInputStreamProvider(csvFileItem); - List grids = Grid.fromCsv(csvStreamProvider, latField, lonField, ignoreFields, zoom, status); + List grids = Grid.fromCsv(csvStreamProvider, latField, lonField, ignoreFields, zoom, null); // TODO verify correctness of this second pass if (latField2 != null && lonField2 != null) { ignoreFields = Arrays.asList(idField, latField, lonField); - grids.addAll(Grid.fromCsv(csvStreamProvider, latField2, lonField2, ignoreFields, zoom, status)); + grids.addAll(Grid.fromCsv(csvStreamProvider, latField2, lonField2, ignoreFields, zoom, null)); } return grids; @@ -430,21 +411,20 @@ private List createGridsFromCsv(FileItem csvFileItem, * Create a grid from an input stream containing a binary grid file. * For those in the know, we can upload manually created binary grid files. */ - private List createGridsFromBinaryGridFiles(List uploadedFiles, - OpportunityDatasetUploadStatus status) throws Exception { + private List createGridsFromBinaryGridFiles(List uploadedFiles) throws Exception { List grids = new ArrayList<>(); - status.totalFeatures = uploadedFiles.size(); + // TODO task size with uploadedFiles.size(); for (FileItem fileItem : uploadedFiles) { Grid grid = Grid.read(fileItem.getInputStream()); String name = fileItem.getName(); // Remove ".grid" from the name if (name.contains(".grid")) name = name.split(".grid")[0]; grid.name = name; + // TODO task progress grids.add(grid); - status.completedFeatures += 1; } - status.completedFeatures = status.totalFeatures; + // TODO mark task complete return grids; } @@ -452,94 +432,8 @@ private List createGridsFromBinaryGridFiles(List uploadedFiles, * Preconditions: fileItems must contain SHP, DBF, and PRJ files, and optionally SHX. All files should have the * same base name, and should not contain any other files but these three or four. */ - private SpatialDatasetSource processShapefile(List fileItems) throws Exception { - - // In the caller, we should have already verified that all files have the same base name and have an extension. - // Extract the relevant files: .shp, .prj, .dbf, and .shx. - // We need the SHX even though we're looping over every feature as they might be sparse. - Map filesByExtension = new HashMap<>(); - for (FileItem fileItem : fileItems) { - filesByExtension.put(FilenameUtils.getExtension(fileItem.getName()).toUpperCase(), fileItem); - } - - // Copy the shapefile component files into a temporary directory with a fixed base name. - File tempDir = Files.createTempDir(); - - File shpFile = new File(tempDir, "grid.shp"); - filesByExtension.get("SHP").write(shpFile); - - File prjFile = new File(tempDir, "grid.prj"); - filesByExtension.get("PRJ").write(prjFile); - - File dbfFile = new File(tempDir, "grid.dbf"); - filesByExtension.get("DBF").write(dbfFile); - - // The .shx file is an index. It is optional, and not needed for dense shapefiles. - if (filesByExtension.containsKey("SHX")) { - File shxFile = new File(tempDir, "grid.shx"); - filesByExtension.get("SHX").write(shxFile); - } - - ShapefileReader reader = new ShapefileReader(shpFile); - Envelope envelope = reader.wgs84Bounds(); - checkWgsEnvelopeSize(envelope); - - - reader.getAttributeTypes(); - - if (uniqueNumericAttributes.size() != numericAttributes.size()) { - throw new IllegalArgumentException("Shapefile has duplicate numeric attributes"); - } - checkPixelCount(extents, numericAttributes.size()); - - int total = reader.getFeatureCount(); - if (progressListener != null) { - progressListener.setTotalItems(total); - } - - AtomicInteger count = new AtomicInteger(0); - Map grids = new HashMap<>(); - - reader.wgs84Stream().forEach(feat -> { - Geometry geom = (Geometry) feat.getDefaultGeometry(); - - for (Property p : feat.getProperties()) { - Object val = p.getValue(); - - if (!(val instanceof Number)) continue; - double numericVal = ((Number) val).doubleValue(); - if (numericVal == 0) continue; - - String attributeName = p.getName().getLocalPart(); - - Grid grid = grids.get(attributeName); - if (grid == null) { - grid = new Grid(extents); - grid.name = attributeName; - grids.put(attributeName, grid); - } - - if (geom instanceof Point) { - Point point = (Point) geom; - // already in WGS 84 - grid.incrementPoint(point.getY(), point.getX(), numericVal); - } else if (geom instanceof Polygon || geom instanceof MultiPolygon) { - grid.rasterize(geom, numericVal); - } else { - throw new IllegalArgumentException("Unsupported geometry type"); - } - } - - int currentCount = count.incrementAndGet(); - if (progressListener != null) { - progressListener.setCompletedItems(currentCount); - } - if (currentCount % 10000 == 0) { - LOG.info("{} / {} features read", human(currentCount), human(total)); - } - }); - reader.close(); - return new ArrayList<>(grids.values()); + private void createGridsFromShapefile(List fileItems) throws Exception { + // TODO implement rasterization methods } /** @@ -593,10 +487,8 @@ private Object downloadOpportunityDataset (Request req, Response res) throws IOE @Override public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/opportunities", () -> { - sparkService.post("", this::createOpportunityDataset, toJson); + sparkService.post("", this::handleUpload, toJson); sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); - sparkService.get("/region/:regionId/status", this::getRegionUploadStatuses, toJson); - sparkService.delete("/region/:regionId/status/:statusId", this::clearStatus, toJson); sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); sparkService.delete("/source/:sourceId", this::deleteSourceSet, toJson); sparkService.delete("/:_id", this::deleteOpportunityDataset, toJson); diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index 45dba98a9..2b1ac8c5b 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -3,28 +3,86 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.spatial.SpatialDataset.GeometryType; import com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; +import com.conveyal.r5.util.ShapefileReader; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.io.Files; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.io.FilenameUtils; +import org.locationtech.jts.geom.Envelope; +import java.io.File; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; + public class SpatialDatasetSource extends BaseModel { + public List files; public String regionId; public String description; public SourceFormat sourceFormat; public GeometryType geometryType; - public Map attributes; + public Map attributes; // TODO map to both type and user-modifiable label? public int featureCount; private SpatialDatasetSource (UserPermissions userPermissions, String sourceName) { super(userPermissions, sourceName); } + @JsonIgnore public static SpatialDatasetSource create (UserPermissions userPermissions, String sourceName) { return new SpatialDatasetSource(userPermissions, sourceName); } + @JsonIgnore public SpatialDatasetSource withRegion (String regionId) { this.regionId = regionId; return this; } + @JsonIgnore + public SpatialDatasetSource fromShapefile (List fileItems) throws Exception { + // In the caller, we should have already verified that all files have the same base name and have an extension. + // Extract the relevant files: .shp, .prj, .dbf, and .shx. + // We need the SHX even though we're looping over every feature as they might be sparse. + Map filesByExtension = new HashMap<>(); + for (FileItem fileItem : fileItems) { + filesByExtension.put(FilenameUtils.getExtension(fileItem.getName()).toUpperCase(), fileItem); + } + + // Copy the shapefile component files into a temporary directory with a fixed base name. + File tempDir = Files.createTempDir(); + + File shpFile = new File(tempDir, "grid.shp"); + filesByExtension.get("SHP").write(shpFile); + + File prjFile = new File(tempDir, "grid.prj"); + filesByExtension.get("PRJ").write(prjFile); + + File dbfFile = new File(tempDir, "grid.dbf"); + filesByExtension.get("DBF").write(dbfFile); + + // The .shx file is an index. It is optional, and not needed for dense shapefiles. + if (filesByExtension.containsKey("SHX")) { + File shxFile = new File(tempDir, "grid.shx"); + filesByExtension.get("SHX").write(shxFile); + } + + ShapefileReader reader = new ShapefileReader(shpFile); + Envelope envelope = reader.wgs84Bounds(); + checkWgsEnvelopeSize(envelope); + + this.attributes = reader.getAttributeTypes(); + this.sourceFormat = SourceFormat.SHAPEFILE; + // TODO this.geometryType = + return this; + } + + @JsonIgnore + public SpatialDatasetSource fromFiles (List fileItemList) { + // TODO this.files from fileItemList; + return this; + } + } From bf7c6363f782cb2329042391f5001e812f4b57ce Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 8 Jun 2021 09:07:47 -0400 Subject: [PATCH 028/187] Revert "feat(nonce): remove locking from newer BaseModel" This reverts commit 99103b3b3cfe8666999b894e5293c12d422f148e. --- src/main/java/com/conveyal/analysis/models/BaseModel.java | 3 +++ .../conveyal/analysis/persistence/AnalysisCollection.java | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index ef3f219a1..975d36493 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -7,6 +7,9 @@ public class BaseModel { // Can retrieve `createdAt` from here public ObjectId _id = new ObjectId(); + // For version management. ObjectId's contain a timestamp, so can retrieve `updatedAt` from here. + public ObjectId nonce = new ObjectId(); + public String createdBy = null; public String updatedBy = null; diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 7e2e2f089..6fc6f7085 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -86,9 +86,14 @@ public T update(T value) { } public T update(T value, String accessGroup) { + // Store the current nonce for querying and to check later if needed. + ObjectId oldNonce = value.nonce; + + value.nonce = new ObjectId(); UpdateResult result = collection.replaceOne(and( eq("_id", value._id), + eq("nonce", oldNonce), eq("accessGroup", accessGroup) ), value); @@ -97,6 +102,8 @@ public T update(T value, String accessGroup) { T model = findById(value._id); if (model == null) { throw AnalysisServerException.notFound(type.getName() + " was not found."); + } else if (model.nonce != oldNonce) { + throw AnalysisServerException.nonce(); } else if (!model.accessGroup.equals(accessGroup)) { throw invalidAccessGroup(); } else { From 88776324df6fff84acbc2717aff7f76b9c0116ec Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 14 Jun 2021 08:47:03 -0400 Subject: [PATCH 029/187] feat(spatial-source): use new spatial setup for aggregation areas --- .../AggregationAreaController.java | 59 ++++---------- .../controllers/SpatialDatasetController.java | 78 +++++++++--------- .../analysis/models/SpatialDatasetSource.java | 81 ++++++++++--------- .../analysis/spatial/GeometryWrapper.java | 10 +++ .../com/conveyal/analysis/spatial/Lines.java | 11 +++ .../com/conveyal/analysis/spatial/Points.java | 19 +++++ .../conveyal/analysis/spatial/Polygons.java | 27 +++++++ .../analysis/spatial/SpatialAttribute.java | 29 +++++++ .../analysis/spatial/SpatialDataset.java | 12 ++- .../com/conveyal/file/FileStorageFormat.java | 6 +- .../com/conveyal/r5/util/ShapefileReader.java | 31 +++++-- 11 files changed, 227 insertions(+), 136 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java create mode 100644 src/main/java/com/conveyal/analysis/spatial/Lines.java create mode 100644 src/main/java/com/conveyal/analysis/spatial/Points.java create mode 100644 src/main/java/com/conveyal/analysis/spatial/Polygons.java create mode 100644 src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 567d4922a..51a01f13e 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -7,6 +7,7 @@ import com.conveyal.analysis.models.SpatialDatasetSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; +import com.conveyal.analysis.spatial.Polygons; import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageKey; @@ -14,6 +15,7 @@ import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ShapefileReader; +import com.google.common.base.Preconditions; import com.google.common.io.Files; import org.apache.commons.fileupload.FileItem; import org.json.simple.JSONObject; @@ -63,6 +65,7 @@ public class AggregationAreaController implements HttpController { private final FileStorage fileStorage; private final TaskScheduler taskScheduler; private final AnalysisCollection aggregationAreaCollection; + private final AnalysisCollection spatialSourceCollection; public AggregationAreaController ( FileStorage fileStorage, @@ -72,6 +75,7 @@ public AggregationAreaController ( this.fileStorage = fileStorage; this.taskScheduler = taskScheduler; this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); + this.spatialSourceCollection = database.getAnalysisCollection("spatialSources", SpatialDatasetSource.class); } private FileStorageKey getStoragePath (AggregationArea area) { @@ -91,45 +95,16 @@ private List createAggregationAreas (Request req, Response res) Map> query = HttpUtils.getRequestFiles(req.raw()); UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); String maskName = query.get("name").get(0).getString("UTF-8"); - String regionId = req.params("regionId"); + String nameProperty = query.get("nameProperty") == null ? null : query.get("nameProperty").get(0).getString( + "UTF-8"); + String sourceId = query.get("sourceId").get(0).getString("UTF-8"); - SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, maskName) - .withRegion(regionId); + // 1. Get shapefile from storage and read its features. ======================================================== + SpatialDatasetSource source = (SpatialDatasetSource) spatialSourceCollection.findById(sourceId); + Preconditions.checkArgument(source.geometryWrapper instanceof Polygons, "Only polygons can be converted to " + + "aggregation areas."); + File shpFile = fileStorage.getFile(source.storageKey()); - // 1. Extract relevant files: .shp, .prj, .dbf, and .shx. ====================================================== - Map filesByName = query.get("files").stream() - .collect(Collectors.toMap(FileItem::getName, f -> f)); - - String fileName = filesByName.keySet().stream().filter(f -> f.endsWith(".shp")).findAny().orElse(null); - if (fileName == null) { - throw AnalysisServerException.fileUpload("Shapefile upload must contain .shp, .prj, and .dbf"); - } - String baseName = fileName.substring(0, fileName.length() - 4); - - if (!filesByName.containsKey(baseName + ".shp") || - !filesByName.containsKey(baseName + ".prj") || - !filesByName.containsKey(baseName + ".dbf")) { - throw AnalysisServerException.fileUpload("Shapefile upload must contain .shp, .prj, and .dbf"); - } - - File tempDir = Files.createTempDir(); - - File shpFile = new File(tempDir, "grid.shp"); - filesByName.get(baseName + ".shp").write(shpFile); - - File prjFile = new File(tempDir, "grid.prj"); - filesByName.get(baseName + ".prj").write(prjFile); - - File dbfFile = new File(tempDir, "grid.dbf"); - filesByName.get(baseName + ".dbf").write(dbfFile); - - // shx is optional, not needed for dense shapefiles - if (filesByName.containsKey(baseName + ".shx")) { - File shxFile = new File(tempDir, "grid.shx"); - filesByName.get(baseName + ".shx").write(shxFile); - } - - // 2. Read features ============================================================================================ ShapefileReader reader = null; List features; try { @@ -141,16 +116,15 @@ private List createAggregationAreas (Request req, Response res) Map areas = new HashMap<>(); - boolean unionRequested = Boolean.parseBoolean(query.get("union").get(0).getString()); String zoomString = query.get("zoom") == null ? null : query.get("zoom").get(0).getString(); final int zoom = parseZoom(zoomString); - if (!unionRequested && features.size() > MAX_FEATURES) { + if (nameProperty != null && features.size() > MAX_FEATURES) { throw AnalysisServerException.fileUpload(MessageFormat.format("The uploaded shapefile has {0} features, " + "which exceeds the limit of {1}", features.size(), MAX_FEATURES)); } - if (unionRequested) { + if (nameProperty == null) { // Union (single combined aggregation area) requested List geometries = features.stream().map(f -> (Geometry) f.getDefaultGeometry()).collect(Collectors.toList()); UnaryUnionOp union = new UnaryUnionOp(geometries); @@ -158,7 +132,6 @@ private List createAggregationAreas (Request req, Response res) areas.put(maskName, union.union()); } else { // Don't union. Name each area by looking up its value for the name property in the request. - String nameProperty = query.get("nameProperty").get(0).getString("UTF-8"); features.forEach(f -> areas.put(readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry())); } @@ -169,7 +142,7 @@ private List createAggregationAreas (Request req, Response res) // TODO move below into .withAction() ); - // 3. Convert to raster grids, then store them. ================================================================ + // 2. Convert to raster grids, then store them. ================================================================ areas.forEach((String name, Geometry geometry) -> { if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); Envelope env = geometry.getEnvelopeInternal(); @@ -188,7 +161,6 @@ private List createAggregationAreas (Request req, Response res) maskGrid.write(os); os.close(); - // Create the aggregation area before generating the S3 key so that the `_id` is generated aggregationAreaCollection.insert(aggregationArea); aggregationAreas.add(aggregationArea); @@ -197,7 +169,6 @@ private List createAggregationAreas (Request req, Response res) throw new AnalysisServerException("Error processing/uploading aggregation area"); } - tempDir.delete(); }); return aggregationAreas; diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 8698a4606..8fcb188c4 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -21,17 +21,14 @@ import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ExceptionUtils; import com.conveyal.r5.util.InputStreamProvider; -import com.conveyal.r5.util.ShapefileReader; -import com.google.common.io.Files; import com.mongodb.QueryBuilder; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.ServletFileUpload; -import org.apache.commons.io.FilenameUtils; import org.json.simple.JSONObject; -import org.locationtech.jts.geom.Envelope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -46,8 +43,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -57,8 +54,9 @@ import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.conveyal.r5.analyst.progress.WorkProductType.SPATIAL_DATASET_SOURCE; /** * Controller that handles fetching opportunity datasets (grids and other pointset formats). @@ -72,7 +70,7 @@ public class SpatialDatasetController implements HttpController { // Component Dependencies private final FileStorage fileStorage; - private final AnalysisCollection spatialSourceCollection; + private final AnalysisCollection spatialSourceCollection; private final TaskScheduler taskScheduler; private final SeamlessCensusGridExtractor extractor; @@ -300,43 +298,41 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { final String sourceName = getFormField(formFields, "Name", true); final String regionId = getFormField(formFields, "regionId", true); - // TODO tracking - - // TODO move to storage - + // Initialize model object SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName).withRegion(regionId); - final List fileItems; - final SourceFormat uploadFormat; - try { - // Validate inputs and parameters, which will throw an exception if there's anything wrong with them. - // Call remove() rather than get() so that subsequent code will see only string parameters, not the files. - fileItems = formFields.remove("files"); - uploadFormat = detectUploadFormatAndValidate(fileItems); - - if (uploadFormat == SourceFormat.GRID) { - // TODO source.fromGrids(fileItems); - } else if (uploadFormat == SourceFormat.SHAPEFILE) { - source.fromShapefile(fileItems); - } else if (uploadFormat == SourceFormat.CSV) { - // TODO source.fromCsv(fileItems); - } else if (uploadFormat == SourceFormat.GEOJSON) { - // TODO source.fromGeojson(fileItems); - } - - spatialSourceCollection.insert(source); - - } catch (Exception e) { - // TODO tracking - throw new AnalysisServerException("Problem reading files"); - } - - LOG.info("Handling uploaded {} file", uploadFormat); - - + taskScheduler.enqueue(Task.create("Storing " + sourceName) + .forUser(userPermissions) + .withWorkProduct(SPATIAL_DATASET_SOURCE, source._id.toString(), regionId) + .withAction(progressListener -> { + + // Loop through uploaded files, registering the extensions and writing to storage (with filenames that + // correspond to the source id) + List files = new ArrayList<>(); + final List fileItems = formFields.remove("files"); + for (FileItem fileItem : fileItems) { + File file = ((DiskFileItem) fileItem).getStoreLocation(); + String filename = file.getName(); + String extension = filename.substring(filename.lastIndexOf(".") + 1).toUpperCase(Locale.ROOT); + FileStorageKey key = new FileStorageKey(RESOURCES, source._id.toString(), extension); + fileStorage.moveIntoStorage(key, file); + files.add(fileStorage.getFile(key)); + } + progressListener.beginTask("Detecting format", 1); + final SourceFormat uploadFormat; + try { + // Validate inputs, which will throw an exception if there's anything wrong with them. + uploadFormat = detectUploadFormatAndValidate(fileItems); + LOG.info("Handling uploaded {} file", uploadFormat); + } catch (Exception e) { + throw AnalysisServerException.fileUpload("Problem reading uploaded spatial files" + e.getMessage()); + } + progressListener.beginTask("Validating files", 1); + source.validateAndSetDetails(uploadFormat, files); + spatialSourceCollection.insert(source); + })); return source; - } private OpportunityDataset editOpportunityDataset(Request request, Response response) throws IOException { @@ -486,7 +482,7 @@ private Object downloadOpportunityDataset (Request req, Response res) throws IOE @Override public void registerEndpoints (spark.Service sparkService) { - sparkService.path("/api/opportunities", () -> { + sparkService.path("/api/spatial", () -> { sparkService.post("", this::handleUpload, toJson); sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index 2b1ac8c5b..cfcfe7105 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -1,11 +1,13 @@ package com.conveyal.analysis.models; +import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.UserPermissions; -import com.conveyal.analysis.spatial.SpatialDataset.GeometryType; -import com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; +import com.conveyal.analysis.spatial.GeometryWrapper; +import com.conveyal.analysis.spatial.SpatialAttribute; +import com.conveyal.analysis.spatial.SpatialDataset; +import com.conveyal.file.FileStorageKey; import com.conveyal.r5.util.ShapefileReader; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.io.Files; import org.apache.commons.fileupload.FileItem; import org.apache.commons.io.FilenameUtils; import org.locationtech.jts.geom.Envelope; @@ -15,16 +17,19 @@ import java.util.List; import java.util.Map; +import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat.*; +import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; public class SpatialDatasetSource extends BaseModel { - public List files; public String regionId; + /** Description editable by end users */ public String description; - public SourceFormat sourceFormat; - public GeometryType geometryType; - public Map attributes; // TODO map to both type and user-modifiable label? - public int featureCount; + public SpatialDataset.SourceFormat sourceFormat; + /** General geometry type, with associated static methods for conversion to spatial datasets */ + public GeometryWrapper geometryWrapper; + /** Attributes, set only after validation (e.g. appropriate format for each feature's attributes) */ + public List attributes; private SpatialDatasetSource (UserPermissions userPermissions, String sourceName) { super(userPermissions, sourceName); @@ -42,41 +47,38 @@ public SpatialDatasetSource withRegion (String regionId) { } @JsonIgnore - public SpatialDatasetSource fromShapefile (List fileItems) throws Exception { + public void validateAndSetDetails (SpatialDataset.SourceFormat uploadFormat, List files) { + this.sourceFormat = uploadFormat; + if (uploadFormat == GRID) { + // TODO source.fromGrids(fileItems); + } else if (uploadFormat == SHAPEFILE) { + this.fromShapefile(files); + } else if (uploadFormat == CSV) { + // TODO source.fromCsv(fileItems); + } else if (uploadFormat == GEOJSON) { + // TODO source.fromGeojson(fileItems); + } + } + + private void fromShapefile (List files) { // In the caller, we should have already verified that all files have the same base name and have an extension. // Extract the relevant files: .shp, .prj, .dbf, and .shx. // We need the SHX even though we're looping over every feature as they might be sparse. - Map filesByExtension = new HashMap<>(); - for (FileItem fileItem : fileItems) { - filesByExtension.put(FilenameUtils.getExtension(fileItem.getName()).toUpperCase(), fileItem); + Map filesByExtension = new HashMap<>(); + for (File file : files) { + filesByExtension.put(FilenameUtils.getExtension(file.getName()).toUpperCase(), file); } - // Copy the shapefile component files into a temporary directory with a fixed base name. - File tempDir = Files.createTempDir(); - - File shpFile = new File(tempDir, "grid.shp"); - filesByExtension.get("SHP").write(shpFile); - - File prjFile = new File(tempDir, "grid.prj"); - filesByExtension.get("PRJ").write(prjFile); - - File dbfFile = new File(tempDir, "grid.dbf"); - filesByExtension.get("DBF").write(dbfFile); - - // The .shx file is an index. It is optional, and not needed for dense shapefiles. - if (filesByExtension.containsKey("SHX")) { - File shxFile = new File(tempDir, "grid.shx"); - filesByExtension.get("SHX").write(shxFile); + try { + ShapefileReader reader = new ShapefileReader(filesByExtension.get("SHP")); + Envelope envelope = reader.wgs84Bounds(); + checkWgsEnvelopeSize(envelope); + this.attributes = reader.getAttributes(); + this.geometryWrapper = reader.getGeometryType(); + } catch (Exception e) { + throw AnalysisServerException.fileUpload("Shapefile transform error. Try uploading an unprojected " + + "(EPSG:4326) file." + e.getMessage()); } - - ShapefileReader reader = new ShapefileReader(shpFile); - Envelope envelope = reader.wgs84Bounds(); - checkWgsEnvelopeSize(envelope); - - this.attributes = reader.getAttributeTypes(); - this.sourceFormat = SourceFormat.SHAPEFILE; - // TODO this.geometryType = - return this; } @JsonIgnore @@ -85,4 +87,9 @@ public SpatialDatasetSource fromFiles (List fileItemList) { return this; } + @JsonIgnore + public FileStorageKey storageKey() { + return new FileStorageKey(RESOURCES, this._id.toString(), sourceFormat.toString()); + } + } diff --git a/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java b/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java new file mode 100644 index 000000000..14f6d9343 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java @@ -0,0 +1,10 @@ +package com.conveyal.analysis.spatial; + +public abstract class GeometryWrapper { + int featureCount; + + public GeometryWrapper (int featureCount) { + this.featureCount = featureCount; + } + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/Lines.java b/src/main/java/com/conveyal/analysis/spatial/Lines.java new file mode 100644 index 000000000..a49de6da6 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/Lines.java @@ -0,0 +1,11 @@ +package com.conveyal.analysis.spatial; + +public class Lines extends GeometryWrapper { + + public Lines (int featureCount) { + super(featureCount); + } + + // TODO transit alignment modification + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/Points.java b/src/main/java/com/conveyal/analysis/spatial/Points.java new file mode 100644 index 000000000..9f9d0ba4a --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/Points.java @@ -0,0 +1,19 @@ +package com.conveyal.analysis.spatial; + +import com.conveyal.analysis.models.FileInfo; + +public class Points extends GeometryWrapper { + + public Points (int featureCount) { + super(featureCount); + } + + public static void toFreeform (FileInfo source) { + // TODO implement + } + + public static void toGrid (FileInfo source) { + // TODO implement + } + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/Polygons.java b/src/main/java/com/conveyal/analysis/spatial/Polygons.java new file mode 100644 index 000000000..00b2ca514 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/Polygons.java @@ -0,0 +1,27 @@ +package com.conveyal.analysis.spatial; + +import com.conveyal.analysis.models.AggregationArea; +import com.conveyal.r5.analyst.progress.Task; + +import java.io.File; +import java.util.ArrayList; + +public class Polygons extends GeometryWrapper { + + public Polygons (int featureCount) { + super(featureCount); + } + + public static ArrayList toAggregationAreas (File file, SpatialDataset.SourceFormat sourceFormat, + Task progressListener) { + ArrayList aggregationAreas = new ArrayList<>(); + // TODO from shapefile + // TODO from geojson + return aggregationAreas; + } + + // TODO toGrid from shapefile and geojson + + // TODO modification polygon + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java new file mode 100644 index 000000000..2a2ed46d8 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java @@ -0,0 +1,29 @@ +package com.conveyal.analysis.spatial; + +import org.opengis.feature.type.AttributeType; + +/** Groups the original names and user-friendly fields from shapefile attributes, CSV columns, etc. */ +public class SpatialAttribute { + /** Name in source file */ + public final String name; + + /** Editable by end users */ + String label; + + Type type; + + enum Type { + NUMBER, // internally, we generally parse as doubles + TEXT, + ERROR + } + + public SpatialAttribute(String name, AttributeType type) { + this.name = name; + this.label = name; + if (Number.class.equals(type.getBinding())) this.type = Type.NUMBER; + else if (String.class.equals(type.getBinding())) this.type = Type.TEXT; + else this.type = Type.ERROR; + } + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java index f0cb2b9e1..3245e590d 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.spatial; import com.conveyal.analysis.AnalysisServerException; + import com.google.common.collect.Sets; import org.apache.commons.fileupload.FileItem; import org.apache.commons.io.FilenameUtils; @@ -14,14 +15,11 @@ */ public class SpatialDataset { + // TODO merge with FileCategory? public enum SourceFormat { SHAPEFILE, CSV, GEOJSON, GRID, SEAMLESS } - public enum GeometryType { - LINE, POLYGON, POINT, GRID - } - /** * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary * grids. In the process we validate the list of uploaded files, making sure certain preconditions are met. @@ -37,6 +35,12 @@ public static SourceFormat detectUploadFormatAndValidate (List fileIte Set fileExtensions = extractFileExtensions(fileItems); + if (fileExtensions.contains("ZIP")) { + throw AnalysisServerException.fileUpload("Upload of spatial .zip files not yet supported"); + // TODO unzip + // detectUploadFormatAndValidate(unzipped) + } + // There was at least one file with an extension, the set must now contain at least one extension. if (fileExtensions.isEmpty()) { throw AnalysisServerException.fileUpload("No file extensions seen, cannot detect upload type."); diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index 90cd8826d..bf5d9666c 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -6,12 +6,14 @@ public enum FileStorageFormat { POINTSET("pointset", "application/octet-stream"), PNG("png", "image/png"), TIFF("tiff", "image/tiff"), - CSV("csv", "text/csv"); + CSV("csv", "text/csv"), // These are not currently used but plan to be in the future. Exact types need to be determined // GTFS("zip", "application/zip"), // PBF("pbf", "application/octet-stream"), - // SHP("shp", "application/octet-stream") // This type does not work as is, it should be a zip? + + // SHP implies .dbf and .prj, and optionally .shx + SHP("shp", "application/octet-stream"); public final String extension; public final String mimeType; diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index d9302470b..ab3ad7349 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -1,6 +1,11 @@ package com.conveyal.r5.util; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.spatial.GeometryWrapper; +import com.conveyal.analysis.spatial.Lines; +import com.conveyal.analysis.spatial.Points; +import com.conveyal.analysis.spatial.Polygons; +import com.conveyal.analysis.spatial.SpatialAttribute; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; import org.geotools.data.FeatureSource; @@ -12,7 +17,9 @@ import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.Lineal; +import org.locationtech.jts.geom.Polygonal; +import org.locationtech.jts.geom.Puntal; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.AttributeDescriptor; @@ -25,6 +32,7 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -114,7 +122,7 @@ public double getAreaSqKm () throws IOException, TransformException, FactoryExce public Stream wgs84Stream () throws IOException, TransformException { return stream().map(f -> { - Geometry g = (Geometry) f.getDefaultGeometry(); + org.locationtech.jts.geom.Geometry g = (org.locationtech.jts.geom.Geometry) f.getDefaultGeometry(); try { // TODO does this leak beyond this function? f.setDefaultGeometry(JTS.transform(g, transform)); @@ -141,21 +149,28 @@ public int getFeatureCount() throws IOException { return source.getCount(Query.ALL); } - public Map getAttributeTypes () { - Map attributes = new HashMap<>(); + public List getAttributes () { + List attributes = new ArrayList<>(); HashSet uniqueAttributes = new HashSet<>(); features.getSchema() .getAttributeDescriptors() .forEach(d -> { String attributeName = d.getLocalName(); AttributeType type = d.getType().getSuper(); - attributes.put(attributeName, type.getBinding()); + attributes.add(new SpatialAttribute(attributeName, type)); uniqueAttributes.add(attributeName); }); if (attributes.size() != uniqueAttributes.size()) { - throw new AnalysisServerException("Shapefile has duplicate " + - "attributes."); + throw new AnalysisServerException("Shapefile has duplicate attributes."); } - return attributes; + return attributes; + } + + public GeometryWrapper getGeometryType () { + Class geometryType = features.getSchema().getGeometryDescriptor().getType().getBinding(); + if (Polygonal.class.isAssignableFrom(geometryType)) return new Polygons(features.size()); + if (Puntal.class.isAssignableFrom(geometryType)) return new Points(features.size()); + if (Lineal.class.isAssignableFrom(geometryType)) return new Lines(features.size()); + else return null; } } From a2cae873791d044948ae6408e96831620e715e2b Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 14 Jun 2021 08:48:05 -0400 Subject: [PATCH 030/187] fix(spatial): ignore correct extension Change originally implemented in #731 --- .../java/com/conveyal/analysis/spatial/SpatialDataset.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java index 3245e590d..6e6418f12 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java @@ -103,8 +103,8 @@ private static Set extractFileExtensions (List fileItems) { private static void verifyBaseNamesSame (List fileItems) { String firstBaseName = null; for (FileItem fileItem : fileItems) { - // Ignore .shp.xml files - if (FilenameUtils.getExtension(fileItem.getName()).equals(".xml")) continue; + // Ignore .shp.xml files, which will fail the verifyBaseNamesSame check + if ("xml".equalsIgnoreCase(FilenameUtils.getExtension(fileItem.getName()))) continue; String baseName = FilenameUtils.getBaseName(fileItem.getName()); if (firstBaseName == null) { firstBaseName = baseName; From 0ca4700da0b4edf19e1620d2cac9f22db43377a4 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 22 Jun 2021 00:25:46 -0400 Subject: [PATCH 031/187] feat(spatial): handle uploaded files --- .../controllers/SpatialDatasetController.java | 11 ++++++----- .../analysis/models/SpatialDatasetSource.java | 8 +++++++- .../conveyal/analysis/spatial/GeometryWrapper.java | 4 ++-- .../conveyal/analysis/spatial/SpatialAttribute.java | 7 +++++-- .../java/com/conveyal/r5/util/ShapefileReader.java | 8 +++++--- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 8fcb188c4..62395a61a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -295,7 +295,7 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { } // Parse required fields. Will throw a ServerException on failure. - final String sourceName = getFormField(formFields, "Name", true); + final String sourceName = getFormField(formFields, "sourceName", true); final String regionId = getFormField(formFields, "regionId", true); // Initialize model object @@ -309,13 +309,14 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { // Loop through uploaded files, registering the extensions and writing to storage (with filenames that // correspond to the source id) List files = new ArrayList<>(); - final List fileItems = formFields.remove("files"); + final List fileItems = formFields.remove("sourceFiles"); for (FileItem fileItem : fileItems) { - File file = ((DiskFileItem) fileItem).getStoreLocation(); - String filename = file.getName(); + String filename = fileItem.getName(); String extension = filename.substring(filename.lastIndexOf(".") + 1).toUpperCase(Locale.ROOT); FileStorageKey key = new FileStorageKey(RESOURCES, source._id.toString(), extension); - fileStorage.moveIntoStorage(key, file); + // FIXME writing not allowed by fileStorage.getFile contract + // FIXME Investigate fileStorage.moveIntoStorage(key, file), for consistency with BundleController; + fileItem.write(fileStorage.getFile(key)); files.add(fileStorage.getFile(key)); } diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index cfcfe7105..05a4c9c0f 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -11,8 +11,11 @@ import org.apache.commons.fileupload.FileItem; import org.apache.commons.io.FilenameUtils; import org.locationtech.jts.geom.Envelope; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -75,7 +78,10 @@ private void fromShapefile (List files) { checkWgsEnvelopeSize(envelope); this.attributes = reader.getAttributes(); this.geometryWrapper = reader.getGeometryType(); - } catch (Exception e) { + } catch (IOException e) { + throw AnalysisServerException.fileUpload("Shapefile parsing error. Ensure the files you are trying to " + + "upload are valid."); + } catch (FactoryException | TransformException e) { throw AnalysisServerException.fileUpload("Shapefile transform error. Try uploading an unprojected " + "(EPSG:4326) file." + e.getMessage()); } diff --git a/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java b/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java index 14f6d9343..024be83c1 100644 --- a/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java +++ b/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java @@ -1,9 +1,9 @@ package com.conveyal.analysis.spatial; public abstract class GeometryWrapper { - int featureCount; + public int featureCount; - public GeometryWrapper (int featureCount) { + GeometryWrapper (int featureCount) { this.featureCount = featureCount; } diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java index 2a2ed46d8..6aa01f276 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.spatial; +import org.locationtech.jts.geom.Geometry; import org.opengis.feature.type.AttributeType; /** Groups the original names and user-friendly fields from shapefile attributes, CSV columns, etc. */ @@ -15,14 +16,16 @@ public class SpatialAttribute { enum Type { NUMBER, // internally, we generally parse as doubles TEXT, + GEOM, ERROR } public SpatialAttribute(String name, AttributeType type) { this.name = name; this.label = name; - if (Number.class.equals(type.getBinding())) this.type = Type.NUMBER; - else if (String.class.equals(type.getBinding())) this.type = Type.TEXT; + if (Number.class.isAssignableFrom(type.getBinding())) this.type = Type.NUMBER; + else if (String.class.isAssignableFrom(type.getBinding())) this.type = Type.TEXT; + else if (Geometry.class.isAssignableFrom(type.getBinding())) this.type = Type.GEOM; else this.type = Type.ERROR; } diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index ab3ad7349..f96965732 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -156,9 +156,11 @@ public List getAttributes () { .getAttributeDescriptors() .forEach(d -> { String attributeName = d.getLocalName(); - AttributeType type = d.getType().getSuper(); - attributes.add(new SpatialAttribute(attributeName, type)); - uniqueAttributes.add(attributeName); + AttributeType type = d.getType(); + if (type != null) { + attributes.add(new SpatialAttribute(attributeName, type)); + uniqueAttributes.add(attributeName); + } }); if (attributes.size() != uniqueAttributes.size()) { throw new AnalysisServerException("Shapefile has duplicate attributes."); From b50c3b60f54fb40934c2257fe7c47305a3326e90 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 22 Jun 2021 00:25:59 -0400 Subject: [PATCH 032/187] feat(spatial): register controller --- .../com/conveyal/analysis/components/BackendComponents.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index d711c64da..65891e859 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -14,6 +14,7 @@ import com.conveyal.analysis.controllers.OpportunityDatasetController; import com.conveyal.analysis.controllers.ProjectController; import com.conveyal.analysis.controllers.RegionalAnalysisController; +import com.conveyal.analysis.controllers.SpatialDatasetController; import com.conveyal.analysis.controllers.TimetableController; import com.conveyal.analysis.controllers.UserActivityController; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; @@ -102,7 +103,8 @@ public List standardHttpControllers () { // InternalHttpApi component with its own spark service, renaming this ExternalHttpApi. new BrokerController(broker, eventBus), new UserActivityController(taskScheduler), - new GtfsTileController(gtfsCache) + new GtfsTileController(gtfsCache), + new SpatialDatasetController(fileStorage, database, taskScheduler, censusExtractor) ); } From d44d6a0523888c2ca7b051cc4e9412890a22a9f8 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 22 Jun 2021 18:04:49 +0800 Subject: [PATCH 033/187] Add GTFS controller --- .../components/BackendComponents.java | 2 + .../analysis/controllers/GTFSController.java | 173 ++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 src/main/java/com/conveyal/analysis/controllers/GTFSController.java diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 197f46b22..16bfd1ac7 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -7,6 +7,7 @@ import com.conveyal.analysis.controllers.BrokerController; import com.conveyal.analysis.controllers.BundleController; import com.conveyal.analysis.controllers.FileStorageController; +import com.conveyal.analysis.controllers.GTFSController; import com.conveyal.analysis.controllers.GTFSGraphQLController; import com.conveyal.analysis.controllers.GtfsTileController; import com.conveyal.analysis.controllers.HttpController; @@ -89,6 +90,7 @@ public List standardHttpControllers () { new ModificationController(), new ProjectController(), new GTFSGraphQLController(gtfsCache), + new GTFSController(gtfsCache), new BundleController(this), new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor), new RegionalAnalysisController(broker, fileStorage), diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java new file mode 100644 index 000000000..c44cd53db --- /dev/null +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -0,0 +1,173 @@ +package com.conveyal.analysis.controllers; + +import com.conveyal.analysis.models.Bundle; +import com.conveyal.analysis.persistence.Persistence; +import com.conveyal.gtfs.GTFSCache; +import com.conveyal.gtfs.GTFSFeed; +import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; +import com.conveyal.gtfs.model.Pattern; +import com.conveyal.gtfs.model.Route; +import com.conveyal.gtfs.model.Stop; +import com.conveyal.gtfs.model.StopTime; +import com.conveyal.gtfs.model.Trip; +import org.mapdb.Fun; +import spark.Request; +import spark.Response; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.conveyal.analysis.util.JsonUtil.toJson; + +public class GTFSController implements HttpController { + private final GTFSCache gtfsCache; + public GTFSController (GTFSCache gtfsCache) { + this.gtfsCache = gtfsCache; + } + + private GTFSFeed getFeedFromRequest (Request req) { + Bundle bundle = Persistence.bundles.findByIdFromRequestIfPermitted(req); + String bundleScopedFeedId = Bundle.bundleScopeFeedId(req.params("feedId"), bundle.feedGroupId); + return gtfsCache.get(bundleScopedFeedId); + } + + static class RouteAPIResponse { + public final String id; + public final String name; + public final int type; + public final String color; + + RouteAPIResponse(Route route) { + id = route.route_id; + color = route.route_color; + name = String.join(" ", route.route_short_name + "", route.route_long_name + "").trim(); + type = route.route_type; + } + } + + private List getRoutes(Request req, Response res) { + GTFSFeed feed = getFeedFromRequest(req); + return feed.routes + .values() + .stream() + .map(RouteAPIResponse::new) + .collect(Collectors.toList()); + } + + static class PatternAPIResponse { + public final String id; + public final com.vividsolutions.jts.geom.LineString geometry; + public final List orderedStopIds; + public final List associatedTripIds; + PatternAPIResponse(Pattern pattern) { + id = pattern.pattern_id; + geometry = pattern.geometry; + orderedStopIds = pattern.orderedStops; + associatedTripIds = pattern.associatedTrips; + } + } + + private List getPatternsForRoute (Request req, Response res) { + GTFSFeed feed = getFeedFromRequest(req); + final String routeId = req.params("routeId"); + return feed.patterns + .values() + .stream() + .filter(p -> Objects.equals(p.route_id, routeId)) + .map(PatternAPIResponse::new) + .collect(Collectors.toList()); + } + + static class StopAPIResponse { + public final String id; + public final String name; + public final double lat; + public final double lon; + + StopAPIResponse(Stop stop) { + id = stop.stop_id; + name = stop.stop_name; + lat = stop.stop_lat; + lon = stop.stop_lon; + } + } + + private List getStops (Request req, Response res) { + GTFSFeed feed = getFeedFromRequest(req); + return feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); + } + + static class FeedStopsAPIResponse { + public final String feedId; + public final List stops; + + FeedStopsAPIResponse(String feedId, List stops) { + this.feedId = feedId; + this.stops = stops; + } + } + + private List getBundleStops (Request req, Response res) { + final Bundle bundle = Persistence.bundles.findByIdFromRequestIfPermitted(req); + return bundle.feeds.stream().map(f -> { + String bundleScopedFeedId = Bundle.bundleScopeFeedId(f.feedId, bundle.feedGroupId); + GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); + List stops = feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); + return new FeedStopsAPIResponse( + f.feedId, + stops + ); + }).collect(Collectors.toList()); + } + + static class TripAPIResponse { + public final String id; + public final String name; + public final String headsign; + public final Integer startTime; + public final Integer duration; + public final int directionId; + + TripAPIResponse(GTFSFeed feed, Trip trip) { + id = trip.trip_id; + name = trip.trip_short_name; + headsign = trip.trip_headsign; + directionId = trip.direction_id; + + Map.Entry st = feed.stop_times.ceilingEntry(new Fun.Tuple2(trip.trip_id, null)); + Map.Entry endStopTime = feed.stop_times.floorEntry(new Fun.Tuple2(trip.trip_id, Fun.HI)); + + startTime = st != null ? st.getValue().departure_time : null; + + if (startTime == null || endStopTime == null || endStopTime.getValue().arrival_time < startTime) { + duration = null; + } else { + duration = endStopTime.getValue().arrival_time - startTime; + } + } + } + + private List getTripsForRoute (Request req, Response res) { + final GTFSFeed feed = getFeedFromRequest(req); + final String routeId = req.params("routeId"); + return feed.trips + .values().stream() + .filter(t -> Objects.equals(t.route_id, routeId)) + .map(t -> new TripAPIResponse(feed, t)) + .sorted(Comparator.comparingInt(t -> t.startTime)) + .collect(Collectors.toList()); + } + + @Override + public void registerEndpoints (spark.Service sparkService) { + sparkService.get("/api/gtfs/:_id/stops", this::getBundleStops, toJson); + sparkService.get("/api/gtfs/:_id/:feedId/routes", this::getRoutes, toJson); + sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/patterns", this::getPatternsForRoute, toJson); + sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/trips", this::getTripsForRoute, toJson); + sparkService.get("/api/gtfs/:_id/:feedId/stops", this::getStops, toJson); + } +} From 4b8e3731dfe3e285dbc1ed4d1b111f5762fdacb2 Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:07:22 -0400 Subject: [PATCH 034/187] refactor(spatial): clean up feature summary using enums instead of subtyping --- .../AggregationAreaController.java | 5 ++- .../analysis/models/SpatialDatasetSource.java | 6 ++-- .../analysis/spatial/FeatureSummary.java | 35 +++++++++++++++++++ .../analysis/spatial/GeometryWrapper.java | 10 ------ .../com/conveyal/analysis/spatial/Lines.java | 6 +--- .../com/conveyal/analysis/spatial/Points.java | 6 +--- .../conveyal/analysis/spatial/Polygons.java | 9 ++--- .../com/conveyal/r5/util/ShapefileReader.java | 16 ++------- 8 files changed, 48 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java delete mode 100644 src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 51a01f13e..0718a0889 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -7,7 +7,6 @@ import com.conveyal.analysis.models.SpatialDatasetSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; -import com.conveyal.analysis.spatial.Polygons; import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageKey; @@ -16,7 +15,6 @@ import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ShapefileReader; import com.google.common.base.Preconditions; -import com.google.common.io.Files; import org.apache.commons.fileupload.FileItem; import org.json.simple.JSONObject; import org.locationtech.jts.geom.Envelope; @@ -42,6 +40,7 @@ import static com.conveyal.analysis.components.HttpApi.USER_GROUP_ATTRIBUTE; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; +import static com.conveyal.analysis.spatial.FeatureSummary.Type.POLYGON; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; @@ -101,7 +100,7 @@ private List createAggregationAreas (Request req, Response res) // 1. Get shapefile from storage and read its features. ======================================================== SpatialDatasetSource source = (SpatialDatasetSource) spatialSourceCollection.findById(sourceId); - Preconditions.checkArgument(source.geometryWrapper instanceof Polygons, "Only polygons can be converted to " + + Preconditions.checkArgument(POLYGON.equals(source.features.type), "Only polygons can be converted to " + "aggregation areas."); File shpFile = fileStorage.getFile(source.storageKey()); diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index 05a4c9c0f..5a4b47684 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -2,7 +2,7 @@ import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.UserPermissions; -import com.conveyal.analysis.spatial.GeometryWrapper; +import com.conveyal.analysis.spatial.FeatureSummary; import com.conveyal.analysis.spatial.SpatialAttribute; import com.conveyal.analysis.spatial.SpatialDataset; import com.conveyal.file.FileStorageKey; @@ -30,7 +30,7 @@ public class SpatialDatasetSource extends BaseModel { public String description; public SpatialDataset.SourceFormat sourceFormat; /** General geometry type, with associated static methods for conversion to spatial datasets */ - public GeometryWrapper geometryWrapper; + public FeatureSummary features; /** Attributes, set only after validation (e.g. appropriate format for each feature's attributes) */ public List attributes; @@ -77,7 +77,7 @@ private void fromShapefile (List files) { Envelope envelope = reader.wgs84Bounds(); checkWgsEnvelopeSize(envelope); this.attributes = reader.getAttributes(); - this.geometryWrapper = reader.getGeometryType(); + this.features = reader.featureSummary(); } catch (IOException e) { throw AnalysisServerException.fileUpload("Shapefile parsing error. Ensure the files you are trying to " + "upload are valid."); diff --git a/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java b/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java new file mode 100644 index 000000000..afe4e19f5 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java @@ -0,0 +1,35 @@ +package com.conveyal.analysis.spatial; + +import org.geotools.feature.FeatureCollection; +import org.locationtech.jts.geom.Lineal; +import org.locationtech.jts.geom.Polygonal; +import org.locationtech.jts.geom.Puntal; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.feature.simple.SimpleFeatureType; + +public class FeatureSummary { + public int count; + public Type type; + + public enum Type { + POLYGON, + POINT, + LINE; + } + + public FeatureSummary (FeatureCollection features) { + Class geometryType = features.getSchema().getGeometryDescriptor().getType().getBinding(); + if (Polygonal.class.isAssignableFrom(geometryType)) this.type = Type.POLYGON; + if (Puntal.class.isAssignableFrom(geometryType)) this.type = Type.POINT; + if (Lineal.class.isAssignableFrom(geometryType)) this.type = Type.LINE; + // TODO throw exception if geometryType is not one of the above + this.count = features.size(); + } + + /** + * No-arg constructor for Mongo serialization + */ + public FeatureSummary () { + } + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java b/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java deleted file mode 100644 index 024be83c1..000000000 --- a/src/main/java/com/conveyal/analysis/spatial/GeometryWrapper.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.conveyal.analysis.spatial; - -public abstract class GeometryWrapper { - public int featureCount; - - GeometryWrapper (int featureCount) { - this.featureCount = featureCount; - } - -} diff --git a/src/main/java/com/conveyal/analysis/spatial/Lines.java b/src/main/java/com/conveyal/analysis/spatial/Lines.java index a49de6da6..beaba4248 100644 --- a/src/main/java/com/conveyal/analysis/spatial/Lines.java +++ b/src/main/java/com/conveyal/analysis/spatial/Lines.java @@ -1,10 +1,6 @@ package com.conveyal.analysis.spatial; -public class Lines extends GeometryWrapper { - - public Lines (int featureCount) { - super(featureCount); - } +public class Lines { // TODO transit alignment modification diff --git a/src/main/java/com/conveyal/analysis/spatial/Points.java b/src/main/java/com/conveyal/analysis/spatial/Points.java index 9f9d0ba4a..0abbb9d94 100644 --- a/src/main/java/com/conveyal/analysis/spatial/Points.java +++ b/src/main/java/com/conveyal/analysis/spatial/Points.java @@ -2,11 +2,7 @@ import com.conveyal.analysis.models.FileInfo; -public class Points extends GeometryWrapper { - - public Points (int featureCount) { - super(featureCount); - } +public class Points { public static void toFreeform (FileInfo source) { // TODO implement diff --git a/src/main/java/com/conveyal/analysis/spatial/Polygons.java b/src/main/java/com/conveyal/analysis/spatial/Polygons.java index 00b2ca514..b61fc044c 100644 --- a/src/main/java/com/conveyal/analysis/spatial/Polygons.java +++ b/src/main/java/com/conveyal/analysis/spatial/Polygons.java @@ -1,18 +1,15 @@ package com.conveyal.analysis.spatial; import com.conveyal.analysis.models.AggregationArea; +import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.Task; import java.io.File; import java.util.ArrayList; -public class Polygons extends GeometryWrapper { +public class Polygons { - public Polygons (int featureCount) { - super(featureCount); - } - - public static ArrayList toAggregationAreas (File file, SpatialDataset.SourceFormat sourceFormat, + public static ArrayList toAggregationAreas (File file, FileStorageFormat sourceFormat, Task progressListener) { ArrayList aggregationAreas = new ArrayList<>(); // TODO from shapefile diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index f96965732..817aadb2b 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -1,10 +1,7 @@ package com.conveyal.r5.util; import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.spatial.GeometryWrapper; -import com.conveyal.analysis.spatial.Lines; -import com.conveyal.analysis.spatial.Points; -import com.conveyal.analysis.spatial.Polygons; +import com.conveyal.analysis.spatial.FeatureSummary; import com.conveyal.analysis.spatial.SpatialAttribute; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; @@ -17,9 +14,6 @@ import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.geom.Lineal; -import org.locationtech.jts.geom.Polygonal; -import org.locationtech.jts.geom.Puntal; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.AttributeDescriptor; @@ -168,11 +162,7 @@ public List getAttributes () { return attributes; } - public GeometryWrapper getGeometryType () { - Class geometryType = features.getSchema().getGeometryDescriptor().getType().getBinding(); - if (Polygonal.class.isAssignableFrom(geometryType)) return new Polygons(features.size()); - if (Puntal.class.isAssignableFrom(geometryType)) return new Points(features.size()); - if (Lineal.class.isAssignableFrom(geometryType)) return new Lines(features.size()); - else return null; + public FeatureSummary featureSummary () { + return new FeatureSummary(features); } } From 81599b9d99c4d3ff63f4684c9dfdc6c76af46714 Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:09:27 -0400 Subject: [PATCH 035/187] refactor(spatial): consolidate use of FileStorageFormat enums instead of using duplicative SourceFormat --- .../OpportunityDatasetController.java | 10 +++--- .../controllers/SpatialDatasetController.java | 4 +-- .../analysis/models/SpatialDatasetSource.java | 32 +++++++++---------- .../analysis/spatial/SpatialDataset.java | 16 ++++------ .../com/conveyal/file/FileStorageFormat.java | 10 ++++-- 5 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index d5864acc2..da063cd75 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -8,7 +8,6 @@ import com.conveyal.analysis.models.Region; import com.conveyal.analysis.models.SpatialDatasetSource; import com.conveyal.analysis.persistence.Persistence; -import com.conveyal.analysis.spatial.SpatialDataset; import com.conveyal.analysis.util.FileItemInputStreamProvider; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; @@ -56,7 +55,6 @@ import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; -import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; @@ -347,7 +345,7 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res addStatusAndRemoveOldStatuses(status); final List fileItems; - final SpatialDataset.SourceFormat uploadFormat; + final FileStorageFormat uploadFormat; final Map parameters; try { // Validate inputs and parameters, which will throw an exception if there's anything wrong with them. @@ -367,13 +365,13 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res try { // A place to accumulate all the PointSets created, both FreeForm and Grids. List pointsets = new ArrayList<>(); - if (uploadFormat == SourceFormat.GRID) { + if (uploadFormat == FileStorageFormat.GRID) { LOG.info("Detected opportunity dataset stored in Conveyal binary format."); pointsets.addAll(createGridsFromBinaryGridFiles(fileItems, status)); - } else if (uploadFormat == SourceFormat.SHAPEFILE) { + } else if (uploadFormat == FileStorageFormat.SHP) { LOG.info("Detected opportunity dataset stored as ESRI shapefile."); pointsets.addAll(createGridsFromShapefile(fileItems, zoom, status)); - } else if (uploadFormat == SourceFormat.CSV) { + } else if (uploadFormat == FileStorageFormat.CSV) { LOG.info("Detected opportunity dataset stored as CSV"); // Create a grid even when user has requested a freeform pointset so we have something to visualize. FileItem csvFileItem = fileItems.get(0); diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 62395a61a..b6dbd2b91 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -25,7 +25,6 @@ import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.FileUploadException; -import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.json.simple.JSONObject; @@ -50,7 +49,6 @@ import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; -import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat; import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; @@ -321,7 +319,7 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { } progressListener.beginTask("Detecting format", 1); - final SourceFormat uploadFormat; + final FileStorageFormat uploadFormat; try { // Validate inputs, which will throw an exception if there's anything wrong with them. uploadFormat = detectUploadFormatAndValidate(fileItems); diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index 5a4b47684..071de82b4 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -4,10 +4,9 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.spatial.FeatureSummary; import com.conveyal.analysis.spatial.SpatialAttribute; -import com.conveyal.analysis.spatial.SpatialDataset; +import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; import com.conveyal.r5.util.ShapefileReader; -import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.fileupload.FileItem; import org.apache.commons.io.FilenameUtils; import org.locationtech.jts.geom.Envelope; @@ -20,7 +19,6 @@ import java.util.List; import java.util.Map; -import static com.conveyal.analysis.spatial.SpatialDataset.SourceFormat.*; import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; @@ -28,8 +26,8 @@ public class SpatialDatasetSource extends BaseModel { public String regionId; /** Description editable by end users */ public String description; - public SpatialDataset.SourceFormat sourceFormat; - /** General geometry type, with associated static methods for conversion to spatial datasets */ + public String sourceFormat; // FIXME figure out Mongo serialization and revert to enum + /** General geometry type */ public FeatureSummary features; /** Attributes, set only after validation (e.g. appropriate format for each feature's attributes) */ public List attributes; @@ -38,27 +36,31 @@ private SpatialDatasetSource (UserPermissions userPermissions, String sourceName super(userPermissions, sourceName); } - @JsonIgnore + /** + * No-arg constructor required for Mongo POJO serialization + */ + public SpatialDatasetSource () { + super(); + } + public static SpatialDatasetSource create (UserPermissions userPermissions, String sourceName) { return new SpatialDatasetSource(userPermissions, sourceName); } - @JsonIgnore public SpatialDatasetSource withRegion (String regionId) { this.regionId = regionId; return this; } - @JsonIgnore - public void validateAndSetDetails (SpatialDataset.SourceFormat uploadFormat, List files) { - this.sourceFormat = uploadFormat; - if (uploadFormat == GRID) { + public void validateAndSetDetails (FileStorageFormat uploadFormat, List files) { + this.sourceFormat = uploadFormat.toString(); + if (uploadFormat == FileStorageFormat.GRID) { // TODO source.fromGrids(fileItems); - } else if (uploadFormat == SHAPEFILE) { + } else if (uploadFormat == FileStorageFormat.SHP) { this.fromShapefile(files); - } else if (uploadFormat == CSV) { + } else if (uploadFormat == FileStorageFormat.CSV) { // TODO source.fromCsv(fileItems); - } else if (uploadFormat == GEOJSON) { + } else if (uploadFormat == FileStorageFormat.GEOJSON) { // TODO source.fromGeojson(fileItems); } } @@ -87,13 +89,11 @@ private void fromShapefile (List files) { } } - @JsonIgnore public SpatialDatasetSource fromFiles (List fileItemList) { // TODO this.files from fileItemList; return this; } - @JsonIgnore public FileStorageKey storageKey() { return new FileStorageKey(RESOURCES, this._id.toString(), sourceFormat.toString()); } diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java index 6e6418f12..e0858d2c0 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java @@ -2,6 +2,7 @@ import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.file.FileStorageFormat; import com.google.common.collect.Sets; import org.apache.commons.fileupload.FileItem; import org.apache.commons.io.FilenameUtils; @@ -15,11 +16,6 @@ */ public class SpatialDataset { - // TODO merge with FileCategory? - public enum SourceFormat { - SHAPEFILE, CSV, GEOJSON, GRID, SEAMLESS - } - /** * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary * grids. In the process we validate the list of uploaded files, making sure certain preconditions are met. @@ -28,7 +24,7 @@ public enum SourceFormat { * @throws AnalysisServerException if the type of the upload can't be detected or preconditions are violated. * @return the expected type of the uploaded file or files, never null. */ - public static SourceFormat detectUploadFormatAndValidate (List fileItems) { + public static FileStorageFormat detectUploadFormatAndValidate (List fileItems) { if (fileItems == null || fileItems.isEmpty()) { throw AnalysisServerException.fileUpload("You must include some files to create an opportunity dataset."); } @@ -46,13 +42,13 @@ public static SourceFormat detectUploadFormatAndValidate (List fileIte throw AnalysisServerException.fileUpload("No file extensions seen, cannot detect upload type."); } - SourceFormat uploadFormat = null; + FileStorageFormat uploadFormat = null; // Check that if upload contains any of the Shapefile sidecar files, it contains all of the required ones. final Set shapefileExtensions = Sets.newHashSet("SHP", "DBF", "PRJ"); if ( ! Sets.intersection(fileExtensions, shapefileExtensions).isEmpty()) { if (fileExtensions.containsAll(shapefileExtensions)) { - uploadFormat = SourceFormat.SHAPEFILE; + uploadFormat = FileStorageFormat.SHP; verifyBaseNamesSame(fileItems); // TODO check that any additional file is SHX, and that there are no more than 4 files. } else { @@ -64,14 +60,14 @@ public static SourceFormat detectUploadFormatAndValidate (List fileIte // Even if we've already detected a shapefile, run the other tests to check for a bad mixture of file types. if (fileExtensions.contains("GRID")) { if (fileExtensions.size() == 1) { - uploadFormat = SourceFormat.GRID; + uploadFormat = FileStorageFormat.GRID; } else { String message = "When uploading grids you may upload multiple files, but they must all be grids."; throw AnalysisServerException.fileUpload(message); } } else if (fileExtensions.contains("CSV")) { if (fileItems.size() == 1) { - uploadFormat = SourceFormat.CSV; + uploadFormat = FileStorageFormat.CSV; } else { String message = "When uploading CSV you may only upload one file at a time."; throw AnalysisServerException.fileUpload(message); diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index bf5d9666c..fae6cae68 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -1,5 +1,7 @@ package com.conveyal.file; +import org.bson.codecs.pojo.annotations.BsonIgnore; + public enum FileStorageFormat { FREEFORM("pointset", "application/octet-stream"), GRID("grid", "application/octet-stream"), @@ -13,12 +15,16 @@ public enum FileStorageFormat { // PBF("pbf", "application/octet-stream"), // SHP implies .dbf and .prj, and optionally .shx - SHP("shp", "application/octet-stream"); + SHP("shp", "application/octet-stream"), + + GEOJSON("json", "application/json"); + @BsonIgnore public final String extension; + @BsonIgnore public final String mimeType; - FileStorageFormat(String extension, String mimeType) { + FileStorageFormat (String extension, String mimeType) { this.extension = extension; this.mimeType = mimeType; } From f6971fd2b1b1957954c4034918713cd38fa3ec48 Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:10:05 -0400 Subject: [PATCH 036/187] feat(mongo): use conventions (including annotation) --- .../java/com/conveyal/analysis/persistence/AnalysisDB.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 35a3306d3..8777e2d37 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -6,6 +6,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import org.bson.codecs.configuration.CodecRegistry; +import org.bson.codecs.pojo.Conventions; import org.bson.codecs.pojo.PojoCodecProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +32,11 @@ public AnalysisDB (Config config) { // Create codec registry for POJOs CodecRegistry pojoCodecRegistry = fromRegistries( MongoClientSettings.getDefaultCodecRegistry(), - fromProviders(PojoCodecProvider.builder().automatic(true).build())); + fromProviders(PojoCodecProvider.builder() + .conventions(Conventions.DEFAULT_CONVENTIONS) + .automatic(true) + .build() + )); database = mongo.getDatabase(config.databaseName()).withCodecRegistry(pojoCodecRegistry); From 0c94005956f1806d3bf8ecf644c13a360ca5e6b2 Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:10:58 -0400 Subject: [PATCH 037/187] fix(spatial): get spatial datasets from correct collection --- .../analysis/controllers/SpatialDatasetController.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index b6dbd2b91..88fb6dcb0 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -55,6 +55,8 @@ import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; import static com.conveyal.r5.analyst.progress.WorkProductType.SPATIAL_DATASET_SOURCE; +import static com.mongodb.client.model.Filters.and; +import static com.mongodb.client.model.Filters.eq; /** * Controller that handles fetching opportunity datasets (grids and other pointset formats). @@ -91,9 +93,9 @@ private JSONObject getJSONURL (FileStorageKey key) { return json; } - private Collection getRegionDatasets(Request req, Response res) { - return Persistence.opportunityDatasets.findPermitted( - QueryBuilder.start("regionId").is(req.params("regionId")).get(), + private List getRegionDatasets(Request req, Response res) { + return spatialSourceCollection.findPermitted( + and(eq("regionId", req.params("regionId"))), req.attribute("accessGroup") ); } From 7125e205023560a4fdc0887b05195ccf00256a6c Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:11:45 -0400 Subject: [PATCH 038/187] refactor(db): make id of BaseModel final --- .../java/com/conveyal/analysis/models/BaseModel.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index 975d36493..86c847486 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -5,7 +5,7 @@ public class BaseModel { // Can retrieve `createdAt` from here - public ObjectId _id = new ObjectId(); + public final ObjectId _id = new ObjectId(); // For version management. ObjectId's contain a timestamp, so can retrieve `updatedAt` from here. public ObjectId nonce = new ObjectId(); @@ -27,9 +27,10 @@ public class BaseModel { this.name = name; } - - BaseModel () { - // No-arg + /** + * No-arg constructor required for Mongo POJO serialization + */ + public BaseModel () { + } - } From 8c51e04bc2d394bcef1bc43035ed071a9e4b7b5e Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:13:04 -0400 Subject: [PATCH 039/187] fix(db): make properties public for mongo serialization --- .../analysis/spatial/SpatialAttribute.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java index 6aa01f276..e14301de9 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java @@ -6,14 +6,14 @@ /** Groups the original names and user-friendly fields from shapefile attributes, CSV columns, etc. */ public class SpatialAttribute { /** Name in source file */ - public final String name; + public String name; /** Editable by end users */ - String label; + public String label; - Type type; + public Type type; - enum Type { + private enum Type { NUMBER, // internally, we generally parse as doubles TEXT, GEOM, @@ -29,4 +29,11 @@ public SpatialAttribute(String name, AttributeType type) { else this.type = Type.ERROR; } + /** + * No-arg constructor required for Mongo POJO serialization + */ + public SpatialAttribute () { + + } + } From 3ca99541eaa24b1aa7ae13078ee888b35071afc2 Mon Sep 17 00:00:00 2001 From: ansons Date: Sun, 27 Jun 2021 17:38:35 -0400 Subject: [PATCH 040/187] fix(db): set new ObjectID on construction only otherwise, _id is different on every read from db --- src/main/java/com/conveyal/analysis/models/BaseModel.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index 86c847486..e3acb73e1 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -5,10 +5,10 @@ public class BaseModel { // Can retrieve `createdAt` from here - public final ObjectId _id = new ObjectId(); + public ObjectId _id; // For version management. ObjectId's contain a timestamp, so can retrieve `updatedAt` from here. - public ObjectId nonce = new ObjectId(); + public ObjectId nonce; public String createdBy = null; public String updatedBy = null; @@ -21,6 +21,8 @@ public class BaseModel { // package private to encourage use of static factory methods BaseModel (UserPermissions user, String name) { + this._id = new ObjectId(); + this.nonce = new ObjectId(); this.createdBy = user.email; this.updatedBy = user.email; this.accessGroup = user.accessGroup; From 3df077d582c5b716db416c67c971563b59f9ee26 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 00:07:22 -0400 Subject: [PATCH 041/187] fix(spatial): store enum instead of string --- .../com/conveyal/analysis/models/SpatialDatasetSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java index 071de82b4..7dd13d683 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java @@ -26,7 +26,7 @@ public class SpatialDatasetSource extends BaseModel { public String regionId; /** Description editable by end users */ public String description; - public String sourceFormat; // FIXME figure out Mongo serialization and revert to enum + public FileStorageFormat sourceFormat; /** General geometry type */ public FeatureSummary features; /** Attributes, set only after validation (e.g. appropriate format for each feature's attributes) */ @@ -53,7 +53,7 @@ public SpatialDatasetSource withRegion (String regionId) { } public void validateAndSetDetails (FileStorageFormat uploadFormat, List files) { - this.sourceFormat = uploadFormat.toString(); + this.sourceFormat = uploadFormat; if (uploadFormat == FileStorageFormat.GRID) { // TODO source.fromGrids(fileItems); } else if (uploadFormat == FileStorageFormat.SHP) { From 61a8cdccd7afadd6384f653729971c7df2cd1465 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 00:09:10 -0400 Subject: [PATCH 042/187] feat(spatial): revise spatial dataset source display/deletion --- .../controllers/SpatialDatasetController.java | 98 +++++-------------- .../r5/analyst/progress/WorkProductType.java | 4 +- 2 files changed, 24 insertions(+), 78 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 88fb6dcb0..9feeb00ba 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -21,7 +21,6 @@ import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ExceptionUtils; import com.conveyal.r5.util.InputStreamProvider; -import com.mongodb.QueryBuilder; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.FileUploadException; @@ -34,7 +33,6 @@ import spark.Response; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -45,7 +43,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; @@ -54,7 +51,7 @@ import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; -import static com.conveyal.r5.analyst.progress.WorkProductType.SPATIAL_DATASET_SOURCE; +import static com.conveyal.r5.analyst.progress.WorkProductType.RESOURCE; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -100,18 +97,8 @@ private List getRegionDatasets(Request req, Response res) ); } - private Object getOpportunityDataset(Request req, Response res) { - OpportunityDataset dataset = Persistence.opportunityDatasets.findByIdFromRequestIfPermitted(req); - if (dataset.format == FileStorageFormat.GRID) { - return getJSONURL(dataset.getStorageKey()); - } else { - // Currently the UI can only visualize grids, not other kinds of datasets (freeform points). - // We do generate a rasterized grid for each of the freeform pointsets we create, so ideally we'd redirect - // to that grid for display and preview, but the freeform and corresponding grid pointset have different - // IDs and there are no references between them. - LOG.error("We cannot yet visualize freeform pointsets. Returning nothing to the UI."); - return null; - } + private Object getSource(Request req, Response res) { + return spatialSourceCollection.findPermittedByRequestParamId(req, res); } private SpatialDatasetSource downloadLODES(Request req, Response res) { @@ -301,9 +288,9 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { // Initialize model object SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName).withRegion(regionId); - taskScheduler.enqueue(Task.create("Storing " + sourceName) + taskScheduler.enqueue(Task.create("Uploading spatial dataset: " + sourceName) .forUser(userPermissions) - .withWorkProduct(SPATIAL_DATASET_SOURCE, source._id.toString(), regionId) + .withWorkProduct(RESOURCE, source._id.toString(), regionId) .withAction(progressListener -> { // Loop through uploaded files, registering the extensions and writing to storage (with filenames that @@ -340,15 +327,22 @@ private OpportunityDataset editOpportunityDataset(Request request, Response resp return Persistence.opportunityDatasets.updateFromJSONRequest(request); } - private Collection deleteSourceSet(Request request, Response response) { - String sourceId = request.params("sourceId"); - String accessGroup = request.attribute("accessGroup"); - Collection datasets = Persistence.opportunityDatasets.findPermitted( - QueryBuilder.start("sourceId").is(sourceId).get(), accessGroup); + private SpatialDatasetSource toAggregationArea(Request request, Response response) { + SpatialDatasetSource source = spatialSourceCollection.findPermittedByRequestParamId(request, response); + // TODO implement + return source; + } - datasets.forEach(dataset -> deleteDataset(dataset._id, accessGroup)); + private Collection deleteSourceSet(Request request, Response response) { + SpatialDatasetSource source = spatialSourceCollection.findPermittedByRequestParamId(request, response); + // TODO delete files from storage + // TODO delete referencing database records + spatialSourceCollection.delete(source); - return datasets; + return spatialSourceCollection.findPermitted( + and(eq("regionId", request.params("regionId"))), + request.attribute("accessGroup") + ); } private OpportunityDataset deleteOpportunityDataset(Request request, Response response) { @@ -433,65 +427,17 @@ private void createGridsFromShapefile(List fileItems) throws Exception // TODO implement rasterization methods } - /** - * Respond to a request with a redirect to a downloadable file. - * @param req should specify regionId, opportunityDatasetId, and an available download format (.tiff or .grid) - */ - private Object downloadOpportunityDataset (Request req, Response res) throws IOException { - FileStorageFormat downloadFormat; - try { - downloadFormat = FileStorageFormat.valueOf(req.params("format").toUpperCase()); - } catch (IllegalArgumentException iae) { - // This code handles the deprecated endpoint for retrieving opportunity datasets - // get("/api/opportunities/:regionId/:gridKey") is the same signature as this endpoint. - String regionId = req.params("_id"); - String gridKey = req.params("format"); - FileStorageKey storageKey = new FileStorageKey(GRIDS, String.format("%s/%s.grid", regionId, gridKey)); - return getJSONURL(storageKey); - } - - if (FileStorageFormat.GRID.equals(downloadFormat)) return getOpportunityDataset(req, res); - - final OpportunityDataset opportunityDataset = Persistence.opportunityDatasets.findByIdFromRequestIfPermitted(req); - - FileStorageKey gridKey = opportunityDataset.getStorageKey(FileStorageFormat.GRID); - FileStorageKey formatKey = opportunityDataset.getStorageKey(downloadFormat); - - // if this grid is not on S3 in the requested format, try to get the .grid format - if (!fileStorage.exists(gridKey)) { - throw AnalysisServerException.notFound("Requested grid does not exist."); - } - - if (!fileStorage.exists(formatKey)) { - // get the grid and convert it to the requested format - File gridFile = fileStorage.getFile(gridKey); - Grid grid = Grid.read(new GZIPInputStream(new FileInputStream(gridFile))); // closes input stream - File localFile = FileUtils.createScratchFile(downloadFormat.toString()); - FileOutputStream fos = new FileOutputStream(localFile); - - if (FileStorageFormat.PNG.equals(downloadFormat)) { - grid.writePng(fos); - } else if (FileStorageFormat.TIFF.equals(downloadFormat)) { - grid.writeGeotiff(fos); - } - - fileStorage.moveIntoStorage(formatKey, localFile); - } - - return getJSONURL(formatKey); - } - @Override public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/spatial", () -> { sparkService.post("", this::handleUpload, toJson); sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); - sparkService.delete("/source/:sourceId", this::deleteSourceSet, toJson); + sparkService.delete("/source/:_id", this::deleteSourceSet, toJson); + sparkService.post("/source/:_id/toAggregationArea", this::toAggregationArea, toJson); sparkService.delete("/:_id", this::deleteOpportunityDataset, toJson); - sparkService.get("/:_id", this::getOpportunityDataset, toJson); + sparkService.get("/:_id", this::getSource, toJson); sparkService.put("/:_id", this::editOpportunityDataset, toJson); - sparkService.get("/:_id/:format", this::downloadOpportunityDataset, toJson); }); } } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java index d3240b203..4002a290a 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java @@ -13,14 +13,14 @@ */ public enum WorkProductType { - BUNDLE, REGIONAL_ANALYSIS, AGGREGATION_AREA, OPPORTUNITY_DATASET, SPATIAL_DATASET_SOURCE; + BUNDLE, REGIONAL_ANALYSIS, AGGREGATION_AREA, OPPORTUNITY_DATASET, RESOURCE; public static WorkProductType forModel (Object model) { if (model instanceof Bundle) return BUNDLE; if (model instanceof OpportunityDataset) return OPPORTUNITY_DATASET; if (model instanceof RegionalAnalysis) return REGIONAL_ANALYSIS; if (model instanceof AggregationArea) return AGGREGATION_AREA; - if (model instanceof SpatialDatasetSource) return SPATIAL_DATASET_SOURCE; + if (model instanceof SpatialDatasetSource) return RESOURCE; // TODO switch to spatial dataset source throw new IllegalArgumentException("Unrecognized work product type."); } } From d92dcdda095a95c69a39686739f2e8921271c417 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 01:17:44 -0400 Subject: [PATCH 043/187] feat(spatial): re-implement creation of aggregation areas (aka mask grids) --- .../AggregationAreaController.java | 140 ++++++++++-------- .../controllers/SpatialDatasetController.java | 7 - .../conveyal/analysis/spatial/Polygons.java | 15 -- 3 files changed, 76 insertions(+), 86 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 0718a0889..ab4420151 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -43,6 +43,8 @@ import static com.conveyal.analysis.spatial.FeatureSummary.Type.POLYGON; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; +import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -91,86 +93,96 @@ private FileStorageKey getStoragePath (AggregationArea area) { */ private List createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); - Map> query = HttpUtils.getRequestFiles(req.raw()); UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - String maskName = query.get("name").get(0).getString("UTF-8"); - String nameProperty = query.get("nameProperty") == null ? null : query.get("nameProperty").get(0).getString( - "UTF-8"); - String sourceId = query.get("sourceId").get(0).getString("UTF-8"); + String sourceId = req.params("sourceId"); + String nameProperty = req.attribute("nameProperty"); + final int zoom = parseZoom(req.attribute("zoom")); - // 1. Get shapefile from storage and read its features. ======================================================== + // 1. Get file from storage and read its features. ============================================================= SpatialDatasetSource source = (SpatialDatasetSource) spatialSourceCollection.findById(sourceId); Preconditions.checkArgument(POLYGON.equals(source.features.type), "Only polygons can be converted to " + "aggregation areas."); - File shpFile = fileStorage.getFile(source.storageKey()); - ShapefileReader reader = null; - List features; - try { - reader = new ShapefileReader(shpFile); - features = reader.wgs84Stream().collect(Collectors.toList()); - } finally { - if (reader != null) reader.close(); - } - - Map areas = new HashMap<>(); + File sourceFile; + List features = null; - String zoomString = query.get("zoom") == null ? null : query.get("zoom").get(0).getString(); - final int zoom = parseZoom(zoomString); - - if (nameProperty != null && features.size() > MAX_FEATURES) { - throw AnalysisServerException.fileUpload(MessageFormat.format("The uploaded shapefile has {0} features, " + - "which exceeds the limit of {1}", features.size(), MAX_FEATURES)); + if (SHP.equals(source.sourceFormat)) { + sourceFile = fileStorage.getFile(source.storageKey()); + ShapefileReader reader = null; + try { + reader = new ShapefileReader(sourceFile); + features = reader.wgs84Stream().collect(Collectors.toList()); + } finally { + if (reader != null) reader.close(); + } } - if (nameProperty == null) { - // Union (single combined aggregation area) requested - List geometries = features.stream().map(f -> (Geometry) f.getDefaultGeometry()).collect(Collectors.toList()); - UnaryUnionOp union = new UnaryUnionOp(geometries); - // Name the area using the name in the request directly - areas.put(maskName, union.union()); - } else { - // Don't union. Name each area by looking up its value for the name property in the request. - features.forEach(f -> areas.put(readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry())); + if (GEOJSON.equals(source.sourceFormat)) { + // TODO implement } - taskScheduler.enqueue(Task.create("Creating aggregation area") + List finalFeatures = features; + taskScheduler.enqueue(Task.create("Aggregation area creation: " + source.name) .forUser(userPermissions) .setHeavy(true) .withWorkProduct(source) - // TODO move below into .withAction() + .withAction(progressListener -> { + progressListener.beginTask("Processing request", 1); + Map areas = new HashMap<>(); + + if (nameProperty != null && finalFeatures.size() > MAX_FEATURES) { + throw AnalysisServerException.fileUpload(MessageFormat.format("The uploaded shapefile has {0} features, " + + "which exceeds the limit of {1}", finalFeatures.size(), MAX_FEATURES)); + } + + if (nameProperty == null) { + // Union (single combined aggregation area) requested + List geometries = finalFeatures.stream().map(f -> (Geometry) f.getDefaultGeometry()).collect(Collectors.toList()); + UnaryUnionOp union = new UnaryUnionOp(geometries); + // Name the area using the name in the request directly + areas.put(source.name, union.union()); + } else { + // Don't union. Name each area by looking up its value for the name property in the request. + finalFeatures.forEach(f -> areas.put(readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry())); + } + + // 2. Convert to raster grids, then store them. ================================================================ + areas.forEach((String name, Geometry geometry) -> { + if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); + Envelope env = geometry.getEnvelopeInternal(); + Grid maskGrid = new Grid(zoom, env); + progressListener.beginTask("Creating grid for " + name, maskGrid.featureCount()); + + // Store the percentage each cell overlaps the mask, scaled as 0 to 100,000 + List weights = maskGrid.getPixelWeights(geometry, true); + weights.forEach(pixel -> { + maskGrid.grid[pixel.x][pixel.y] = pixel.weight * 100_000; + progressListener.increment(); + }); + + AggregationArea aggregationArea = AggregationArea.create(userPermissions, name) + .withSource(source); + + try { + File gridFile = FileUtils.createScratchFile("grid"); + OutputStream os = new GZIPOutputStream(FileUtils.getOutputStream(gridFile)); + maskGrid.write(os); + os.close(); + + aggregationAreaCollection.insert(aggregationArea); + aggregationAreas.add(aggregationArea); + + fileStorage.moveIntoStorage(getStoragePath(aggregationArea), gridFile); + } catch (IOException e) { + throw new AnalysisServerException("Error processing/uploading aggregation area"); + } + progressListener.increment(); + }); + }) ); - // 2. Convert to raster grids, then store them. ================================================================ - areas.forEach((String name, Geometry geometry) -> { - if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); - Envelope env = geometry.getEnvelopeInternal(); - Grid maskGrid = new Grid(zoom, env); - - // Store the percentage each cell overlaps the mask, scaled as 0 to 100,000 - List weights = maskGrid.getPixelWeights(geometry, true); - weights.forEach(pixel -> maskGrid.grid[pixel.x][pixel.y] = pixel.weight * 100_000); - - AggregationArea aggregationArea = AggregationArea.create(userPermissions, name) - .withSource(source); - - try { - File gridFile = FileUtils.createScratchFile("grid"); - OutputStream os = new GZIPOutputStream(FileUtils.getOutputStream(gridFile)); - maskGrid.write(os); - os.close(); - - aggregationAreaCollection.insert(aggregationArea); - aggregationAreas.add(aggregationArea); - - fileStorage.moveIntoStorage(getStoragePath(aggregationArea), gridFile); - } catch (IOException e) { - throw new AnalysisServerException("Error processing/uploading aggregation area"); - } - - }); - return aggregationAreas; + } private String readProperty (SimpleFeature feature, String propertyName) { @@ -206,7 +218,7 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/region/", () -> { sparkService.get("/:regionId/aggregationArea", this::getAggregationAreas, toJson); sparkService.get("/:regionId/aggregationArea/:maskId", this::getAggregationArea, toJson); - sparkService.post("/:regionId/aggregationArea", this::createAggregationAreas, toJson); + sparkService.post("/:regionId/aggregationArea/:sourceId", this::createAggregationAreas, toJson); }); } diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 9feeb00ba..1a8a3658b 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -327,12 +327,6 @@ private OpportunityDataset editOpportunityDataset(Request request, Response resp return Persistence.opportunityDatasets.updateFromJSONRequest(request); } - private SpatialDatasetSource toAggregationArea(Request request, Response response) { - SpatialDatasetSource source = spatialSourceCollection.findPermittedByRequestParamId(request, response); - // TODO implement - return source; - } - private Collection deleteSourceSet(Request request, Response response) { SpatialDatasetSource source = spatialSourceCollection.findPermittedByRequestParamId(request, response); // TODO delete files from storage @@ -434,7 +428,6 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); sparkService.delete("/source/:_id", this::deleteSourceSet, toJson); - sparkService.post("/source/:_id/toAggregationArea", this::toAggregationArea, toJson); sparkService.delete("/:_id", this::deleteOpportunityDataset, toJson); sparkService.get("/:_id", this::getSource, toJson); sparkService.put("/:_id", this::editOpportunityDataset, toJson); diff --git a/src/main/java/com/conveyal/analysis/spatial/Polygons.java b/src/main/java/com/conveyal/analysis/spatial/Polygons.java index b61fc044c..32fe52c60 100644 --- a/src/main/java/com/conveyal/analysis/spatial/Polygons.java +++ b/src/main/java/com/conveyal/analysis/spatial/Polygons.java @@ -1,22 +1,7 @@ package com.conveyal.analysis.spatial; -import com.conveyal.analysis.models.AggregationArea; -import com.conveyal.file.FileStorageFormat; -import com.conveyal.r5.analyst.progress.Task; - -import java.io.File; -import java.util.ArrayList; - public class Polygons { - public static ArrayList toAggregationAreas (File file, FileStorageFormat sourceFormat, - Task progressListener) { - ArrayList aggregationAreas = new ArrayList<>(); - // TODO from shapefile - // TODO from geojson - return aggregationAreas; - } - // TODO toGrid from shapefile and geojson // TODO modification polygon From a9b7bdd921ac84dddd7448f48961ef8888cf6a19 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 14:28:54 -0400 Subject: [PATCH 044/187] fix(spatial): reimplement legacy upload status updates Our plan is eventually to migrate to Task tracking; but keep the legacy status tracking in place for now to pass integration tests. --- .../analysis/controllers/OpportunityDatasetController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index da063cd75..f2e704d27 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -177,7 +177,8 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) private void updateAndStoreDatasets (SpatialDatasetSource source, OpportunityDatasetUploadStatus status, List pointSets) { - + status.status = Status.UPLOADING; + status.totalGrids = pointSets.size(); // Create an OpportunityDataset holding some metadata about each PointSet (Grid or FreeForm). final List datasets = new ArrayList<>(); for (PointSet pointSet : pointSets) { From 7278f41d5fcc50c9667a3dcef4b83be8191d940f Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 15:18:28 -0400 Subject: [PATCH 045/187] tests(spatial): bump ui to latest dev commit which has removed aggregation upload from regional analysis testing --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index 31072191a..cc839c334 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: a869cd11919343a163e110e812b5d27f3a4ad4c8 + ref: 12c62421dfe376c471303d85b302427cdcf2a17f path: ui - uses: actions/checkout@v2 with: From 4fea11ca269b952600d878cb3fdce89b09413510 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 17:02:57 -0400 Subject: [PATCH 046/187] fix(spatial): use queryParams for aggregation area options instead of request attributes --- .../analysis/controllers/AggregationAreaController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index ab4420151..e2416a69e 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -95,8 +95,8 @@ private List createAggregationAreas (Request req, Response res) ArrayList aggregationAreas = new ArrayList<>(); UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); String sourceId = req.params("sourceId"); - String nameProperty = req.attribute("nameProperty"); - final int zoom = parseZoom(req.attribute("zoom")); + String nameProperty = req.queryParams("nameProperty"); + final int zoom = parseZoom(req.queryParams("zoom")); // 1. Get file from storage and read its features. ============================================================= SpatialDatasetSource source = (SpatialDatasetSource) spatialSourceCollection.findById(sourceId); From 933eedc5a1029bc99a7c91fd505648aa3bac155d Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 23:27:19 -0400 Subject: [PATCH 047/187] refactor(spatial): fix line width in aggregation area controller --- .../controllers/AggregationAreaController.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index e2416a69e..443d23395 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -131,22 +131,28 @@ private List createAggregationAreas (Request req, Response res) Map areas = new HashMap<>(); if (nameProperty != null && finalFeatures.size() > MAX_FEATURES) { - throw AnalysisServerException.fileUpload(MessageFormat.format("The uploaded shapefile has {0} features, " + - "which exceeds the limit of {1}", finalFeatures.size(), MAX_FEATURES)); + throw AnalysisServerException.fileUpload( + MessageFormat.format("The uploaded shapefile has {0} features, " + + "which exceeds the limit of {1}", finalFeatures.size(), MAX_FEATURES) + ); } if (nameProperty == null) { // Union (single combined aggregation area) requested - List geometries = finalFeatures.stream().map(f -> (Geometry) f.getDefaultGeometry()).collect(Collectors.toList()); + List geometries = finalFeatures.stream().map(f -> + (Geometry) f.getDefaultGeometry()).collect(Collectors.toList() + ); UnaryUnionOp union = new UnaryUnionOp(geometries); // Name the area using the name in the request directly areas.put(source.name, union.union()); } else { // Don't union. Name each area by looking up its value for the name property in the request. - finalFeatures.forEach(f -> areas.put(readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry())); + finalFeatures.forEach(f -> areas.put( + readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry()) + ); } - // 2. Convert to raster grids, then store them. ================================================================ + // 2. Convert to raster grids, then store them. ==================================================== areas.forEach((String name, Geometry geometry) -> { if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); Envelope env = geometry.getEnvelopeInternal(); From b446ceb4a087a719896f8d5499df05f595afec75 Mon Sep 17 00:00:00 2001 From: ansons Date: Mon, 28 Jun 2021 23:27:49 -0400 Subject: [PATCH 048/187] refactor(spatial): remove unused methods (copied from OpportunityDatasetController for development) --- .../controllers/SpatialDatasetController.java | 222 ------------------ 1 file changed, 222 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 1a8a3658b..5c0068182 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -61,11 +61,9 @@ public class SpatialDatasetController implements HttpController { private static final Logger LOG = LoggerFactory.getLogger(SpatialDatasetController.class); - private static final FileItemFactory fileItemFactory = new DiskFileItemFactory(); // Component Dependencies - private final FileStorage fileStorage; private final AnalysisCollection spatialSourceCollection; private final TaskScheduler taskScheduler; @@ -83,13 +81,6 @@ public SpatialDatasetController ( this.extractor = extractor; } - private JSONObject getJSONURL (FileStorageKey key) { - JSONObject json = new JSONObject(); - String url = fileStorage.getURL(key); - json.put("url", url); - return json; - } - private List getRegionDatasets(Request req, Response res) { return spatialSourceCollection.findPermitted( and(eq("regionId", req.params("regionId"))), @@ -104,13 +95,7 @@ private Object getSource(Request req, Response res) { private SpatialDatasetSource downloadLODES(Request req, Response res) { final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - final Region region = Persistence.regions.findByIdIfPermitted(regionId, userPermissions.accessGroup); - // Common UUID for all LODES datasets created in this download (e.g. so they can be grouped together and - // deleted as a batch using deleteSourceSet) - // The bucket name contains the specific lodes data set and year so works as an appropriate name - SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, extractor.sourceName) .withRegion(regionId); @@ -125,125 +110,6 @@ private SpatialDatasetSource downloadLODES(Request req, Response res) { return source; } - /** - * Given a list of new PointSets, serialize each PointSet and save it to S3, then create a metadata object about - * that PointSet and store it in Mongo. - */ - private void updateAndStoreDatasets (SpatialDatasetSource source, - List pointSets) { - - // Create an OpportunityDataset holding some metadata about each PointSet (Grid or FreeForm). - final List datasets = new ArrayList<>(); - for (PointSet pointSet : pointSets) { - OpportunityDataset dataset = new OpportunityDataset(); - dataset.sourceName = source.name; - dataset.sourceId = source._id.toString(); - dataset.createdBy = source.createdBy; - dataset.accessGroup = source.accessGroup; - dataset.regionId = source.regionId; - dataset.name = pointSet.name; - dataset.totalPoints = pointSet.featureCount(); - dataset.totalOpportunities = pointSet.sumTotalOpportunities(); - dataset.format = getFormatCode(pointSet); - if (dataset.format == FileStorageFormat.FREEFORM) { - dataset.name = String.join(" ", pointSet.name, "(freeform)"); - } - dataset.setWebMercatorExtents(pointSet); - // TODO make origin and destination pointsets reference each other and indicate they are suitable - // for one-to-one analyses - - // Store the PointSet metadata in Mongo and accumulate these objects into the method return list. - Persistence.opportunityDatasets.create(dataset); - datasets.add(dataset); - - // Persist a serialized representation of each PointSet (not the metadata) to S3 or other object storage. - // TODO this should probably be pulled out to another method, and possibly called one frame up. - // Persisting the PointSets to S3 is a separate task than making metadata and storing in Mongo. - try { - if (pointSet instanceof Grid) { - File gridFile = FileUtils.createScratchFile("grid"); - - OutputStream fos = new GZIPOutputStream(new FileOutputStream(gridFile)); - ((Grid)pointSet).write(fos); - - fileStorage.moveIntoStorage(dataset.getStorageKey(FileStorageFormat.GRID), gridFile); - } else if (pointSet instanceof FreeFormPointSet) { - // Upload serialized freeform pointset back to S3 - FileStorageKey fileStorageKey = new FileStorageKey(GRIDS, source.regionId + "/" + dataset._id + - ".pointset"); - File pointsetFile = FileUtils.createScratchFile("pointset"); - - OutputStream os = new GZIPOutputStream(new FileOutputStream(pointsetFile)); - ((FreeFormPointSet)pointSet).write(os); - - fileStorage.moveIntoStorage(fileStorageKey, pointsetFile); - } else { - throw new IllegalArgumentException("Unrecognized PointSet type, cannot persist it."); - } - // TODO task tracking - } catch (NumberFormatException e) { - throw new AnalysisServerException("Error attempting to parse number in uploaded file: " + e.toString()); - } catch (Exception e) { - throw AnalysisServerException.unknown(e); - } - } - } - - private static FileStorageFormat getFormatCode (PointSet pointSet) { - if (pointSet instanceof FreeFormPointSet) { - return FileStorageFormat.FREEFORM; - } else if (pointSet instanceof Grid) { - return FileStorageFormat.GRID; - } else { - throw new RuntimeException("Unknown pointset type."); - } - } - - /** - * Given a CSV file, converts each property (CSV column) into a freeform (non-gridded) pointset. - * - * The provided multipart form data must include latField and lonField. To indicate paired origins and destinations - * (e.g. to use results from an origin-destination survey in a one-to-one regional analysis), the form data should - * include the optional latField2 and lonField2 fields. - * - * This method executes in a blocking (synchronous) manner, but it can take a while so should be called within an - * non-blocking asynchronous task. - */ - private List createFreeFormPointSetsFromCsv(FileItem csvFileItem, Map params) { - - String latField = params.get("latField"); - String lonField = params.get("lonField"); - if (latField == null || lonField == null) { - throw AnalysisServerException.fileUpload("You must specify a latitude and longitude column."); - } - - // The name of the column containing a unique identifier for each row. May be missing (null). - String idField = params.get("idField"); - - // The name of the column containing the opportunity counts at each point. May be missing (null). - String countField = params.get("countField"); - - // Optional secondary latitude, longitude, and count fields. - // This allows you to create two matched parallel pointsets of the same size with the same IDs. - String latField2 = params.get("latField2"); - String lonField2 = params.get("lonField2"); - - try { - List pointSets = new ArrayList<>(); - InputStreamProvider csvStreamProvider = new FileItemInputStreamProvider(csvFileItem); - pointSets.add(FreeFormPointSet.fromCsv(csvStreamProvider, latField, lonField, idField, countField)); - // The second pair of lat and lon fields allow creating two matched pointsets from the same CSV. - // This is used for one-to-one travel times between specific origins/destinations. - if (latField2 != null && lonField2 != null) { - pointSets.add(FreeFormPointSet.fromCsv(csvStreamProvider, latField2, lonField2, idField, countField)); - } - return pointSets; - } catch (Exception e) { - throw AnalysisServerException.fileUpload("Could not convert CSV to Freeform PointSet: " + e.toString()); - } - - } - /** * Get the specified field from a map representing a multipart/form-data POST request, as a UTF-8 String. * FileItems represent any form item that was received within a multipart/form-data POST request, not just files. @@ -323,10 +189,6 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { return source; } - private OpportunityDataset editOpportunityDataset(Request request, Response response) throws IOException { - return Persistence.opportunityDatasets.updateFromJSONRequest(request); - } - private Collection deleteSourceSet(Request request, Response response) { SpatialDatasetSource source = spatialSourceCollection.findPermittedByRequestParamId(request, response); // TODO delete files from storage @@ -339,88 +201,6 @@ private Collection deleteSourceSet(Request request, Respon ); } - private OpportunityDataset deleteOpportunityDataset(Request request, Response response) { - String opportunityDatasetId = request.params("_id"); - return deleteDataset(opportunityDatasetId, request.attribute("accessGroup")); - } - - /** - * Delete an Opportunity Dataset from the database and all formats from the file store. - */ - private OpportunityDataset deleteDataset(String id, String accessGroup) { - OpportunityDataset dataset = Persistence.opportunityDatasets.removeIfPermitted(id, accessGroup); - - if (dataset == null) { - throw AnalysisServerException.notFound("Opportunity dataset could not be found."); - } else { - fileStorage.delete(dataset.getStorageKey(FileStorageFormat.GRID)); - fileStorage.delete(dataset.getStorageKey(FileStorageFormat.PNG)); - fileStorage.delete(dataset.getStorageKey(FileStorageFormat.TIFF)); - } - - return dataset; - } - - /** - * Create a grid from WGS 84 points in a CSV file. - * The supplied CSV file will not be deleted - it may be used again to make another (freeform) pointset. - * TODO explain latField2 usage - * @return one or two Grids for each numeric column in the CSV input. - */ - private List createGridsFromCsv(FileItem csvFileItem, - Map> query, - int zoom) throws Exception { - - String latField = getFormField(query, "latField", true); - String lonField = getFormField(query, "lonField", true); - String idField = getFormField(query, "idField", false); - - // Optional fields to run grid construction twice with two different sets of points. - // This is only really useful when creating grids to visualize freeform pointsets for one-to-one analyses. - String latField2 = getFormField(query, "latField2", false); - String lonField2 = getFormField(query, "lonField2", false); - - List ignoreFields = Arrays.asList(idField, latField2, lonField2); - InputStreamProvider csvStreamProvider = new FileItemInputStreamProvider(csvFileItem); - List grids = Grid.fromCsv(csvStreamProvider, latField, lonField, ignoreFields, zoom, null); - // TODO verify correctness of this second pass - if (latField2 != null && lonField2 != null) { - ignoreFields = Arrays.asList(idField, latField, lonField); - grids.addAll(Grid.fromCsv(csvStreamProvider, latField2, lonField2, ignoreFields, zoom, null)); - } - - return grids; - } - - /** - * Create a grid from an input stream containing a binary grid file. - * For those in the know, we can upload manually created binary grid files. - */ - private List createGridsFromBinaryGridFiles(List uploadedFiles) throws Exception { - - List grids = new ArrayList<>(); - // TODO task size with uploadedFiles.size(); - for (FileItem fileItem : uploadedFiles) { - Grid grid = Grid.read(fileItem.getInputStream()); - String name = fileItem.getName(); - // Remove ".grid" from the name - if (name.contains(".grid")) name = name.split(".grid")[0]; - grid.name = name; - // TODO task progress - grids.add(grid); - } - // TODO mark task complete - return grids; - } - - /** - * Preconditions: fileItems must contain SHP, DBF, and PRJ files, and optionally SHX. All files should have the - * same base name, and should not contain any other files but these three or four. - */ - private void createGridsFromShapefile(List fileItems) throws Exception { - // TODO implement rasterization methods - } - @Override public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/spatial", () -> { @@ -428,9 +208,7 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); sparkService.delete("/source/:_id", this::deleteSourceSet, toJson); - sparkService.delete("/:_id", this::deleteOpportunityDataset, toJson); sparkService.get("/:_id", this::getSource, toJson); - sparkService.put("/:_id", this::editOpportunityDataset, toJson); }); } } From 69e83e80875ade750a1dbec9669eeefd51f2688c Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 29 Jun 2021 00:00:49 -0400 Subject: [PATCH 049/187] refactor(spatial): clean up upload handling --- .../controllers/SpatialDatasetController.java | 42 +++++-------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java index 5c0068182..32f91d09a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java @@ -4,51 +4,35 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; -import com.conveyal.analysis.models.OpportunityDataset; -import com.conveyal.analysis.models.Region; import com.conveyal.analysis.models.SpatialDatasetSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; -import com.conveyal.analysis.persistence.Persistence; -import com.conveyal.analysis.util.FileItemInputStreamProvider; +import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; -import com.conveyal.file.FileUtils; -import com.conveyal.r5.analyst.FreeFormPointSet; -import com.conveyal.r5.analyst.Grid; -import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.progress.Task; -import com.conveyal.r5.util.ExceptionUtils; -import com.conveyal.r5.util.InputStreamProvider; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; -import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.fileupload.disk.DiskFileItemFactory; -import org.apache.commons.fileupload.servlet.ServletFileUpload; -import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.zip.GZIPOutputStream; +import java.util.StringJoiner; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; import static com.conveyal.r5.analyst.progress.WorkProductType.RESOURCE; @@ -61,7 +45,6 @@ public class SpatialDatasetController implements HttpController { private static final Logger LOG = LoggerFactory.getLogger(SpatialDatasetController.class); - private static final FileItemFactory fileItemFactory = new DiskFileItemFactory(); // Component Dependencies private final FileStorage fileStorage; @@ -138,14 +121,7 @@ private String getFormField(Map> formFields, String field */ private SpatialDatasetSource handleUpload(Request req, Response res) { final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - final Map> formFields; - try { - ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); - formFields = sfu.parseParameterMap(req.raw()); - } catch (FileUploadException e) { - // We can't even get enough information to create a status tracking object. Re-throw an exception. - throw AnalysisServerException.fileUpload("Unable to parse uploaded file(s). " + ExceptionUtils.stackTraceString(e)); - } + final Map> formFields = HttpUtils.getRequestFiles(req.raw()); // Parse required fields. Will throw a ServerException on failure. final String sourceName = getFormField(formFields, "sourceName", true); @@ -162,14 +138,15 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { // Loop through uploaded files, registering the extensions and writing to storage (with filenames that // correspond to the source id) List files = new ArrayList<>(); - final List fileItems = formFields.remove("sourceFiles"); + StringJoiner fileNames = new StringJoiner(", "); + final List fileItems = formFields.get("sourceFiles"); for (FileItem fileItem : fileItems) { + DiskFileItem dfi = (DiskFileItem) fileItem; String filename = fileItem.getName(); + fileNames.add(filename); String extension = filename.substring(filename.lastIndexOf(".") + 1).toUpperCase(Locale.ROOT); FileStorageKey key = new FileStorageKey(RESOURCES, source._id.toString(), extension); - // FIXME writing not allowed by fileStorage.getFile contract - // FIXME Investigate fileStorage.moveIntoStorage(key, file), for consistency with BundleController; - fileItem.write(fileStorage.getFile(key)); + fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); files.add(fileStorage.getFile(key)); } @@ -183,6 +160,7 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { throw AnalysisServerException.fileUpload("Problem reading uploaded spatial files" + e.getMessage()); } progressListener.beginTask("Validating files", 1); + source.description = "From uploaded files: " + fileNames; source.validateAndSetDetails(uploadFormat, files); spatialSourceCollection.insert(source); })); From b528aac96ff6e4989b95c99114bc2eac8d7b91b2 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 29 Jun 2021 01:01:07 -0400 Subject: [PATCH 050/187] refactor(spatial): consolidate "resource" terminology --- .../components/BackendComponents.java | 4 +- .../AggregationAreaController.java | 8 ++- .../OpportunityDatasetController.java | 10 ++-- .../RegionalAnalysisController.java | 1 - ...er.java => SpatialResourceController.java} | 52 +++++++++---------- .../analysis/models/AggregationArea.java | 2 +- ...atasetSource.java => SpatialResource.java} | 19 ++++--- ...SpatialDataset.java => SpatialLayers.java} | 4 +- .../r5/analyst/progress/WorkProductType.java | 4 +- 9 files changed, 52 insertions(+), 52 deletions(-) rename src/main/java/com/conveyal/analysis/controllers/{SpatialDatasetController.java => SpatialResourceController.java} (77%) rename src/main/java/com/conveyal/analysis/models/{SpatialDatasetSource.java => SpatialResource.java} (82%) rename src/main/java/com/conveyal/analysis/spatial/{SpatialDataset.java => SpatialLayers.java} (97%) diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 65891e859..408a8356d 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -14,7 +14,7 @@ import com.conveyal.analysis.controllers.OpportunityDatasetController; import com.conveyal.analysis.controllers.ProjectController; import com.conveyal.analysis.controllers.RegionalAnalysisController; -import com.conveyal.analysis.controllers.SpatialDatasetController; +import com.conveyal.analysis.controllers.SpatialResourceController; import com.conveyal.analysis.controllers.TimetableController; import com.conveyal.analysis.controllers.UserActivityController; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; @@ -104,7 +104,7 @@ public List standardHttpControllers () { new BrokerController(broker, eventBus), new UserActivityController(taskScheduler), new GtfsTileController(gtfsCache), - new SpatialDatasetController(fileStorage, database, taskScheduler, censusExtractor) + new SpatialResourceController(fileStorage, database, taskScheduler, censusExtractor) ); } diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 443d23395..ee1cfcebd 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -4,10 +4,9 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.models.AggregationArea; -import com.conveyal.analysis.models.SpatialDatasetSource; +import com.conveyal.analysis.models.SpatialResource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; -import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; @@ -15,7 +14,6 @@ import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ShapefileReader; import com.google.common.base.Preconditions; -import org.apache.commons.fileupload.FileItem; import org.json.simple.JSONObject; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -76,7 +74,7 @@ public AggregationAreaController ( this.fileStorage = fileStorage; this.taskScheduler = taskScheduler; this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); - this.spatialSourceCollection = database.getAnalysisCollection("spatialSources", SpatialDatasetSource.class); + this.spatialSourceCollection = database.getAnalysisCollection("spatialSources", SpatialResource.class); } private FileStorageKey getStoragePath (AggregationArea area) { @@ -99,7 +97,7 @@ private List createAggregationAreas (Request req, Response res) final int zoom = parseZoom(req.queryParams("zoom")); // 1. Get file from storage and read its features. ============================================================= - SpatialDatasetSource source = (SpatialDatasetSource) spatialSourceCollection.findById(sourceId); + SpatialResource source = (SpatialResource) spatialSourceCollection.findById(sourceId); Preconditions.checkArgument(POLYGON.equals(source.features.type), "Only polygons can be converted to " + "aggregation areas."); diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index f2e704d27..1c5e42c54 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -6,7 +6,7 @@ import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.Region; -import com.conveyal.analysis.models.SpatialDatasetSource; +import com.conveyal.analysis.models.SpatialResource; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.FileItemInputStreamProvider; import com.conveyal.file.FileStorage; @@ -55,7 +55,7 @@ import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; -import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; +import static com.conveyal.analysis.spatial.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; @@ -149,7 +149,7 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) final OpportunityDatasetUploadStatus status = new OpportunityDatasetUploadStatus(regionId, extractor.sourceName); addStatusAndRemoveOldStatuses(status); - SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, extractor.sourceName) + SpatialResource source = SpatialResource.create(userPermissions, extractor.sourceName) .withRegion(regionId); taskScheduler.enqueue(Task.create("Extracting LODES data") @@ -174,7 +174,7 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) * Given a list of new PointSets, serialize each PointSet and save it to S3, then create a metadata object about * that PointSet and store it in Mongo. */ - private void updateAndStoreDatasets (SpatialDatasetSource source, + private void updateAndStoreDatasets (SpatialResource source, OpportunityDatasetUploadStatus status, List pointSets) { status.status = Status.UPLOADING; @@ -406,7 +406,7 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res // Create a single unique ID string that will be referenced by all opportunity datasets produced by // this upload. This allows us to group together datasets from the same source and associate them with // the file(s) that produced them. - SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName) + SpatialResource source = SpatialResource.create(userPermissions, sourceName) .withRegion(regionId); updateAndStoreDatasets(source, status, pointsets); } catch (Exception e) { diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 894c62ac0..3ff07d6ed 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -11,7 +11,6 @@ import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.results.CsvResultType; import com.conveyal.analysis.util.JsonUtil; -import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java similarity index 77% rename from src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java rename to src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java index 32f91d09a..afddbf157 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java @@ -4,7 +4,7 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; -import com.conveyal.analysis.models.SpatialDatasetSource; +import com.conveyal.analysis.models.SpatialResource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.HttpUtils; @@ -13,9 +13,7 @@ import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.Task; import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.disk.DiskFileItem; -import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -31,7 +29,7 @@ import java.util.StringJoiner; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; -import static com.conveyal.analysis.spatial.SpatialDataset.detectUploadFormatAndValidate; +import static com.conveyal.analysis.spatial.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; @@ -40,46 +38,46 @@ import static com.mongodb.client.model.Filters.eq; /** - * Controller that handles fetching opportunity datasets (grids and other pointset formats). + * Controller that handles CRUD of spatial resources. */ -public class SpatialDatasetController implements HttpController { +public class SpatialResourceController implements HttpController { - private static final Logger LOG = LoggerFactory.getLogger(SpatialDatasetController.class); + private static final Logger LOG = LoggerFactory.getLogger(SpatialResourceController.class); // Component Dependencies private final FileStorage fileStorage; - private final AnalysisCollection spatialSourceCollection; + private final AnalysisCollection spatialResourceCollection; private final TaskScheduler taskScheduler; private final SeamlessCensusGridExtractor extractor; - public SpatialDatasetController ( + public SpatialResourceController ( FileStorage fileStorage, AnalysisDB database, TaskScheduler taskScheduler, SeamlessCensusGridExtractor extractor ) { this.fileStorage = fileStorage; - this.spatialSourceCollection = database.getAnalysisCollection("spatialSources", SpatialDatasetSource.class); + this.spatialResourceCollection = database.getAnalysisCollection("spatialSources", SpatialResource.class); this.taskScheduler = taskScheduler; this.extractor = extractor; } - private List getRegionDatasets(Request req, Response res) { - return spatialSourceCollection.findPermitted( + private List getRegionResources (Request req, Response res) { + return spatialResourceCollection.findPermitted( and(eq("regionId", req.params("regionId"))), req.attribute("accessGroup") ); } - private Object getSource(Request req, Response res) { - return spatialSourceCollection.findPermittedByRequestParamId(req, res); + private Object getResource (Request req, Response res) { + return spatialResourceCollection.findPermittedByRequestParamId(req, res); } - private SpatialDatasetSource downloadLODES(Request req, Response res) { + private SpatialResource downloadLODES(Request req, Response res) { final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, extractor.sourceName) + SpatialResource source = SpatialResource.create(userPermissions, extractor.sourceName) .withRegion(regionId); taskScheduler.enqueue(Task.create("Extracting LODES data") @@ -119,7 +117,7 @@ private String getFormField(Map> formFields, String field * Handle many types of spatial upload. * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. */ - private SpatialDatasetSource handleUpload(Request req, Response res) { + private SpatialResource handleUpload(Request req, Response res) { final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); final Map> formFields = HttpUtils.getRequestFiles(req.raw()); @@ -128,9 +126,9 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { final String regionId = getFormField(formFields, "regionId", true); // Initialize model object - SpatialDatasetSource source = SpatialDatasetSource.create(userPermissions, sourceName).withRegion(regionId); + SpatialResource source = SpatialResource.create(userPermissions, sourceName).withRegion(regionId); - taskScheduler.enqueue(Task.create("Uploading spatial dataset: " + sourceName) + taskScheduler.enqueue(Task.create("Uploading spatial files: " + sourceName) .forUser(userPermissions) .withWorkProduct(RESOURCE, source._id.toString(), regionId) .withAction(progressListener -> { @@ -162,18 +160,18 @@ private SpatialDatasetSource handleUpload(Request req, Response res) { progressListener.beginTask("Validating files", 1); source.description = "From uploaded files: " + fileNames; source.validateAndSetDetails(uploadFormat, files); - spatialSourceCollection.insert(source); + spatialResourceCollection.insert(source); })); return source; } - private Collection deleteSourceSet(Request request, Response response) { - SpatialDatasetSource source = spatialSourceCollection.findPermittedByRequestParamId(request, response); + private Collection deleteResource (Request request, Response response) { + SpatialResource source = spatialResourceCollection.findPermittedByRequestParamId(request, response); // TODO delete files from storage // TODO delete referencing database records - spatialSourceCollection.delete(source); + spatialResourceCollection.delete(source); - return spatialSourceCollection.findPermitted( + return spatialResourceCollection.findPermitted( and(eq("regionId", request.params("regionId"))), request.attribute("accessGroup") ); @@ -184,9 +182,9 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/spatial", () -> { sparkService.post("", this::handleUpload, toJson); sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); - sparkService.get("/region/:regionId", this::getRegionDatasets, toJson); - sparkService.delete("/source/:_id", this::deleteSourceSet, toJson); - sparkService.get("/:_id", this::getSource, toJson); + sparkService.get("/region/:regionId", this::getRegionResources, toJson); + sparkService.delete("/source/:_id", this::deleteResource, toJson); + sparkService.get("/:_id", this::getResource, toJson); }); } } diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index 565827218..a1ee8d794 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -23,7 +23,7 @@ public static AggregationArea create (UserPermissions user, String name) { return new AggregationArea(user, name); } - public AggregationArea withSource (SpatialDatasetSource source) { + public AggregationArea withSource (SpatialResource source) { this.regionId = source.regionId; this.sourceId = source._id.toString(); return this; diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java b/src/main/java/com/conveyal/analysis/models/SpatialResource.java similarity index 82% rename from src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java rename to src/main/java/com/conveyal/analysis/models/SpatialResource.java index 7dd13d683..8f993c124 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDatasetSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialResource.java @@ -22,7 +22,12 @@ import static com.conveyal.file.FileCategory.RESOURCES; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; -public class SpatialDatasetSource extends BaseModel { +/** + * Record of a spatial source file (e.g. shapefile, CSV) that has been validated and can be processed into specific + * Conveyal formats (e.g. grids and other spatial layers). Eventually a general Resource class could be extended by + * SpatialResource, GtfsResource, and OsmResource? + */ +public class SpatialResource extends BaseModel { public String regionId; /** Description editable by end users */ public String description; @@ -32,22 +37,22 @@ public class SpatialDatasetSource extends BaseModel { /** Attributes, set only after validation (e.g. appropriate format for each feature's attributes) */ public List attributes; - private SpatialDatasetSource (UserPermissions userPermissions, String sourceName) { + private SpatialResource (UserPermissions userPermissions, String sourceName) { super(userPermissions, sourceName); } /** * No-arg constructor required for Mongo POJO serialization */ - public SpatialDatasetSource () { + public SpatialResource () { super(); } - public static SpatialDatasetSource create (UserPermissions userPermissions, String sourceName) { - return new SpatialDatasetSource(userPermissions, sourceName); + public static SpatialResource create (UserPermissions userPermissions, String sourceName) { + return new SpatialResource(userPermissions, sourceName); } - public SpatialDatasetSource withRegion (String regionId) { + public SpatialResource withRegion (String regionId) { this.regionId = regionId; return this; } @@ -89,7 +94,7 @@ private void fromShapefile (List files) { } } - public SpatialDatasetSource fromFiles (List fileItemList) { + public SpatialResource fromFiles (List fileItemList) { // TODO this.files from fileItemList; return this; } diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java b/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java similarity index 97% rename from src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java rename to src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java index e0858d2c0..7a46a02aa 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialDataset.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java @@ -12,9 +12,9 @@ import java.util.Set; /** - * Common methods for validating and processing uploaded spatial data files. + * Utility class with common methods for validating and processing uploaded spatial data files. */ -public class SpatialDataset { +public class SpatialLayers { /** * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java index 4002a290a..05ce4fbc4 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java @@ -4,7 +4,7 @@ import com.conveyal.analysis.models.Bundle; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.RegionalAnalysis; -import com.conveyal.analysis.models.SpatialDatasetSource; +import com.conveyal.analysis.models.SpatialResource; /** * There is some implicit and unenforced correspondence between these values and those in FileCategory, as well @@ -20,7 +20,7 @@ public static WorkProductType forModel (Object model) { if (model instanceof OpportunityDataset) return OPPORTUNITY_DATASET; if (model instanceof RegionalAnalysis) return REGIONAL_ANALYSIS; if (model instanceof AggregationArea) return AGGREGATION_AREA; - if (model instanceof SpatialDatasetSource) return RESOURCE; // TODO switch to spatial dataset source + if (model instanceof SpatialResource) return RESOURCE; // TODO switch to spatial dataset source throw new IllegalArgumentException("Unrecognized work product type."); } } From b80a1721550dd04c1f4c2078a49664058623c2af Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 30 Jun 2021 22:44:23 +0800 Subject: [PATCH 051/187] set stored file permissions to read only by user --- src/main/java/com/conveyal/file/LocalFileStorage.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/conveyal/file/LocalFileStorage.java b/src/main/java/com/conveyal/file/LocalFileStorage.java index b0b398356..1e20ffe55 100644 --- a/src/main/java/com/conveyal/file/LocalFileStorage.java +++ b/src/main/java/com/conveyal/file/LocalFileStorage.java @@ -9,6 +9,8 @@ import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Set; /** * This implementation of FileStorage stores files in a local directory hierarchy and does not mirror anything to @@ -56,6 +58,8 @@ public void moveIntoStorage(FileStorageKey key, File file) { LOG.info("Could not move {} because of FileSystem restrictions (probably NTFS). Copying instead.", file.getName()); } + // Set the file to be read-only and accessible only by the current user. + Files.setPosixFilePermissions(file.toPath(), Set.of(PosixFilePermission.OWNER_READ)); } catch (IOException e) { throw new RuntimeException(e); } From 1dd94e2a147b2eb25ed4c53d5e7c77af2f4dc8ee Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 30 Jun 2021 22:45:03 +0800 Subject: [PATCH 052/187] factor out duplicate code from FORBIDDEN case --- .../conveyal/analysis/components/HttpApi.java | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index a5315c728..ab0e7ef81 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -137,20 +137,7 @@ private spark.Service configureSparkService () { // Can we consolidate all these exception handlers and get rid of the hard-wired "BAD_REQUEST" parameters? sparkService.exception(AnalysisServerException.class, (e, request, response) -> { - // Include a stack trace, except when the error is known to be about unauthenticated or unauthorized access, - // in which case we don't want to leak information about the server to people scanning it for weaknesses. - if (e.type == AnalysisServerException.Type.UNAUTHORIZED || - e.type == AnalysisServerException.Type.FORBIDDEN - ){ - JSONObject body = new JSONObject(); - body.put("type", e.type.toString()); - body.put("message", e.message); - response.status(e.httpCode); - response.type("application/json"); - response.body(body.toJSONString()); - } else { - respondToException(e, request, response, e.type, e.message, e.httpCode); - } + respondToException(e, request, response, e.type, e.message, e.httpCode); }); sparkService.exception(IOException.class, (e, request, response) -> { @@ -182,8 +169,11 @@ private void respondToException(Exception e, Request request, Response response, JSONObject body = new JSONObject(); body.put("type", type.toString()); body.put("message", message); - body.put("stackTrace", errorEvent.stackTrace); - + // Include a stack trace except when the error is known to be about unauthenticated or unauthorized access, + // in which case we don't want to leak information about the server to people scanning it for weaknesses. + if (type != AnalysisServerException.Type.UNAUTHORIZED && type != AnalysisServerException.Type.FORBIDDEN) { + body.put("stackTrace", errorEvent.stackTrace); + } response.status(code); response.type("application/json"); response.body(body.toJSONString()); From 26af7b62953a02bbe4e5e31e73e822bf3f640166 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 13 Jul 2021 15:02:15 +0800 Subject: [PATCH 053/187] apply egress table preload logic in regional tasks in common use cases this should cause egress tables to be calculated before we fire the HandleRegionalEvent and begin routing, allowing better tracking of whether workers are still in a slow preparation phase. --- .../conveyal/r5/analyst/NetworkPreloader.java | 18 +++++++++++-- .../r5/analyst/cluster/AnalysisWorker.java | 3 +-- .../r5/transit/TransportNetworkCache.java | 26 ++++++++++--------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java b/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java index b95c49fa5..52e915133 100644 --- a/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java +++ b/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java @@ -72,7 +72,7 @@ public class NetworkPreloader extends AsyncLoader preloadData (AnalysisWorkerTask task) { return get(Key.forTask(task)); } + /** + * A blocking way to ensure the network and all linkages and precomputed tables are prepared in advance of routing. + * Note that this does not perform any blocking or locking of its own - any synchronization will be that of the + * underlying caches (synchronized methods on TransportNetworkCache or LinkedPointSet). It also bypasses the + * AsyncLoader locking that would usually allow only one buildValue operation at a time. All threads that call with + * similar tasks will make interleaved calls to setProgress (with superficial map synchronization). Other than + * causing a value to briefly revert from PRESENT to BUILDING this doesn't seem deeply problematic. + * This is provided specifically for regional tasks, to ensure that they remain in preloading mode while all this + * data is prepared. + */ + public TransportNetwork synchronousPreload (AnalysisWorkerTask task) { + return buildValue(Key.forTask(task)); + } + @Override protected TransportNetwork buildValue(Key key) { @@ -102,7 +116,7 @@ protected TransportNetwork buildValue(Key key) { // Now rebuild grid linkages as needed. One linkage per mode, and one cost table per egress mode. // Cost tables are slow to compute and not needed for access or direct legs, only egress modes. - // Note that we're able to pass a progress listener down into the EgressCostTable contruction process, + // Note that we're able to pass a progress listener down into the EgressCostTable construction process, // but not into the linkage process, because the latter is encapsulated as a Google/Caffeine // LoadingCache. We'll need some way to get LoadingCache's per-key locking, while still allowing a // progress listener specific to the single request. Perhaps this will mean registering 0..N diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index ee88a1272..fc5e0e6b6 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -462,8 +462,7 @@ protected void handleOneRegionalTask (RegionalTask task) throws Throwable { // Note we're completely bypassing the async loader here and relying on the older nested LoadingCaches. // If those are ever removed, the async loader will need a synchronous mode with per-path blocking (kind of // reinventing the wheel of LoadingCache) or we'll need to make preparation for regional tasks async. - TransportNetwork transportNetwork = networkPreloader.transportNetworkCache.getNetworkForScenario(task - .graphId, task.scenarioId); + TransportNetwork transportNetwork = networkPreloader.synchronousPreload(task); // Static site tasks do not specify destinations, but all other regional tasks should. // Load the PointSets based on the IDs (actually, full storage keys including IDs) in the task. diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java index b8587feaf..424f47b71 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java @@ -94,20 +94,22 @@ public void rememberScenario (Scenario scenario) { } /** - * Find or create a TransportNetwork for the scenario specified in a ProfileRequest. - * ProfileRequests may contain an embedded complete scenario, or it may contain only the ID of a scenario that - * must be fetched from S3. - * By design a particular scenario is always defined relative to a single base graph (it's never applied to multiple - * different base graphs). Therefore we can look up cached scenario networks based solely on their scenarioId - * rather than a compound key of (networkId, scenarioId). + * Find or create a TransportNetwork for the scenario specified in a ProfileRequest. ProfileRequests may contain an + * embedded complete scenario, or it may contain only the ID of a scenario that must be fetched from S3. By design + * a particular scenario is always defined relative to a single base graph (it's never applied to multiple different + * base graphs). Therefore we can look up cached scenario networks based solely on their scenarioId rather than a + * compound key of (networkId, scenarioId). * - * The fact that scenario networks are cached means that PointSet linkages will be automatically reused when + * The fact that scenario networks are cached means that PointSet linkages will be automatically reused. * TODO it seems to me that this method should just take a Scenario as its second parameter, and that resolving - * the scenario against caches on S3 or local disk should be pulled out into a separate function - * the problem is that then you resolve the scenario every time, even when the ID is enough to look up the already built network. - * So we need to pass the whole task in here, so either the ID or full scenario are visible. - * FIXME the fact that this whole thing is synchronized will cause each new scenario to be applied in sequence. - * I guess that's good as long as building distance tables is already parallelized. + * the scenario against caches on S3 or local disk should be pulled out into a separate function. + * The problem is that then you resolve the scenario every time, even when the ID is enough to look up the already + * built network. So we need to pass the whole task in here, so either the ID or full scenario are visible. + * + * Thread safety notes: This entire method is synchronized so access by multiple threads will be sequential. + * The first thread will have a chance to build and store the requested scenario before any others see it. + * This means each new scenario will be applied one after the other. This is probably OK as long as building egress + * tables is already parallelized. */ public synchronized TransportNetwork getNetworkForScenario (String networkId, String scenarioId) { // If the networkId is different than previous calls, a new network will be loaded. Its transient nested map From 64dddd7824ed30b039400f6145171c6a8c1779ec Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Wed, 4 Aug 2021 21:02:21 -0400 Subject: [PATCH 054/187] Move link to product page higher up in readme --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 6cb8b12d1..fced34b93 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,12 @@ # Conveyal R5 Routing Engine ## R5: Rapid Realistic Routing on Real-world and Reimagined networks - -R5 is Conveyal's routing engine for multimodal (transit/bike/walk/car) networks, with a particular focus on public transit. It is intended primarily for analysis applications (one-to-many trees, travel time matrices, and cumulative opportunities accessibility indicators). +R5 is the routing engine for [Conveyal](https://www.conveyal.com/learn), a web-based system that allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. See the [Conveyal user manual](https://docs.conveyal.com/) for more information. We refer to the routing method as "realistic" because it works by planning many trips at different departure times in a time window, which better reflects how people use transportation system than planning a single trip at an exact departure time. R5 handles both scheduled public transit and headway-based lines, using novel methods to characterize variation and uncertainty in travel times. We say "Real-world and Reimagined" networks because R5's networks are built from widely available open OSM and GTFS data describing baseline transportation systems, but R5 includes a system for applying light-weight patches to those networks for immediate, interactive scenario comparison. -R5 is a core component of [Conveyal Analysis](https://www.conveyal.com/learn), which allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. See the [methodology section](https://docs.conveyal.com/analysis/methodology) of the [Conveyal user manual](https://docs.conveyal.com/) for more information. - **Please note** that the Conveyal team does not provide technical support for third-party deployments of its analysis platform. We provide paid subscriptions to a cloud-based deployment of this system, which performs these complex calculations hundreds of times faster using a compute cluster. This project is open source primarily to ensure transparency and reproducibility in public planning and decision making processes, and in hopes that it may help researchers, students, and potential collaborators to understand and build upon our methodology. ## Methodology From 10ac876dec531fbe6cf2aff7f65c38386b3263d0 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 15:30:00 +0800 Subject: [PATCH 055/187] Improve error reporting in TimeCsvResultWriter 1. task.destinationPointSets is usually null on the backend unless we've got a single freeform pointset. we were checking its length without checking for null, giving confusing NPEs instead of intended message. 2. Drop incoming work results if job is not "active" (errored or complete). This prevents spamming us with dozens (or more) repeated error messages from all outstanding tasks. --- .../analysis/components/broker/Broker.java | 16 ++++++++++------ .../analysis/results/TimeCsvResultWriter.java | 10 ++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/broker/Broker.java b/src/main/java/com/conveyal/analysis/components/broker/Broker.java index e9c0221da..1358ced95 100644 --- a/src/main/java/com/conveyal/analysis/components/broker/Broker.java +++ b/src/main/java/com/conveyal/analysis/components/broker/Broker.java @@ -338,7 +338,12 @@ public synchronized void markTaskCompleted (Job job, int taskId) { } } - /** This method ensures synchronization of writes to Jobs from the unsynchronized worker poll HTTP handler. */ + /** + * When job.errors is non-empty, job.isErrored() becomes true and job.isActive() becomes false. + * The Job will stop delivering tasks, allowing workers to shut down, but will continue to exist allowing the user + * to see the error message. User will then need to manually delete it, which will remove the result assembler. + * This method ensures synchronization of writes to Jobs from the unsynchronized worker poll HTTP handler. + */ private synchronized void recordJobError (Job job, String error) { job.errors.add(error); } @@ -446,14 +451,13 @@ public void handleRegionalWorkResult(RegionalWorkResult workResult) { job = findJob(workResult.jobId); assembler = resultAssemblers.get(workResult.jobId); } - if (job == null || assembler == null) { - // This will happen naturally for all delivered tasks when a job is deleted by the user. - LOG.debug("Received result for unrecognized job ID {}, discarding.", workResult.jobId); + if (job == null || assembler == null || !job.isActive()) { + // This will happen naturally for all delivered tasks when a job is deleted by the user or after it errors. + LOG.debug("Received result for unrecognized, deleted, or inactive job ID {}, discarding.", workResult.jobId); return; } if (workResult.error != null) { - // Just record the error reported by the worker and don't pass the result on to regional result assembly. - // The Job will stop delivering tasks, allowing workers to shut down. User will need to manually delete it. + // Record any error reported by the worker and don't pass the (bad) result on to regional result assembly. recordJobError(job, workResult.error); return; } diff --git a/src/main/java/com/conveyal/analysis/results/TimeCsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/TimeCsvResultWriter.java index 6d9137f5a..144da7713 100644 --- a/src/main/java/com/conveyal/analysis/results/TimeCsvResultWriter.java +++ b/src/main/java/com/conveyal/analysis/results/TimeCsvResultWriter.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.results; import com.conveyal.file.FileStorage; +import com.conveyal.r5.analyst.FreeFormPointSet; import com.conveyal.r5.analyst.cluster.RegionalTask; import com.conveyal.r5.analyst.cluster.RegionalWorkResult; @@ -8,6 +9,7 @@ import java.util.ArrayList; import java.util.List; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; public class TimeCsvResultWriter extends CsvResultWriter { @@ -31,9 +33,13 @@ public String[] columnHeaders () { */ @Override protected void checkDimension (RegionalWorkResult workResult) { - // This CSV writer expects only a single freeform destination pointset. // TODO handle multiple destination pointsets at once? - checkState(task.destinationPointSets.length == 1); + checkState( + task.destinationPointSets != null && + task.destinationPointSets.length == 1 && + task.destinationPointSets[0] instanceof FreeFormPointSet, + "Time CSV writer expects only a single freeform destination pointset." + ); // In one-to-one mode, we expect only one value per origin, the destination point at the same pointset index as // the origin point. Otherwise, for each origin, we expect one value per destination. final int nDestinations = task.oneToOne ? 1 : task.destinationPointSets[0].featureCount(); From aa035293277428f653e9660df3d815bb99dc0b02 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Wed, 11 Aug 2021 16:06:58 +0800 Subject: [PATCH 056/187] Remove unused controllers --- .../components/BackendComponents.java | 8 - .../components/eventbus/SinglePointEvent.java | 14 +- .../controllers/BrokerController.java | 27 ++- .../analysis/controllers/GTFSController.java | 28 ++- .../controllers/GTFSGraphQLController.java | 208 ------------------ .../controllers/ModificationController.java | 96 -------- .../controllers/ProjectController.java | 175 --------------- .../RegionalAnalysisController.java | 21 +- .../controllers/TimetableController.java | 86 -------- .../analysis/models/AnalysisRequest.java | 77 ++----- .../com/conveyal/analysis/models/Project.java | 3 - .../analysis/models/RegionalAnalysis.java | 1 + .../analysis/persistence/MongoMap.java | 3 +- 13 files changed, 87 insertions(+), 660 deletions(-) delete mode 100644 src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java delete mode 100644 src/main/java/com/conveyal/analysis/controllers/ModificationController.java delete mode 100644 src/main/java/com/conveyal/analysis/controllers/ProjectController.java delete mode 100644 src/main/java/com/conveyal/analysis/controllers/TimetableController.java diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 16bfd1ac7..59cdb9f07 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -8,14 +8,10 @@ import com.conveyal.analysis.controllers.BundleController; import com.conveyal.analysis.controllers.FileStorageController; import com.conveyal.analysis.controllers.GTFSController; -import com.conveyal.analysis.controllers.GTFSGraphQLController; import com.conveyal.analysis.controllers.GtfsTileController; import com.conveyal.analysis.controllers.HttpController; -import com.conveyal.analysis.controllers.ModificationController; import com.conveyal.analysis.controllers.OpportunityDatasetController; -import com.conveyal.analysis.controllers.ProjectController; import com.conveyal.analysis.controllers.RegionalAnalysisController; -import com.conveyal.analysis.controllers.TimetableController; import com.conveyal.analysis.controllers.UserActivityController; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; import com.conveyal.analysis.persistence.AnalysisDB; @@ -87,15 +83,11 @@ public List standardHttpControllers () { return Lists.newArrayList( // These handlers are at paths beginning with /api // and therefore subject to authentication and authorization. - new ModificationController(), - new ProjectController(), - new GTFSGraphQLController(gtfsCache), new GTFSController(gtfsCache), new BundleController(this), new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor), new RegionalAnalysisController(broker, fileStorage), new AggregationAreaController(fileStorage), - new TimetableController(), new FileStorageController(fileStorage, database), // This broker controller registers at least one handler at URL paths beginning with /internal, which // is exempted from authentication and authorization, but should be hidden from the world diff --git a/src/main/java/com/conveyal/analysis/components/eventbus/SinglePointEvent.java b/src/main/java/com/conveyal/analysis/components/eventbus/SinglePointEvent.java index a223a5a3b..bbce44c18 100644 --- a/src/main/java/com/conveyal/analysis/components/eventbus/SinglePointEvent.java +++ b/src/main/java/com/conveyal/analysis/components/eventbus/SinglePointEvent.java @@ -9,16 +9,16 @@ public class SinglePointEvent extends Event { // but also has a CRC since the scenario with a given index can change over time. public final String scenarioId; - public final String projectId; + public final String bundleId; - public final int variant; + public final String regionId; public final int durationMsec; - public SinglePointEvent (String scenarioId, String projectId, int variant, int durationMsec) { + public SinglePointEvent (String scenarioId, String bundleId, String regionId, int durationMsec) { this.scenarioId = scenarioId; - this.projectId = projectId; - this.variant = variant; + this.bundleId = bundleId; + this.regionId = regionId; this.durationMsec = durationMsec; } @@ -26,8 +26,8 @@ public SinglePointEvent (String scenarioId, String projectId, int variant, int d public String toString () { return "SinglePointEvent{" + "scenarioId='" + scenarioId + '\'' + - ", projectId='" + projectId + '\'' + - ", variant=" + variant + + ", regionId='" + regionId + '\'' + + ", bundleId=" + bundleId + ", durationMsec=" + durationMsec + ", user='" + user + '\'' + ", accessGroup=" + accessGroup + diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 39f9cd18f..455c339c4 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -10,8 +10,8 @@ import com.conveyal.analysis.components.eventbus.SinglePointEvent; import com.conveyal.analysis.models.AnalysisRequest; import com.conveyal.analysis.models.Bundle; +import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; -import com.conveyal.analysis.models.Project; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.HttpStatus; import com.conveyal.analysis.util.JsonUtil; @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.mongodb.QueryBuilder; -import com.sun.net.httpserver.Headers; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -49,6 +48,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static com.conveyal.r5.common.Util.notNullOrEmpty; import static com.google.common.base.Preconditions.checkNotNull; @@ -132,10 +132,19 @@ private Object singlePoint(Request request, Response response) { final String accessGroup = request.attribute("accessGroup"); final String userEmail = request.attribute("email"); final long startTimeMsec = System.currentTimeMillis(); - AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); - Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, accessGroup); + + final AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); + + final List modifications = Persistence.modifications.findPermitted( + QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(), + accessGroup); + // Transform the analysis UI/backend task format into a slightly different type for R5 workers. - TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest.populateTask(new TravelTimeSurfaceTask(), project); + TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest.populateTask( + new TravelTimeSurfaceTask(), + modifications.stream().map(Modification::toR5).collect(Collectors.toList()) + ); + // If destination opportunities are supplied, prepare to calculate accessibility worker-side if (notNullOrEmpty(analysisRequest.destinationPointSetIds)){ // Look up all destination opportunity data sets from the database and derive their storage keys. @@ -170,7 +179,7 @@ private Object singlePoint(Request request, Response response) { String address = broker.getWorkerAddress(workerCategory); if (address == null) { // There are no workers that can handle this request. Request some. - WorkerTags workerTags = new WorkerTags(accessGroup, userEmail, project._id, project.regionId); + WorkerTags workerTags = new WorkerTags(accessGroup, userEmail, analysisRequest.projectId, analysisRequest.regionId); broker.createOnDemandWorkerInCategory(workerCategory, workerTags); // No workers exist. Kick one off and return "service unavailable". response.header("Retry-After", "30"); @@ -207,9 +216,9 @@ private Object singlePoint(Request request, Response response) { if (response.status() == 200) { int durationMsec = (int) (System.currentTimeMillis() - startTimeMsec); eventBus.send(new SinglePointEvent( - task.scenarioId, - analysisRequest.projectId, - analysisRequest.variantIndex, + analysisRequest.scenarioId, + analysisRequest.bundleId, + analysisRequest.regionId, durationMsec ).forUser(userEmail, accessGroup) ); diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index c44cd53db..3c8d9b0f0 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -4,7 +4,6 @@ import com.conveyal.analysis.persistence.Persistence; import com.conveyal.gtfs.GTFSCache; import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; import com.conveyal.gtfs.model.Pattern; import com.conveyal.gtfs.model.Route; import com.conveyal.gtfs.model.Stop; @@ -14,12 +13,12 @@ import spark.Request; import spark.Response; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.conveyal.analysis.util.JsonUtil.toJson; @@ -44,7 +43,10 @@ static class RouteAPIResponse { RouteAPIResponse(Route route) { id = route.route_id; color = route.route_color; - name = String.join(" ", route.route_short_name + "", route.route_long_name + "").trim(); + String tempName = ""; + if (route.route_short_name != null) tempName += route.route_short_name; + if (route.route_long_name != null) tempName += " " + route.route_long_name; + name = tempName.trim(); type = route.route_type; } } @@ -60,15 +62,26 @@ private List getRoutes(Request req, Response res) { static class PatternAPIResponse { public final String id; - public final com.vividsolutions.jts.geom.LineString geometry; + public final String name; + public final GeoJSONLineString geometry; public final List orderedStopIds; public final List associatedTripIds; PatternAPIResponse(Pattern pattern) { id = pattern.pattern_id; - geometry = pattern.geometry; + name = pattern.name; + geometry = serialize(pattern.geometry); orderedStopIds = pattern.orderedStops; associatedTripIds = pattern.associatedTrips; } + + GeoJSONLineString serialize (com.vividsolutions.jts.geom.LineString geometry) { + GeoJSONLineString ret = new GeoJSONLineString(); + ret.coordinates = Stream.of(geometry.getCoordinates()) + .map(c -> new double[] { c.x, c.y }) + .toArray(double[][]::new); + + return ret; + } } private List getPatternsForRoute (Request req, Response res) { @@ -170,4 +183,9 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/trips", this::getTripsForRoute, toJson); sparkService.get("/api/gtfs/:_id/:feedId/stops", this::getStops, toJson); } + + private static class GeoJSONLineString { + public final String type = "LineString"; + public double[][] coordinates; + } } diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java deleted file mode 100644 index a3ba0a264..000000000 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java +++ /dev/null @@ -1,208 +0,0 @@ -package com.conveyal.analysis.controllers; - -import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.models.Bundle; -import com.conveyal.analysis.persistence.Persistence; -import com.conveyal.analysis.util.JsonUtil; -import com.conveyal.gtfs.GTFSCache; -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.api.graphql.fetchers.RouteFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.StopFetcher; -import com.conveyal.gtfs.model.FeedInfo; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.util.concurrent.UncheckedExecutionException; -import com.mongodb.QueryBuilder; -import graphql.ExceptionWhileDataFetching; -import graphql.ExecutionResult; -import graphql.GraphQL; -import graphql.GraphQLError; -import graphql.execution.ExecutionContext; -import graphql.schema.DataFetchingEnvironment; -import graphql.schema.GraphQLEnumType; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.conveyal.analysis.controllers.BundleController.setBundleServiceDates; -import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.conveyal.gtfs.api.graphql.GraphQLGtfsSchema.routeType; -import static com.conveyal.gtfs.api.graphql.GraphQLGtfsSchema.stopType; -import static com.conveyal.gtfs.api.util.GraphQLUtil.multiStringArg; -import static com.conveyal.gtfs.api.util.GraphQLUtil.string; -import static graphql.Scalars.GraphQLLong; -import static graphql.schema.GraphQLEnumType.newEnum; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * GraphQL interface for fetching GTFS feed contents (generally used for scenario editing). - * For now it just wraps the GTFS API graphql response with a bundle object. - */ -public class GTFSGraphQLController implements HttpController { - - private static final Logger LOG = LoggerFactory.getLogger(GTFSGraphQLController.class); - - private final GTFSCache gtfsCache; - - public GTFSGraphQLController (GTFSCache gtfsCache) { - this.gtfsCache = gtfsCache; - } - - private Object handleQuery (Request req, Response res) throws IOException { - res.type("application/json"); - - Map variables = JsonUtil.objectMapper.readValue(req.queryParams("variables"), new TypeReference>() { - }); - - QueryContext context = new QueryContext(); - context.accessGroup = req.attribute("accessGroup"); - - ExecutionResult er = graphql.execute(req.queryParams("query"), null, context, variables); - - List errs = er.getErrors(); - errs.addAll(context.getErrors()); - if (!errs.isEmpty()) { - throw AnalysisServerException.graphQL(errs); - } - - return er.getData(); - } - - /** Special feed type that also includes checksum */ - public GraphQLObjectType feedType = newObject() - .name("feed") - .field(string("feed_id")) - .field(string("feed_publisher_name")) - .field(string("feed_publisher_url")) - .field(string("feed_lang")) - .field(string("feed_version")) - // We have a custom wrapped GTFS Entity type for FeedInfo that includes feed checksum - .field(newFieldDefinition() - .name("checksum") - .type(GraphQLLong) - .dataFetcher(env -> ((WrappedFeedInfo) env.getSource()).checksum) - .build() - ) - .field(newFieldDefinition() - .name("routes") - .type(new GraphQLList(routeType)) - .argument(multiStringArg("route_id")) - .dataFetcher(RouteFetcher::fromFeed) - .build() - ) - .field(newFieldDefinition() - .name("stops") - .type(new GraphQLList(stopType)) - .dataFetcher(StopFetcher::fromFeed) - .build() - ) - .build(); - - private GraphQLEnumType bundleStatus = newEnum() - .name("status") - .value("PROCESSING_GTFS", Bundle.Status.PROCESSING_GTFS) - .value("PROCESSING_OSM", Bundle.Status.PROCESSING_OSM) - .value("ERROR", Bundle.Status.ERROR) - .value("DONE", Bundle.Status.DONE) - .build(); - - private GraphQLObjectType bundleType = newObject() - .name("bundle") - .field(string("_id")) - .field(string("name")) - .field(newFieldDefinition() - .name("status") - .type(bundleStatus) - .dataFetcher((env) -> ((Bundle) env.getSource()).status) - .build() - ) - .field(newFieldDefinition() - .name("feeds") - .type(new GraphQLList(feedType)) - .dataFetcher(this::fetchFeeds) - .build() - ) - .build(); - - private GraphQLObjectType bundleQuery = newObject() - .name("bundleQuery") - .field(newFieldDefinition() - .name("bundle") - .type(new GraphQLList(bundleType)) - .argument(multiStringArg("bundle_id")) - .dataFetcher(this::fetchBundle) - .build() - ) - .build(); - - public GraphQLSchema schema = GraphQLSchema.newSchema().query(bundleQuery).build(); - private GraphQL graphql = new GraphQL(schema); - - private Collection fetchBundle(DataFetchingEnvironment environment) { - QueryContext context = (QueryContext) environment.getContext(); - return Persistence.bundles.findPermitted( - QueryBuilder.start("_id").in(environment.getArgument("bundle_id")).get(), - context.accessGroup - ); - } - - /** - * Returns a list of wrapped FeedInfo objects. These objects have all fields null (default), except feed_id, which - * is set from the cached copy of each requested feed. This method previously returned the fully populated FeedInfo - * objects, but incremental changes led to incompatibilities with Analysis (see analysis-internal #102). The - * current implementation is a stopgap for backward compatibility. - */ - private List> fetchFeeds(DataFetchingEnvironment environment) { - Bundle bundle = (Bundle) environment.getSource(); - ExecutionContext context = (ExecutionContext) environment.getContext(); - - // Old bundles were created without computing the service start and end dates. Will only compute if needed. - try { - setBundleServiceDates(bundle, gtfsCache); - } catch (Exception e) { - context.addError(new ExceptionWhileDataFetching(e)); - } - - return bundle.feeds.stream() - .map(summary -> { - String bundleScopedFeedId = Bundle.bundleScopeFeedId(summary.feedId, bundle.feedGroupId); - try { - GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); - FeedInfo ret = new FeedInfo(); - ret.feed_id = feed.feedId; - return new WrappedFeedInfo(summary.bundleScopedFeedId, ret, summary.checksum); - } catch (UncheckedExecutionException nsee) { - Exception e = new Exception(String.format("Feed %s does not exist in the cache.", summary.name), nsee); - context.addError(new ExceptionWhileDataFetching(e)); - return null; - } catch (Exception e) { - context.addError(new ExceptionWhileDataFetching(e)); - return null; - } - }) - .collect(Collectors.toList()); - } - - @Override - public void registerEndpoints (spark.Service sparkService) { - // TODO make this `post` as per GraphQL convention - sparkService.get("/api/graphql", this::handleQuery, toJson); - } - - /** Context for a graphql query. Currently contains authorization info. */ - public static class QueryContext extends ExecutionContext { - public String accessGroup; - } - -} diff --git a/src/main/java/com/conveyal/analysis/controllers/ModificationController.java b/src/main/java/com/conveyal/analysis/controllers/ModificationController.java deleted file mode 100644 index b4b3c68e8..000000000 --- a/src/main/java/com/conveyal/analysis/controllers/ModificationController.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.conveyal.analysis.controllers; - -import com.conveyal.analysis.models.AbstractTimetable; -import com.conveyal.analysis.models.AddTripPattern; -import com.conveyal.analysis.models.ConvertToFrequency; -import com.conveyal.analysis.models.Modification; -import com.conveyal.analysis.persistence.Persistence; -import org.bson.types.ObjectId; -import spark.Request; -import spark.Response; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.conveyal.analysis.util.JsonUtil.toJson; - -public class ModificationController implements HttpController { - - public ModificationController () { - // NO COMPONENT DEPENDENCIES - // Eventually Persistence will be a component (AnalysisDatabase) instead of static. - } - - private Modification getModification (Request req, Response res) { - return Persistence.modifications.findByIdFromRequestIfPermitted(req); - } - - private Modification create (Request request, Response response) throws IOException { - return Persistence.modifications.createFromJSONRequest(request); - } - - private Modification update (Request request, Response response) throws IOException { - return Persistence.modifications.updateFromJSONRequest(request); - } - - private Modification deleteModification (Request req, Response res) { - return Persistence.modifications.removeIfPermitted(req.params("_id"), req.attribute("accessGroup")); - } - - private void mapPhaseIds (List timetables, String oldModificationId, String newModificationId) { - Map idPairs = new HashMap(); - timetables.forEach(tt -> { - String newId = ObjectId.get().toString(); - idPairs.put(tt._id, newId); - tt._id = newId; - }); - - timetables - .stream() - .filter(tt -> tt.phaseFromTimetable != null && tt.phaseFromTimetable.length() > 0) - .filter(tt -> tt.phaseFromTimetable.contains(oldModificationId)) - .forEach(tt -> { - String oldTTId = tt.phaseFromTimetable.split(":")[1]; - tt.phaseFromTimetable = newModificationId + ":" + idPairs.get(oldTTId); - }); - } - - private Modification copyModification (Request req, Response res) { - Modification modification = Persistence.modifications.findByIdFromRequestIfPermitted(req); - - String oldId = modification._id; - Modification clone = Persistence.modifications.create(modification); - - // Matched up the phased entries and timetables - if (modification.getType().equals(AddTripPattern.type)) { - mapPhaseIds((List)(List)((AddTripPattern) clone).timetables, oldId, clone._id); - } else if (modification.getType().equals(ConvertToFrequency.type)) { - mapPhaseIds((List)(List)((ConvertToFrequency) clone).entries, oldId, clone._id); - } - - // Set `name` to include "(copy)" - clone.name = clone.name + " (copy)"; - - // Set `updateBy` manually, `createdBy` stays with the original modification author - clone.updatedBy = req.attribute("email"); - - // Update the clone - return Persistence.modifications.put(clone); - } - - @Override - public void registerEndpoints (spark.Service sparkService) { - sparkService.path("/api/modification", () -> { - sparkService.get("/:_id", this::getModification, toJson); - sparkService.post("/:_id/copy", this::copyModification, toJson); - sparkService.post("", this::create, toJson); - // Handle HTTP OPTIONS request to provide any configured CORS headers. - sparkService.options("", (q, s) -> ""); - sparkService.put("/:_id", this::update, toJson); - sparkService.options("/:_id", (q, s) -> ""); - sparkService.delete("/:_id", this::deleteModification, toJson); - }); - } -} diff --git a/src/main/java/com/conveyal/analysis/controllers/ProjectController.java b/src/main/java/com/conveyal/analysis/controllers/ProjectController.java deleted file mode 100644 index cd7cb0440..000000000 --- a/src/main/java/com/conveyal/analysis/controllers/ProjectController.java +++ /dev/null @@ -1,175 +0,0 @@ -package com.conveyal.analysis.controllers; - -import com.conveyal.analysis.models.AddTripPattern; -import com.conveyal.analysis.models.ConvertToFrequency; -import com.conveyal.analysis.models.Modification; -import com.conveyal.analysis.models.Project; -import com.conveyal.analysis.persistence.Persistence; -import com.mongodb.QueryBuilder; -import org.bson.types.ObjectId; -import spark.Request; -import spark.Response; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.conveyal.analysis.util.JsonUtil.toJson; - -public class ProjectController implements HttpController { - - public ProjectController () { - // NO COMPONENT DEPENDENCIES - // Eventually persistence will be a component (AnalysisDatabase) instead of static. - } - - private Project findById(Request req, Response res) { - return Persistence.projects.findByIdFromRequestIfPermitted(req); - } - - private Collection getAllProjects (Request req, Response res) { - return Persistence.projects.findPermitted( - QueryBuilder.start("regionId").is(req.params("region")).get(), - req.attribute("accessGroup") - ); - } - - private Project create(Request req, Response res) throws IOException { - return Persistence.projects.createFromJSONRequest(req); - } - - private Project update(Request req, Response res) throws IOException { - return Persistence.projects.updateFromJSONRequest(req); - } - - private Collection modifications (Request req, Response res) { - return Persistence.modifications.findPermitted( - QueryBuilder.start("projectId").is(req.params("_id")).get(), - req.attribute("accessGroup") - ); - } - - private Collection importModifications (Request req, Response res) { - final String importId = req.params("_importId"); - final String newId = req.params("_id"); - final String accessGroup = req.attribute("accessGroup"); - final Project project = Persistence.projects.findByIdIfPermitted(newId, accessGroup); - final Project importProject = Persistence.projects.findByIdIfPermitted(importId, accessGroup); - final boolean bundlesAreNotEqual = !project.bundleId.equals(importProject.bundleId); - - QueryBuilder query = QueryBuilder.start("projectId").is(importId); - if (bundlesAreNotEqual) { - // Different bundle? Only copy add trip modifications - query = query.and("type").is("add-trip-pattern"); - } - final Collection modifications = Persistence.modifications.findPermitted(query.get(), accessGroup); - - // This would be a lot easier if we just used the actual `_id`s and dealt with it elsewhere when searching. They - // should be unique anyways. Hmmmmmmmmmmmm. Trade offs. - // Need to make two passes to create all the pairs and rematch for phasing - final Map modificationIdPairs = new HashMap<>(); - final Map timetableIdPairs = new HashMap<>(); - - return modifications - .stream() - .map(modification -> { - String oldModificationId = modification._id; - Modification clone = Persistence.modifications.create(modification); - modificationIdPairs.put(oldModificationId, clone._id); - - // Change the projectId, most important part! - clone.projectId = newId; - - // Set `name` to include "(import)" - clone.name = clone.name + " (import)"; - - // Set `updatedBy` by manually, `createdBy` stays with the original author - clone.updatedBy = req.attribute("email"); - - // Matched up the phased entries and timetables - if (modification.getType().equals(AddTripPattern.type)) { - if (bundlesAreNotEqual) { - // Remove references to real stops in the old bundle - ((AddTripPattern) clone).segments.forEach(segment -> { - segment.fromStopId = null; - segment.toStopId = null; - }); - - // Remove all phasing - ((AddTripPattern) clone).timetables.forEach(tt -> { - tt.phaseFromTimetable = null; - tt.phaseAtStop = null; - tt.phaseFromStop = null; - }); - } - - ((AddTripPattern) clone).timetables.forEach(tt -> { - String oldTTId = tt._id; - tt._id = new ObjectId().toString(); - timetableIdPairs.put(oldTTId, tt._id); - }); - } else if (modification.getType().equals(ConvertToFrequency.type)) { - ((ConvertToFrequency) clone).entries.forEach(tt -> { - String oldTTId = tt._id; - tt._id = new ObjectId().toString(); - timetableIdPairs.put(oldTTId, tt._id); - }); - } - - return clone; - }) - .collect(Collectors.toList()) - .stream() - .map(modification -> { - // A second pass is needed to map the phase pairs - if (modification.getType().equals(AddTripPattern.type)) { - ((AddTripPattern) modification).timetables.forEach(tt -> { - String pft = tt.phaseFromTimetable; - if (pft != null && pft.length() > 0) { - String[] pfts = pft.split(":"); - tt.phaseFromTimetable = modificationIdPairs.get(pfts[0]) + ":" + timetableIdPairs.get(pfts[1]); - } - }); - } else if (modification.getType().equals(ConvertToFrequency.type)) { - ((ConvertToFrequency) modification).entries.forEach(tt -> { - String pft = tt.phaseFromTimetable; - if (pft != null && pft.length() > 0) { - String[] pfts = pft.split(":"); - tt.phaseFromTimetable = modificationIdPairs.get(pfts[0]) + ":" + timetableIdPairs.get(pfts[1]); - } - }); - } - - return Persistence.modifications.put(modification); - }) - .collect(Collectors.toList()); - } - - private Project deleteProject (Request req, Response res) { - return Persistence.projects.removeIfPermitted(req.params("_id"), req.attribute("accessGroup")); - } - - public Collection getProjects (Request req, Response res) { - return Persistence.projects.findPermittedForQuery(req); - } - - @Override - public void registerEndpoints (spark.Service sparkService) { - sparkService.path("/api/project", () -> { - sparkService.get("", this::getProjects, toJson); - sparkService.get("/:_id", this::findById, toJson); - sparkService.get("/:_id/modifications", this::modifications, toJson); - sparkService.post("/:_id/import/:_importId", this::importModifications, toJson); - sparkService.post("", this::create, toJson); - sparkService.options("", (q, s) -> ""); - sparkService.put("/:_id", this::update, toJson); - sparkService.delete("/:_id", this::deleteProject, toJson); - sparkService.options("/:_id", (q, s) -> ""); - }); - // Note this one is under the /api/region path, not /api/project - sparkService.get("/api/region/:region/projects", this::getAllProjects); // TODO response transformer? - } - -} diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 894c62ac0..8eec1ee25 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -5,6 +5,7 @@ import com.conveyal.analysis.components.broker.Broker; import com.conveyal.analysis.components.broker.JobStatus; import com.conveyal.analysis.models.AnalysisRequest; +import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.Project; import com.conveyal.analysis.models.RegionalAnalysis; @@ -40,6 +41,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.util.JsonUtil.toJson; @@ -377,11 +379,20 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro analysisRequest.percentiles = DEFAULT_REGIONAL_PERCENTILES; } + final List modificationIds = new ArrayList<>(); + List modifications = Persistence.modifications.findPermitted( + QueryBuilder.start("_id").in(modificationIds).get(), + accessGroup + ); + // Create an internal RegionalTask and RegionalAnalysis from the AnalysisRequest sent by the client. - Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, accessGroup); + // TODO now this is setting cutoffs and percentiles in the regional (template) task. // why is some stuff set in this populate method, and other things set here in the caller? - RegionalTask task = (RegionalTask) analysisRequest.populateTask(new RegionalTask(), project); + RegionalTask task = (RegionalTask) analysisRequest.populateTask( + new RegionalTask(), + modifications.stream().map(Modification::toR5).collect(Collectors.toList()) + ); // Set the destination PointSets, which are required for all non-Taui regional requests. if (! analysisRequest.makeTauiSite) { @@ -471,13 +482,13 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro regionalAnalysis.width = task.width; regionalAnalysis.accessGroup = accessGroup; - regionalAnalysis.bundleId = project.bundleId; + regionalAnalysis.bundleId = analysisRequest.bundleId; regionalAnalysis.createdBy = email; regionalAnalysis.destinationPointSetIds = analysisRequest.destinationPointSetIds; regionalAnalysis.name = analysisRequest.name; regionalAnalysis.projectId = analysisRequest.projectId; - regionalAnalysis.regionId = project.regionId; - regionalAnalysis.variant = analysisRequest.variantIndex; + regionalAnalysis.regionId = analysisRequest.regionId; + regionalAnalysis.scenarioId = analysisRequest.scenarioId; regionalAnalysis.workerVersion = analysisRequest.workerVersion; regionalAnalysis.zoom = task.zoom; diff --git a/src/main/java/com/conveyal/analysis/controllers/TimetableController.java b/src/main/java/com/conveyal/analysis/controllers/TimetableController.java deleted file mode 100644 index 4bccee924..000000000 --- a/src/main/java/com/conveyal/analysis/controllers/TimetableController.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.conveyal.analysis.controllers; - -import com.conveyal.analysis.models.AddTripPattern; -import com.conveyal.analysis.models.Modification; -import com.conveyal.analysis.models.Project; -import com.conveyal.analysis.models.Region; -import com.conveyal.analysis.persistence.Persistence; -import com.conveyal.analysis.util.JsonUtil; -import com.mongodb.QueryBuilder; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; - -import java.util.Collection; -import java.util.List; - -/** - * Created by evan siroky on 5/3/18. - */ -public class TimetableController implements HttpController { - - private static final Logger LOG = LoggerFactory.getLogger(TimetableController.class); - - public TimetableController () { - // NO COMPONENT DEPENDENCIES - // Eventually persistence will be a component (AnalysisDatabase) instead of static. - } - - // Unlike many other methods, rather than serializing a Java type to JSON, - // this builds up the JSON using a map-like API. It looks like we're using org.json.simple here - // instead of Jackson which we're using elsewhere. We should use one or the other. - private String getTimetables (Request req, Response res) { - JSONArray json = new JSONArray(); - Collection regions = Persistence.regions.findAllForRequest(req); - - for (Region region : regions) { - JSONObject r = new JSONObject(); - r.put("_id", region._id); - r.put("name", region.name); - JSONArray regionProjects = new JSONArray(); - List projects = Persistence.projects.find(QueryBuilder.start("regionId").is(region._id).get()).toArray(); - for (Project project : projects) { - JSONObject p = new JSONObject(); - p.put("_id", project._id); - p.put("name", project.name); - JSONArray projectModifications = new JSONArray(); - List modifications = Persistence.modifications.find( - QueryBuilder.start("projectId").is(project._id).and("type").is("add-trip-pattern").get() - ).toArray(); - for (Modification modification : modifications) { - AddTripPattern tripPattern = (AddTripPattern) modification; - JSONObject m = new JSONObject(); - m.put("_id", modification._id); - m.put("name", modification.name); - m.put("segments", JsonUtil.objectMapper.valueToTree(tripPattern.segments)); - JSONArray modificationTimetables = new JSONArray(); - for (AddTripPattern.Timetable timetable : tripPattern.timetables) { - modificationTimetables.add(JsonUtil.objectMapper.valueToTree(timetable)); - } - m.put("timetables", modificationTimetables); - if (modificationTimetables.size() > 0) { - projectModifications.add(m); - } - } - p.put("modifications", projectModifications); - if (projectModifications.size() > 0) { - regionProjects.add(p); - } - } - r.put("projects", regionProjects); - if (regionProjects.size() > 0) { - json.add(r); - } - } - - return json.toString(); - } - - @Override - public void registerEndpoints (spark.Service sparkService) { - sparkService.get("/api/timetables", this::getTimetables); - } -} diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 7eb39b629..52e845b8f 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -1,7 +1,6 @@ package com.conveyal.analysis.models; import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.persistence.Persistence; import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; import com.conveyal.r5.analyst.decay.DecayFunction; @@ -12,7 +11,6 @@ import com.conveyal.r5.api.util.LegMode; import com.conveyal.r5.api.util.TransitModes; import com.conveyal.r5.common.JsonUtilities; -import com.mongodb.QueryBuilder; import java.time.LocalDate; import java.util.ArrayList; @@ -27,13 +25,16 @@ * sends/forwards to R5 workers (see {@link AnalysisWorkerTask}), though it has many of the same fields. */ public class AnalysisRequest { - private static int MIN_ZOOM = 9; private static int MAX_ZOOM = 12; private static int MAX_GRID_CELLS = 5_000_000; + public String regionId; public String projectId; - public int variantIndex; + public String scenarioId; + + public String bundleId; + public List modificationIds = new ArrayList<>(); public String workerVersion; public String accessModes; @@ -148,23 +149,6 @@ public class AnalysisRequest { */ public DecayFunction decayFunction; - /** - * Get all of the modifications for a project id that are in the Variant and map them to their - * corresponding r5 mod - */ - private static List modificationsForProject ( - String accessGroup, - String projectId, - int variantIndex) - { - return Persistence.modifications - .findPermitted(QueryBuilder.start("projectId").is(projectId).get(), accessGroup) - .stream() - .filter(m -> variantIndex < m.variants.length && m.variants[variantIndex]) - .map(com.conveyal.analysis.models.Modification::toR5) - .collect(Collectors.toList()); - } - /** * Finds the modifications for the specified project and variant, maps them to their * corresponding R5 modification types, creates a checksum from those modifications, and adds @@ -178,55 +162,34 @@ private static List modificationsForProject ( * TODO arguably this should be done by a method on the task classes themselves, with common parts factored out * to the same method on the superclass. */ - public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project) { - - // Fetch the modifications associated with this project, filtering for the selected scenario - // (denoted here as "variant"). There are no modifications in the baseline scenario - // (which is denoted by special index -1). - List modifications = new ArrayList<>(); - String scenarioName; - if (variantIndex > -1) { - if (variantIndex >= project.variants.length) { - throw AnalysisServerException.badRequest("Scenario does not exist. Please select a new scenario."); - } - modifications = modificationsForProject(project.accessGroup, projectId, variantIndex); - scenarioName = project.variants[variantIndex]; - } else { - scenarioName = "Baseline"; - } + public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, List modifications) { + if (bounds == null) throw AnalysisServerException.badRequest("Analysis bounds must be set."); - // The CRC of the modifications in this scenario is appended to the scenario ID to - // identify a unique revision of the scenario (still denoted here as variant) allowing - // the worker to cache and reuse networks built by applying that exact revision of the + // The CRC of the modifications in this scenario is appended to the bundle ID to identify a unique set of + // modifications allowing the worker to cache and reuse networks built by applying that exact revision of the // scenario to a base network. CRC32 crc = new CRC32(); crc.update(JsonUtilities.objectToJsonBytes(modifications)); long crcValue = crc.getValue(); - - task.scenario = new Scenario(); // FIXME Job IDs need to be unique. Why are we setting this to the project and variant? // This only works because the job ID is overwritten when the job is enqueued. // Its main effect is to cause the scenario ID to have this same pattern! // We should probably leave the JobID null on single point tasks. Needed: polymorphic task initialization. - task.jobId = String.format("%s-%s-%s", projectId, variantIndex, crcValue); - task.scenario.id = task.scenarioId = task.jobId; - task.scenario.modifications = modifications; - task.scenario.description = scenarioName; - task.graphId = project.bundleId; + task.jobId = String.format("%s-%s", bundleId, crcValue); + + Scenario scenario = new Scenario(); + scenario.id = task.scenarioId = task.jobId; + scenario.modifications = modifications; + // task.scenario.description = scenarioName; + + task.scenario = scenario; + task.graphId = bundleId; task.workerVersion = workerVersion; - task.maxFare = this.maxFare; - task.inRoutingFareCalculator = this.inRoutingFareCalculator; - - Bounds bounds = this.bounds; - if (bounds == null) { - // If no bounds were specified, fall back on the bounds of the entire region. - Region region = Persistence.regions.findByIdIfPermitted(project.regionId, project.accessGroup); - bounds = region.bounds; - } + task.maxFare = maxFare; + task.inRoutingFareCalculator = inRoutingFareCalculator; // TODO define class with static factory function WebMercatorGridBounds.fromLatLonBounds(). // Also include getIndex(x, y), getX(index), getY(index), totalTasks() - WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(bounds.envelope(), zoom); checkGridSize(extents); task.height = extents.height; diff --git a/src/main/java/com/conveyal/analysis/models/Project.java b/src/main/java/com/conveyal/analysis/models/Project.java index 9f633c209..184a3df30 100644 --- a/src/main/java/com/conveyal/analysis/models/Project.java +++ b/src/main/java/com/conveyal/analysis/models/Project.java @@ -6,9 +6,6 @@ * Represents a TAUI project */ public class Project extends Model implements Cloneable { - /** Names of the variants of this project */ - public String[] variants; - public String regionId; public String bundleId; diff --git a/src/main/java/com/conveyal/analysis/models/RegionalAnalysis.java b/src/main/java/com/conveyal/analysis/models/RegionalAnalysis.java index 932ac5218..670833251 100644 --- a/src/main/java/com/conveyal/analysis/models/RegionalAnalysis.java +++ b/src/main/java/com/conveyal/analysis/models/RegionalAnalysis.java @@ -16,6 +16,7 @@ public class RegionalAnalysis extends Model implements Cloneable { public String regionId; public String bundleId; public String projectId; + public String scenarioId; public int variant; diff --git a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java index b88bddac4..fae87cfcb 100644 --- a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java +++ b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; /** * An attempt at simulating a MapDB-style interface, for storing Java objects in MongoDB. @@ -74,7 +75,7 @@ public Collection findAllForRequest(Request req) { * Helper function that adds the `accessGroup` to the query if the user is not an admin. If you want to query using * the `accessGroup` as an admin it must be added to the query. */ - public Collection findPermitted(DBObject query, String accessGroup) { + public List findPermitted(DBObject query, String accessGroup) { DBCursor cursor = find(QueryBuilder.start().and( query, QueryBuilder.start("accessGroup").is(accessGroup).get() From 358b41246a926a202e1ef74317703ce987740dcf Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 16:20:57 +0800 Subject: [PATCH 057/187] check for freeform destinations at job creation This warns the user of the mistake before the regional job ever starts. Previously we'd get one error in each result returned from the worker. --- .../analysis/controllers/RegionalAnalysisController.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 894c62ac0..9d0fcf43a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -16,6 +16,7 @@ import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; +import com.conveyal.r5.analyst.FreeFormPointSet; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.PointSetCache; @@ -456,6 +457,14 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro PointSetCache.readFreeFormFromFileStore(task.destinationPointSetKeys[0]) }; } + if (task.recordTimes) { + checkArgument( + task.destinationPointSets != null && + task.destinationPointSets.length == 1 && + task.destinationPointSets[0] instanceof FreeFormPointSet, + "recordTimes can only be used with a single destination pointset, which must be freeform (non-grid)." + ); + } // TODO remove duplicate fields from RegionalAnalysis that are already in the nested task. // The RegionalAnalysis should just be a minimal wrapper around the template task, adding the origin point set. From f33e0056b632254b5356374d646dcc1d685cbea1 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 23:22:11 +0800 Subject: [PATCH 058/187] checkState when looking up frequency offsets Confirmed that both of these assertions will fail on Montreal STM Metro data for 2019-05-16. This weekday filters out all weekend patterns and trips. The filtered trip index ends up pointing to an unfiltered trip with a different number of entries, and whose headway is greater than the filtered trip's headway. --- .../com/conveyal/r5/profile/FastRaptorWorker.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java b/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java index 833fbfcd1..6b8c9f487 100644 --- a/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java +++ b/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java @@ -671,7 +671,12 @@ private void doFrequencySearchForRound (RaptorState outputState, FrequencyBoardi // this looks like a good candidate for polymorphism (board time strategy passed in). // The offset could be looked up by the getDepartureTime method itself, not passed in. if (frequencyBoardingMode == MONTE_CARLO) { - int offset = offsets.offsets.get(patternIndex)[tripScheduleIndex][frequencyEntryIdx]; + int[] offsetsPerEntry = offsets.offsets.get(patternIndex)[tripScheduleIndex]; + checkState( + schedule.nFrequencyEntries() == offsetsPerEntry.length, + "Offsets array length should exactly match number of freq entries in TripSchedule." + ); + int offset = offsetsPerEntry[frequencyEntryIdx]; newBoardingDepartureTimeAtStop = getRandomFrequencyDepartureTime( schedule, stopPositionInPattern, @@ -736,7 +741,9 @@ public int getRandomFrequencyDepartureTime ( int frequencyEntryIdx, int earliestTime ) { - checkState(offset >= 0); + checkState(offset >= 0, "Offset should be non-negative."); + checkState(offset < schedule.headwaySeconds[frequencyEntryIdx], "Offset should be less than headway."); + // Find the time the first vehicle in this entry will depart from the current stop: // The start time of the entry window, plus travel time from first stop to current stop, plus phase offset. // TODO check that frequency trips' stop times are always normalized to zero at first departure. From be766a9e202fbd620ad9a9d4ffb033e3962a13f3 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 23:23:20 +0800 Subject: [PATCH 059/187] main method to extract single gtfs route_type this is used to isolate a small number of routes for use in testing --- .../com/conveyal/gtfs/ExtractGTFSMode.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 src/main/java/com/conveyal/gtfs/ExtractGTFSMode.java diff --git a/src/main/java/com/conveyal/gtfs/ExtractGTFSMode.java b/src/main/java/com/conveyal/gtfs/ExtractGTFSMode.java new file mode 100644 index 000000000..58474451c --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/ExtractGTFSMode.java @@ -0,0 +1,121 @@ +package com.conveyal.gtfs; + +import com.conveyal.gtfs.model.Frequency; +import com.conveyal.gtfs.model.Route; +import com.conveyal.gtfs.model.Stop; +import com.conveyal.gtfs.model.StopTime; +import com.conveyal.gtfs.model.Transfer; +import com.conveyal.gtfs.model.Trip; +import com.google.common.base.Strings; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Geometry; +import org.mapdb.Fun; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * This main method will filter an input GTFS file, retaining only the given route_type (mode of transport). + * All routes, trips, stops, and frequencies for other route_types will be removed. This is useful for preparing + * minimal GTFS inputs for tests. For example, we have extracted only the subway / metro routes from the STM + * Montreal feed - they are useful for testing Monte Carlo code because they have many frequency entries per trip. + */ +public class ExtractGTFSMode { + + private static final String inputFile = "/Users/abyrd/geodata/stm.gtfs.zip"; + private static final String outputFile = "/Users/abyrd/geodata/stm-metro.gtfs.zip"; + + // Remove all shapes from the GTFS to make it simpler to render in a web UI + private static final boolean REMOVE_SHAPES = true; + + private static final int RETAIN_ROUTE_TYPE = Route.SUBWAY; + + public static void main (String[] args) { + + GTFSFeed feed = GTFSFeed.writableTempFileFromGtfs(inputFile); + + System.out.println("Removing routes that are not on mode " + RETAIN_ROUTE_TYPE); + Set retainRoutes = new HashSet<>(); + Iterator routeIterator = feed.routes.values().iterator(); + while (routeIterator.hasNext()) { + Route route = routeIterator.next(); + if (route.route_type == RETAIN_ROUTE_TYPE) { + retainRoutes.add(route.route_id); + } else { + routeIterator.remove(); + } + } + + System.out.println("Removing trips that are not on mode " + RETAIN_ROUTE_TYPE); + Set retainTrips = new HashSet<>(); + Iterator tripIterator = feed.trips.values().iterator(); + while (tripIterator.hasNext()) { + Trip trip = tripIterator.next(); + if (retainRoutes.contains(trip.route_id)) { + retainTrips.add(trip.trip_id); + } else { + tripIterator.remove(); + } + } + + System.out.println("Removing frequencies that are not on mode " + RETAIN_ROUTE_TYPE); + Iterator> freqIterator = feed.frequencies.iterator(); + while (freqIterator.hasNext()) { + Frequency frequency = freqIterator.next().b; + if (!retainTrips.contains(frequency.trip_id)) { + freqIterator.remove(); + } + } + + System.out.println("Removing stop_times that are not on mode " + RETAIN_ROUTE_TYPE); + Set referencedStops = new HashSet<>(); + Iterator stIterator = feed.stop_times.values().iterator(); + while (stIterator.hasNext()) { + StopTime stopTime = stIterator.next(); + if (retainTrips.contains(stopTime.trip_id)) { + referencedStops.add(stopTime.stop_id); + } else { + stIterator.remove(); + } + } + + System.out.println("Removing unreferenced stops..."); + Iterator stopIterator = feed.stops.values().iterator(); + while (stopIterator.hasNext()) { + Stop stop = stopIterator.next(); + if (!referencedStops.contains(stop.stop_id)) { + stopIterator.remove(); + } + } + + if (REMOVE_SHAPES) { + System.out.println("Removing shapes table and removing shape IDs from trips..."); + feed.shape_points.clear(); + for (String tripId : feed.trips.keySet()) { + Trip trip = feed.trips.get(tripId); + trip.shape_id = null; + // Entry.setValue is an unsupported operation in MapDB, just re-put the trip. + feed.trips.put(tripId, trip); + } + } + + System.out.println("Filtering transfers for removed stops..."); + Iterator ti = feed.transfers.values().iterator(); + while (ti.hasNext()) { + Transfer t = ti.next(); + if ( ! (referencedStops.contains(t.from_stop_id) && referencedStops.contains(t.to_stop_id))) { + ti.remove(); + } + } + + System.out.println("Writing GTFS..."); + feed.toFile(outputFile); + feed.close(); + } + +} From 484d6ecb5689b4fb43980fe7c93bc6853db71796 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 23:59:51 +0800 Subject: [PATCH 060/187] multiple timetables and services in grid tests This allows different weekday and weekend service, with different frequency entries attached to the trip for each service. This is useful for testing Monte Carlo randomization, which currently makes heavy use of int indexes into unfiltered lists of trips and frequency entries. --- .../r5/analyst/network/GridGtfsGenerator.java | 94 ++++++++++++------- .../r5/analyst/network/GridRoute.java | 61 ++++++++---- .../network/GridSinglePointTaskBuilder.java | 17 +++- .../analyst/network/SimpsonDesertTests.java | 8 +- 4 files changed, 119 insertions(+), 61 deletions(-) diff --git a/src/test/java/com/conveyal/r5/analyst/network/GridGtfsGenerator.java b/src/test/java/com/conveyal/r5/analyst/network/GridGtfsGenerator.java index 36648f113..6699dc879 100644 --- a/src/test/java/com/conveyal/r5/analyst/network/GridGtfsGenerator.java +++ b/src/test/java/com/conveyal/r5/analyst/network/GridGtfsGenerator.java @@ -2,7 +2,6 @@ import com.conveyal.gtfs.GTFSFeed; import com.conveyal.gtfs.model.Agency; -import com.conveyal.gtfs.model.Calendar; import com.conveyal.gtfs.model.CalendarDate; import com.conveyal.gtfs.model.Frequency; import com.conveyal.gtfs.model.Route; @@ -10,11 +9,15 @@ import com.conveyal.gtfs.model.Stop; import com.conveyal.gtfs.model.StopTime; import com.conveyal.gtfs.model.Trip; -import org.checkerframework.checker.units.qual.A; import org.mapdb.Fun; import java.time.LocalDate; -import java.util.stream.IntStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import static com.conveyal.r5.analyst.network.GridRoute.Materialization.EXACT_TIMES; +import static com.conveyal.r5.analyst.network.GridRoute.Materialization.STOP_TIMES; /** * Create a MapDB backed GTFS object from a GridLayout, not necessarily to be written out as a standard CSV/ZIP feed, @@ -24,9 +27,9 @@ public class GridGtfsGenerator { public static final String FEED_ID = "GRID"; public static final String AGENCY_ID = "AGENCY"; - public static final String SERVICE_ID = "ALL"; - public static final LocalDate GTFS_DATE = LocalDate.of(2020, 1, 1); + public static final LocalDate WEEKDAY_DATE = LocalDate.of(2020, 1, 1); + public static final LocalDate WEEKEND_DATE = LocalDate.of(2020, 1, 4); public final GridLayout gridLayout; @@ -54,14 +57,21 @@ private void addCommonTables () { agency.agency_id = AGENCY_ID; agency.agency_name = AGENCY_ID; feed.agency.put(agency.agency_id, agency); + addService(GridRoute.Services.WEEKDAY, 1, 2, 3); + addService(GridRoute.Services.WEEKEND, 4, 5); + addService(GridRoute.Services.ALL, 1, 2, 3, 4, 5); + } - Service service = new Service(SERVICE_ID); - CalendarDate calendarDate = new CalendarDate(); - calendarDate.date = LocalDate.of(2020, 01, 01); - calendarDate.service_id = SERVICE_ID; - calendarDate.exception_type = 1; - service.calendar_dates.put(calendarDate.date, calendarDate); - feed.services.put(service.service_id, service); + private void addService (GridRoute.Services grs, int... daysOfJanuary2020) { + Service gtfsService = new Service(grs.name()); + for (int day : daysOfJanuary2020) { + CalendarDate calendarDate = new CalendarDate(); + calendarDate.date = LocalDate.of(2020, 01, day); + calendarDate.service_id = gtfsService.service_id; + calendarDate.exception_type = 1; + gtfsService.calendar_dates.put(calendarDate.date, calendarDate); + } + feed.services.put(gtfsService.service_id, gtfsService); } private void addRoute (GridRoute gridRoute) { @@ -98,40 +108,52 @@ public void addStopsForRoute (GridRoute route, boolean back) { } public void addTripsForRoute (GridRoute route, boolean back) { - if (route.headwayMinutes > 0) { - int tripIndex = 0; - int start = route.startHour * 60 * 60; - int end = route.endHour * 60 * 60; - int headway = route.headwayMinutes * 60; - - // Maybe we should use exact_times = 1 instead of generating individual trips. - for (int startTime = start; startTime < end; startTime += headway, tripIndex++) { - String tripId = addTrip(route, back, startTime, tripIndex); - if (route.pureFrequency) { - Frequency frequency = new Frequency(); - frequency.start_time = start; - frequency.end_time = end; - frequency.headway_secs = headway; - frequency.exact_times = 0; - feed.frequencies.add(new Fun.Tuple2<>(tripId, frequency)); - // Do not make any additional trips, frequency entry represents them. - break; - } - } - } else { + // An explicit array of trip start times takes precedence over timetables. + if (route.startTimes != null) { for (int i = 0; i < route.startTimes.length; i++) { - addTrip(route, back, route.startTimes[i], i); + addTrip(route, back, route.startTimes[i], i, GridRoute.Services.ALL); + } + return; + } + // For the non-STOP_TIMES case, a single trip per service that will be referenced by all the timetables. + // We should somehow also allow for different travel speeds per timetable, and default fallback speeds. + Map tripIdForService = new HashMap<>(); + int tripIndex = 0; + for (GridRoute.Timetable timetable : route.timetables) { + int start = timetable.startHour * 60 * 60; + int end = timetable.endHour * 60 * 60; + int headway = timetable.headwayMinutes * 60; + if (route.materialization == STOP_TIMES) { + // For STOP_TIMES, make N different trips. + for (int startTime = start; startTime < end; startTime += headway, tripIndex++) { + addTrip(route, back, startTime, tripIndex, timetable.service); + } + } else { + // Not STOP_TIMES, so this is a frequency entry (either EXACT_TIMES or PURE_FREQ). + // Make only one trip per service ID, all frequency entries reference this single trip. + String tripId = tripIdForService.get(timetable.service); + if (tripId == null) { + tripId = addTrip(route, back, 0, tripIndex, timetable.service); + tripIdForService.put(timetable.service, tripId); + tripIndex++; + } + Frequency frequency = new Frequency(); + frequency.start_time = start; + frequency.end_time = end; + frequency.headway_secs = headway; + frequency.exact_times = (route.materialization == EXACT_TIMES) ? 1 : 0; + feed.frequencies.add(new Fun.Tuple2<>(tripId, frequency)); } } } - private String addTrip (GridRoute route, boolean back, int startTime, int tripIndex) { + private String addTrip (GridRoute route, boolean back, int startTime, int tripIndex, GridRoute.Services service) { Trip trip = new Trip(); trip.direction_id = back ? 1 : 0; String tripId = String.format("%s:%d:%d", route.id, tripIndex, trip.direction_id); trip.trip_id = tripId; trip.route_id = route.id; - trip.service_id = SERVICE_ID; + trip.service_id = service.name(); feed.trips.put(trip.trip_id, trip); int dwell = gridLayout.transitDwellSeconds; int departureTime = startTime; diff --git a/src/test/java/com/conveyal/r5/analyst/network/GridRoute.java b/src/test/java/com/conveyal/r5/analyst/network/GridRoute.java index d269a7ed7..a108204bf 100644 --- a/src/test/java/com/conveyal/r5/analyst/network/GridRoute.java +++ b/src/test/java/com/conveyal/r5/analyst/network/GridRoute.java @@ -1,5 +1,7 @@ package com.conveyal.r5.analyst.network; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Stream; @@ -18,24 +20,37 @@ public class GridRoute { public int stopSpacingBlocks; public Orientation orientation; public boolean bidirectional; - public int startHour; - public int endHour; + /** Explicit departure times from first stop; if set, startHour and endHour will be ignored*/ public int[] startTimes; - /** - * Override default hop times. Map of (trip, stopAtStartOfHop) to factor by which default hop is multiplied - */ + + /** Override default hop times. Map of (trip, stopAtStartOfHop) to factor by which default hop is multiplied. */ public Map hopTimeScaling; - public int headwayMinutes; - public boolean pureFrequency; + + /** These will be services codes, and can be referenced in timetables. */ + public static enum Services { ALL, WEEKDAY, WEEKEND } + + /** How a Timetable will be translated into GTFS data - stop_times or frequencies with or without exact_times. */ + public static enum Materialization { STOP_TIMES, PURE_FREQ, EXACT_TIMES } + + /** All Timetables on a GridRoute will be materialized in the same way, according to this field. */ + public Materialization materialization = Materialization.STOP_TIMES; + + /** This defines something like a frequency in GTFS, but can also be used to generate normal stop_times trips. */ + public static class Timetable { + Services service; + public int startHour; + public int endHour; + int headwayMinutes; + } + + public List timetables = new ArrayList<>(); private Stream stopIds() { return null; } - public static enum Orientation { - HORIZONTAL, VERTICAL - } + public static enum Orientation { HORIZONTAL, VERTICAL } public int nBlocksLength () { return (nStops - 1) * stopSpacingBlocks; @@ -88,11 +103,9 @@ private static GridRoute newBareRoute (GridLayout gridLayout, int headwayMinutes route.id = gridLayout.nextIntegerId(); // Avoid collisions when same route is added multiple times route.stopSpacingBlocks = 1; route.gridLayout = gridLayout; - route.startHour = 5; - route.endHour = 10; route.bidirectional = true; - route.headwayMinutes = headwayMinutes; route.nStops = gridLayout.widthAndHeightInBlocks + 1; + route.addTimetable(Services.WEEKDAY, 5, 10, headwayMinutes); return route; } @@ -105,6 +118,21 @@ public static GridRoute newHorizontalRoute (GridLayout gridLayout, int row, int return route; } + public GridRoute pureFrequency () { + this.materialization = Materialization.PURE_FREQ; + return this; + } + + public GridRoute addTimetable (Services service, int startHour, int endHour, int headwayMinutes) { + Timetable timetable = new Timetable(); + timetable.service = service; + timetable.startHour = startHour; + timetable.endHour = endHour; + timetable.headwayMinutes = headwayMinutes; + this.timetables.add(timetable); + return this; + } + public static GridRoute newVerticalRoute (GridLayout gridLayout, int col, int headwayMinutes) { GridRoute route = newBareRoute(gridLayout, headwayMinutes); route.orientation = Orientation.VERTICAL; @@ -114,12 +142,7 @@ public static GridRoute newVerticalRoute (GridLayout gridLayout, int col, int he return route; } - public GridRoute pureFrequency () { - pureFrequency = true; - return this; - } - - public static class TripHop{ + public static class TripHop { int trip; int hop; diff --git a/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java b/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java index ced99afa3..a4b71fd9c 100644 --- a/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java +++ b/src/test/java/com/conveyal/r5/analyst/network/GridSinglePointTaskBuilder.java @@ -14,7 +14,8 @@ import java.util.EnumSet; import java.util.stream.IntStream; -import static com.conveyal.r5.analyst.network.GridGtfsGenerator.GTFS_DATE; +import static com.conveyal.r5.analyst.network.GridGtfsGenerator.WEEKDAY_DATE; +import static com.conveyal.r5.analyst.network.GridGtfsGenerator.WEEKEND_DATE; /** * This creates a task for use in tests. It uses a builder pattern but for a non-immutable task object. @@ -35,7 +36,7 @@ public GridSinglePointTaskBuilder (GridLayout gridLayout) { this.gridLayout = gridLayout; // We will accumulate settings into this task. task = new TravelTimeSurfaceTask(); - task.date = GTFS_DATE; + task.date = WEEKDAY_DATE; // Set defaults that can be overridden by calling builder methods. task.accessModes = EnumSet.of(LegMode.WALK); task.egressModes = EnumSet.of(LegMode.WALK); @@ -78,6 +79,18 @@ public GridSinglePointTaskBuilder setDestination (int gridX, int gridY) { return this; } + public GridSinglePointTaskBuilder weekdayMorningPeak () { + task.date = WEEKDAY_DATE; + morningPeak(); + return this; + } + + public GridSinglePointTaskBuilder weekendMorningPeak () { + task.date = WEEKEND_DATE; + morningPeak(); + return this; + } + public GridSinglePointTaskBuilder morningPeak () { task.fromTime = LocalTime.of(7, 00).toSecondOfDay(); task.toTime = LocalTime.of(9, 00).toSecondOfDay(); diff --git a/src/test/java/com/conveyal/r5/analyst/network/SimpsonDesertTests.java b/src/test/java/com/conveyal/r5/analyst/network/SimpsonDesertTests.java index a702fd621..327716429 100644 --- a/src/test/java/com/conveyal/r5/analyst/network/SimpsonDesertTests.java +++ b/src/test/java/com/conveyal/r5/analyst/network/SimpsonDesertTests.java @@ -46,7 +46,7 @@ public void testGridScheduled () throws Exception { // gridLayout.exportFiles("test"); AnalysisWorkerTask task = gridLayout.newTaskBuilder() - .morningPeak() + .weekdayMorningPeak() .setOrigin(20, 20) .uniformOpportunityDensity(10) .build(); @@ -85,7 +85,7 @@ public void testGridFrequency () throws Exception { TransportNetwork network = gridLayout.generateNetwork(); AnalysisWorkerTask task = gridLayout.newTaskBuilder() - .morningPeak() + .weekdayMorningPeak() .setOrigin(20, 20) .uniformOpportunityDensity(10) .build(); @@ -120,7 +120,7 @@ public void testGridFrequencyAlternatives () throws Exception { TransportNetwork network = gridLayout.generateNetwork(); AnalysisWorkerTask task = gridLayout.newTaskBuilder() - .morningPeak() + .weekdayMorningPeak() .setOrigin(20, 20) .uniformOpportunityDensity(10) .monteCarloDraws(20000) @@ -229,7 +229,7 @@ public void testExperiments () throws Exception { TransportNetwork network = gridLayout.generateNetwork(); AnalysisWorkerTask task = gridLayout.newTaskBuilder() - .morningPeak() + .weekdayMorningPeak() .setOrigin(20, 20) .uniformOpportunityDensity(10) .monteCarloDraws(4000) From fa620677d0a8fe4ae05aae2d9a0640597c438745 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 12 Aug 2021 00:14:11 +0800 Subject: [PATCH 061/187] add test for mismatched frequency offsets #740 This test fails and must be followed with a patch to fix the problem. --- .../network/RandomFrequencyPhasingTests.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/test/java/com/conveyal/r5/analyst/network/RandomFrequencyPhasingTests.java diff --git a/src/test/java/com/conveyal/r5/analyst/network/RandomFrequencyPhasingTests.java b/src/test/java/com/conveyal/r5/analyst/network/RandomFrequencyPhasingTests.java new file mode 100644 index 000000000..4f762a631 --- /dev/null +++ b/src/test/java/com/conveyal/r5/analyst/network/RandomFrequencyPhasingTests.java @@ -0,0 +1,62 @@ +package com.conveyal.r5.analyst.network; + +import com.conveyal.r5.OneOriginResult; +import com.conveyal.r5.analyst.TravelTimeComputer; +import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; +import com.conveyal.r5.transit.TransportNetwork; +import org.junit.jupiter.api.Test; + +import static com.conveyal.r5.analyst.network.SimpsonDesertTests.SIMPSON_DESERT_CORNER; + +public class RandomFrequencyPhasingTests { + + /** + * This recreates the problem in issue #740, inspired by the structure of STM Montréal GTFS métro schedules. + * This involves a pattern with two different frequency trips on two different services. The first service has + * N freq entries, and the second has more than N entries. + * + * Here we make one weekday trip with a single frequency entry, and one weekend trip with two frequency entries. + * We search on the weekend, causing the weekday trip to be filtered out. The Monte Carlo boarding code expects + * an array of two offsets for the weekend trip, but sees a single offset intended for the weekday trip. In addition + * the offsets generated for the weekday service will be in the range (0...30) while the headway of the weekend + * service is much shorter - many generated offsets will exceed the headway. So at least two different assertions + * can fail here, but whether they will actually fail depends on the order of the TripSchedules in the TripPattern + * (the weekend trip must come after the weekday one, so filtering causes a decrease in trip index). This depends + * in turn on the iteration order of the GtfsFeed.trips map, which follows the natural order of its keys so should + * be determined by the lexical order of trip IDs, which are determined by the order in which timetables are added + * to the GridRoute. + * + * The problem exists even if the mismatched trips have the same number of entries but can fail silently as offsets + * are computed for the wrong headways. The output travel time range will even be correct, but the distribution will + * be skewed by the offsets being randomly selected from a different headway, while the headway itself is accurate. + * + * The structure created here can only be created by input GTFS, and cannot be produced by our Conveyal scenarios. + * In scenarios, we always produce exactly one frequency entry per trip. This does not mean scenarios are immune + * to the problem of using offsets from the wrong trip - it's just much harder to detect the problem as all arrays + * are the same length, so it can fail silently. + */ + @Test + public void testFilteredTripRandomization () throws Exception { + + GridLayout gridLayout = new GridLayout(SIMPSON_DESERT_CORNER, 40); + // TODO DSL revision: + // gridLayout.newRoute(/*construct and add to routes*/).horizontal(20).addTimetable()... + gridLayout.routes.add(GridRoute + .newHorizontalRoute(gridLayout, 20, 30) + .addTimetable(GridRoute.Services.WEEKEND, 6, 10, 4) + .addTimetable(GridRoute.Services.WEEKEND, 10, 22, 8) + .pureFrequency() + ); + TransportNetwork network = gridLayout.generateNetwork(); + AnalysisWorkerTask task = gridLayout.newTaskBuilder() + .weekendMorningPeak() + .setOrigin(20, 20) + .monteCarloDraws(1000) + .build(); + + TravelTimeComputer computer = new TravelTimeComputer(task, network); + OneOriginResult oneOriginResult = computer.computeTravelTimes(); + + } + +} From 6b1c21858275c3887800098797e248257fc18d3c Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 6 Aug 2021 19:52:36 +0800 Subject: [PATCH 062/187] key random offsets on TripSchedules (issue #740) this is a minimal fix that builds a secondary map, avoiding translation from filtered to unfiltered pattern and trip indexes. There are better ways to do this but they all seem to cascade series of refactors so carry some risk of introducing more bugs. This one is easier to test and trace, so makes a good first step. --- .../conveyal/r5/profile/FastRaptorWorker.java | 9 +--- .../r5/profile/FrequencyRandomOffsets.java | 45 +++++++++++++++++-- .../McRaptorSuboptimalPathProfileRouter.java | 2 +- .../transit/FrequencyRandomOffsetsTest.java | 8 ++-- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java b/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java index 6b8c9f487..0ac37b77f 100644 --- a/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java +++ b/src/main/java/com/conveyal/r5/profile/FastRaptorWorker.java @@ -623,9 +623,7 @@ private void doFrequencySearchForRound (RaptorState outputState, FrequencyBoardi ) { FilteredPattern filteredPattern = filteredPatterns.patterns.get(patternIndex); TripPattern pattern = transit.tripPatterns.get(patternIndex); - int tripScheduleIndex = -1; // First loop iteration will immediately increment to 0. for (TripSchedule schedule : filteredPattern.runningFrequencyTrips) { - tripScheduleIndex++; // Loop through all the entries for this trip (time windows with service at a given frequency). for (int frequencyEntryIdx = 0; frequencyEntryIdx < schedule.headwaySeconds.length; @@ -671,12 +669,7 @@ private void doFrequencySearchForRound (RaptorState outputState, FrequencyBoardi // this looks like a good candidate for polymorphism (board time strategy passed in). // The offset could be looked up by the getDepartureTime method itself, not passed in. if (frequencyBoardingMode == MONTE_CARLO) { - int[] offsetsPerEntry = offsets.offsets.get(patternIndex)[tripScheduleIndex]; - checkState( - schedule.nFrequencyEntries() == offsetsPerEntry.length, - "Offsets array length should exactly match number of freq entries in TripSchedule." - ); - int offset = offsetsPerEntry[frequencyEntryIdx]; + int offset = offsets.getOffsetSeconds(schedule, frequencyEntryIdx); newBoardingDepartureTimeAtStop = getRandomFrequencyDepartureTime( schedule, stopPositionInPattern, diff --git a/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java b/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java index 8a47fa61b..9cb816dc9 100644 --- a/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java +++ b/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java @@ -9,8 +9,14 @@ import org.apache.commons.math3.random.MersenneTwister; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; - /** +import static com.google.common.base.Preconditions.checkElementIndex; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +/** * Generates and stores departure time offsets for every frequency-based set of trips. * This holds only one set of offsets at a time. It is re-randomized before each Monte Carlo iteration. * Therefore we have no memory of exactly which offsets were used in a particular Monte Carlo search. @@ -18,9 +24,19 @@ * we'd need to make alternate implementations that pre-generate the entire set or use deterministic seeded generators. */ public class FrequencyRandomOffsets { + /** map from trip pattern index to a list of offsets for trip i and frequency entry j on that pattern */ - public final TIntObjectMap offsets = new TIntObjectHashMap<>(); - public final TransitLayer data; + private final TIntObjectMap offsets = new TIntObjectHashMap<>(); + private final TransitLayer data; + + /** + * Secondary copy of the offsets keyed on TripSchedule objects. + * This allows lookups where patterns and trips have been filtered and int indexes no longer match unfiltered ones. + * This can't simply replace the other offsets map because we need to fetch stop sequences from TripPatterns + * to look up phase targets on the fly. It's awkward to store them if they're looked up in advance because the + * natural key is frequency entries, which are not objects but just array slots. + */ + private final Map offsetsForTripSchedule = new HashMap<>(); /** The mersenne twister is a higher quality random number generator than the one included with Java */ private MersenneTwister mt = new MersenneTwister(); @@ -47,6 +63,17 @@ public FrequencyRandomOffsets(TransitLayer data) { } } + public int getOffsetSeconds (TripSchedule tripSchedule, int freqEntryIndex) { + int[] offsetsPerEntry = offsetsForTripSchedule.get(tripSchedule); + checkState( + tripSchedule.nFrequencyEntries() == offsetsPerEntry.length, + "Offsets array length should exactly match number of freq entries in TripSchedule." + ); + int offset = offsetsPerEntry[freqEntryIndex]; + checkState(offset >= 0, "Frequency entry offset was not randomized."); + return offset; + } + /** * Take a new Monte Carlo draw if requested (i.e. if boarding assumption is not half-headway): for each * frequency-based route, choose how long after service starts the first vehicle leaves (the route's "phase"). @@ -179,10 +206,20 @@ public void randomize () { } } } - if (remainingAfterPreviousRound == remaining && remaining > 0) { throw new IllegalArgumentException("Cannot solve phasing, you may have a circular reference!"); } + // Copy results of randomization to a Map keyed on TripSchedules (instead of TripPattern index ints) + offsetsForTripSchedule.clear(); + for (TIntObjectIterator it = offsets.iterator(); it.hasNext(); ) { + it.advance(); + TripPattern tripPattern = data.tripPatterns.get(it.key()); + int[][] offsetsForTrip = it.value(); + for (int ts = 0; ts < tripPattern.tripSchedules.size(); ts++) { + TripSchedule tripSchedule = tripPattern.tripSchedules.get(ts); + offsetsForTripSchedule.put(tripSchedule, offsetsForTrip[ts]); + } + } } } } diff --git a/src/main/java/com/conveyal/r5/profile/McRaptorSuboptimalPathProfileRouter.java b/src/main/java/com/conveyal/r5/profile/McRaptorSuboptimalPathProfileRouter.java index 181a8a845..b38380060 100644 --- a/src/main/java/com/conveyal/r5/profile/McRaptorSuboptimalPathProfileRouter.java +++ b/src/main/java/com/conveyal/r5/profile/McRaptorSuboptimalPathProfileRouter.java @@ -445,7 +445,7 @@ private boolean doOneRound () { // we have to check all trips and frequency entries because, unlike // schedule-based trips, these are not sorted int departure = tripSchedule.startTimes[frequencyEntry] + - offsets.offsets.get(patIdx)[currentTrip][frequencyEntry] + + offsets.getOffsetSeconds(tripSchedule, frequencyEntry) + tripSchedule.departures[stopPositionInPattern]; int latestDeparture = tripSchedule.endTimes[frequencyEntry] + diff --git a/src/test/java/com/conveyal/r5/transit/FrequencyRandomOffsetsTest.java b/src/test/java/com/conveyal/r5/transit/FrequencyRandomOffsetsTest.java index 21774d9bb..f0fd04763 100644 --- a/src/test/java/com/conveyal/r5/transit/FrequencyRandomOffsetsTest.java +++ b/src/test/java/com/conveyal/r5/transit/FrequencyRandomOffsetsTest.java @@ -74,8 +74,8 @@ public void testPhasing () { // check that phasing is correct // offset indices are trip pattern, trip, frequency entry - int timeAtTargetStop = ts2.startTimes[0] + ts2.departures[1] + fro.offsets.get(1)[0][0]; - int timeAtSourceStop = ts1.startTimes[0] + ts1.departures[2] + fro.offsets.get(0)[0][0]; + int timeAtTargetStop = ts2.startTimes[0] + ts2.departures[1] + fro.getOffsetSeconds(ts2, 0); + int timeAtSourceStop = ts1.startTimes[0] + ts1.departures[2] + fro.getOffsetSeconds(ts1, 0); int timeDifference = timeAtTargetStop - timeAtSourceStop; // Depending on how large the offset on the first route is, the new route may come 10 minutes after on its first // trip, or 20 minutes before (which is the same phasing, just changing which route arrives first). @@ -141,8 +141,8 @@ public void testPhasingAtLastStop () { // check that phasing is correct // offset indices are trip pattern, trip, frequency entry - int timeAtTargetStop = ts2.startTimes[0] + ts2.arrivals[3] + fro.offsets.get(1)[0][0]; - int timeAtSourceStop = ts1.startTimes[0] + ts1.arrivals[3] + fro.offsets.get(0)[0][0]; + int timeAtTargetStop = ts2.startTimes[0] + ts2.arrivals[3] + fro.getOffsetSeconds(ts2, 0); + int timeAtSourceStop = ts1.startTimes[0] + ts1.arrivals[3] + fro.getOffsetSeconds(ts1, 0); int timeDifference = timeAtTargetStop - timeAtSourceStop; // Depending on how large the offset on the first route is, the new route may come 10 minutes after on its first // trip, or 20 minutes before (which is the same phasing, just changing which route arrives first). From a64678215f5e93564466501d9d5b5deb02f0da8e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 19:28:13 +0800 Subject: [PATCH 063/187] do not add null arrays to map. add comments. --- .../conveyal/r5/profile/FrequencyRandomOffsets.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java b/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java index 9cb816dc9..a960d3175 100644 --- a/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java +++ b/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java @@ -209,7 +209,10 @@ public void randomize () { if (remainingAfterPreviousRound == remaining && remaining > 0) { throw new IllegalArgumentException("Cannot solve phasing, you may have a circular reference!"); } - // Copy results of randomization to a Map keyed on TripSchedules (instead of TripPattern index ints) + // Copy results of randomization to a Map keyed on TripSchedules (instead of TripPattern index ints). This + // allows looking up offsets in a context where we only have a filtered list of running frequency trips and + // don't know the original unfiltered index of the trip within the pattern. Ideally we'd just build the + // original map keyed on TripSchedules (or hypothetical FreqEntries) but that reqires a lot of refactoring. offsetsForTripSchedule.clear(); for (TIntObjectIterator it = offsets.iterator(); it.hasNext(); ) { it.advance(); @@ -217,7 +220,11 @@ public void randomize () { int[][] offsetsForTrip = it.value(); for (int ts = 0; ts < tripPattern.tripSchedules.size(); ts++) { TripSchedule tripSchedule = tripPattern.tripSchedules.get(ts); - offsetsForTripSchedule.put(tripSchedule, offsetsForTrip[ts]); + // On patterns with mixed scheduled and frequency trips, scheduled trip slots will be null. + // Maps can store null values, but there's no point in storing them. We only store non-null arrays. + if (offsetsForTrip[ts] != null) { + offsetsForTripSchedule.put(tripSchedule, offsetsForTrip[ts]); + } } } } From 9f8d24214f4638ae49048b7efe879988a4146e54 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 11 Aug 2021 21:33:41 +0800 Subject: [PATCH 064/187] add comments and conditional block brackets --- .../r5/profile/FrequencyRandomOffsets.java | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java b/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java index a960d3175..ccc04c704 100644 --- a/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java +++ b/src/main/java/com/conveyal/r5/profile/FrequencyRandomOffsets.java @@ -25,8 +25,13 @@ */ public class FrequencyRandomOffsets { - /** map from trip pattern index to a list of offsets for trip i and frequency entry j on that pattern */ + /** + * Map from trip pattern index (which is the same between filtered and unfiltered patterns) to a list of offsets + * (in seconds) for each frequency entry on each trip of that pattern. Final dimension null for non-frequency trips. + * In other words, patternIndex -> offsetsSeconds[tripOnPattern][frequencyEntryInTrip]. + */ private final TIntObjectMap offsets = new TIntObjectHashMap<>(); + private final TransitLayer data; /** @@ -43,15 +48,15 @@ public class FrequencyRandomOffsets { public FrequencyRandomOffsets(TransitLayer data) { this.data = data; - - if (!data.hasFrequencies) + if (!data.hasFrequencies) { return; - + } + // Create skeleton empty data structure with slots for all offsets that will be generated. for (int pattIdx = 0; pattIdx < data.tripPatterns.size(); pattIdx++) { TripPattern tp = data.tripPatterns.get(pattIdx); - - if (!tp.hasFrequencies) continue; - + if (!tp.hasFrequencies) { + continue; + } int[][] offsetsThisPattern = new int[tp.tripSchedules.size()][]; for (int tripIdx = 0; tripIdx < tp.tripSchedules.size(); tripIdx++) { @@ -63,6 +68,10 @@ public FrequencyRandomOffsets(TransitLayer data) { } } + /** + * Return the random offset ("phase") in seconds generated for the given frequency entry of the given TripSchedule. + * Lookup is now by TripSchedule object as trips are filtered, losing track of their int indexes in unfiltered lists. + */ public int getOffsetSeconds (TripSchedule tripSchedule, int freqEntryIndex) { int[] offsetsPerEntry = offsetsForTripSchedule.get(tripSchedule); checkState( @@ -80,9 +89,10 @@ public int getOffsetSeconds (TripSchedule tripSchedule, int freqEntryIndex) { * We run all Raptor rounds with one draw before proceeding to the next draw. */ public void randomize () { + // The number of TripSchedules for which we still need to generate a random offset. int remaining = 0; - // First, initialize all offsets for all trips and entries on this pattern with -1s + // First, initialize all offsets for all trips and entries on this pattern with -1 ("not yet randomized"). for (TIntObjectIterator it = offsets.iterator(); it.hasNext(); ) { it.advance(); for (int[] offsetsPerEntry : it.value()) { @@ -94,12 +104,16 @@ public void randomize () { } } + // If some randomized schedules are synchronized with other schedules ("phased") we perform multiple passes. In + // each pass we randomize only schedules whose phasing target is already known (randomized in a previous pass). + // This will loop forever if the phasing dependency graph has cycles - we must catch stalled progress. This is + // essentially performing depth-first traversal of the dependency graph iteratively without materializing it. while (remaining > 0) { - int remainingAfterPreviousRound = remaining; + int remainingAfterPreviousPass = remaining; for (TIntObjectIterator it = offsets.iterator(); it.hasNext(); ) { it.advance(); - + // The only thing we need from the TripPattern is the stop sequence, which is used only in phase solving. TripPattern pattern = data.tripPatterns.get(it.key()); int[][] val = it.value(); @@ -113,20 +127,24 @@ public void randomize () { } else { for (int frequencyEntryIndex = 0; frequencyEntryIndex < val[tripScheduleIndex].length; frequencyEntryIndex++) { if (schedule.phaseFromId == null || schedule.phaseFromId[frequencyEntryIndex] == null) { - // not phased. also, don't overwrite with new random number on each iteration, as other - // trips may be phased from this one + // This trip is not phased so does not require solving. Generate a random offset + // immediately. Do this only once - don't overwrite with a new random number on each + // phase solving pass, as other trips may be be phased from this one. if (val[tripScheduleIndex][frequencyEntryIndex] == -1) { val[tripScheduleIndex][frequencyEntryIndex] = mt.nextInt(schedule.headwaySeconds[frequencyEntryIndex]); remaining--; } } else { - if (val[tripScheduleIndex][frequencyEntryIndex] != -1) continue; // already randomized - - // find source phase information + // This trip is phased from another. + if (val[tripScheduleIndex][frequencyEntryIndex] != -1) { + continue; // Offset has already have been generated. + } + // No random offset has been generated for this trip yet. + // Find source phase information. TODO refactor to use references instead of ints. int[] source = data.frequencyEntryIndexForId.get(schedule.phaseFromId[frequencyEntryIndex]); // Throw a meaningful error when invalid IDs are encountered instead of NPE. - // Really this should be done when applying the modifications rather than during the search. + // Really this should be done when resolving or applying the modifications rather than during search. if (source == null) { throw new RuntimeException("This pattern ID specified in a scenario does not exist: " + schedule.phaseFromId[frequencyEntryIndex]); @@ -147,7 +165,7 @@ public void randomize () { int sourceStopIndexInNetwork = data.indexForStopId.get(schedule.phaseFromStop[frequencyEntryIndex]); // TODO check that stop IDs were found. - + // TODO find all stop IDs in advance when resolving/applying modifications or constructing FrequencyRandomOffsets. while (sourceStopIndexInPattern < phaseFromPattern.stops.length && phaseFromPattern.stops[sourceStopIndexInPattern] != sourceStopIndexInNetwork) { sourceStopIndexInPattern++; @@ -206,7 +224,7 @@ public void randomize () { } } } - if (remainingAfterPreviousRound == remaining && remaining > 0) { + if (remainingAfterPreviousPass == remaining && remaining > 0) { throw new IllegalArgumentException("Cannot solve phasing, you may have a circular reference!"); } // Copy results of randomization to a Map keyed on TripSchedules (instead of TripPattern index ints). This From 5bf15554ac0b938e454f347930f77fd4555ceb69 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 13 Aug 2021 15:05:32 +0800 Subject: [PATCH 065/187] refactoring (renaming) based on PR review comments - renamed source to resource (Mongo collection, variables, parameters) Note: this implies an API change of sourceId -> resourceId - specify parametric type for AnalysisCollections to eliminate casts - move getStorageKey method alongside getS3Key method on AggregationArea - Expanded Javadoc and comments - Finally removed redundant email and accessGroup request attributes --- .../conveyal/analysis/components/HttpApi.java | 7 +-- .../AggregationAreaController.java | 49 ++++++++----------- .../SpatialResourceController.java | 2 +- .../analysis/models/AggregationArea.java | 10 ++++ .../analysis/models/SpatialResource.java | 15 +++--- .../persistence/AnalysisCollection.java | 25 +++++++++- .../analysis/spatial/FeatureSummary.java | 5 ++ .../analysis/spatial/SpatialAttribute.java | 21 ++++---- 8 files changed, 79 insertions(+), 55 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index a5315c728..6149ade60 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -36,8 +36,6 @@ public class HttpApi implements Component { // These "attributes" are attached to an incoming HTTP request with String keys, making them available in handlers private static final String REQUEST_START_TIME_ATTRIBUTE = "requestStartTime"; public static final String USER_PERMISSIONS_ATTRIBUTE = "permissions"; - public static final String USER_EMAIL_ATTRIBUTE = "email"; - public static final String USER_GROUP_ATTRIBUTE = "accessGroup"; public interface Config { boolean offline (); // TODO remove this parameter, use different Components types instead @@ -99,12 +97,9 @@ private spark.Service configureSparkService () { if (authorize) { // Determine which user is sending the request, and which permissions that user has. // This method throws an exception if the user cannot be authenticated. - // Store the resulting permissions object in the request so it can be examined by any handler. UserPermissions userPermissions = authentication.authenticate(req); + // Store the resulting permissions object in the request so it can be examined by any handler. req.attribute(USER_PERMISSIONS_ATTRIBUTE, userPermissions); - // TODO stop using these two separate attributes, and use the permissions object directly - req.attribute(USER_EMAIL_ATTRIBUTE, userPermissions.email); - req.attribute(USER_GROUP_ATTRIBUTE, userPermissions.accessGroup); } }); diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index ee1cfcebd..88b4445bd 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -36,8 +36,8 @@ import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; -import static com.conveyal.analysis.components.HttpApi.USER_GROUP_ATTRIBUTE; import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; +import static com.conveyal.analysis.persistence.AnalysisCollection.getAccessGroup; import static com.conveyal.analysis.spatial.FeatureSummary.Type.POLYGON; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; @@ -63,8 +63,8 @@ public class AggregationAreaController implements HttpController { private final FileStorage fileStorage; private final TaskScheduler taskScheduler; - private final AnalysisCollection aggregationAreaCollection; - private final AnalysisCollection spatialSourceCollection; + private final AnalysisCollection aggregationAreaCollection; + private final AnalysisCollection spatialResourceCollection; public AggregationAreaController ( FileStorage fileStorage, @@ -74,11 +74,7 @@ public AggregationAreaController ( this.fileStorage = fileStorage; this.taskScheduler = taskScheduler; this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); - this.spatialSourceCollection = database.getAnalysisCollection("spatialSources", SpatialResource.class); - } - - private FileStorageKey getStoragePath (AggregationArea area) { - return new FileStorageKey(GRIDS, area.getS3Key()); + this.spatialResourceCollection = database.getAnalysisCollection("spatialSources", SpatialResource.class); } /** @@ -92,20 +88,19 @@ private FileStorageKey getStoragePath (AggregationArea area) { private List createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - String sourceId = req.params("sourceId"); + String resourceId = req.params("resourceId"); String nameProperty = req.queryParams("nameProperty"); final int zoom = parseZoom(req.queryParams("zoom")); // 1. Get file from storage and read its features. ============================================================= - SpatialResource source = (SpatialResource) spatialSourceCollection.findById(sourceId); - Preconditions.checkArgument(POLYGON.equals(source.features.type), "Only polygons can be converted to " + - "aggregation areas."); - + SpatialResource resource = spatialResourceCollection.findById(resourceId); + Preconditions.checkArgument(POLYGON.equals(resource.features.type), + "Only polygons can be converted to aggregation areas."); File sourceFile; List features = null; - if (SHP.equals(source.sourceFormat)) { - sourceFile = fileStorage.getFile(source.storageKey()); + if (SHP.equals(resource.sourceFormat)) { + sourceFile = fileStorage.getFile(resource.storageKey()); ShapefileReader reader = null; try { reader = new ShapefileReader(sourceFile); @@ -115,15 +110,15 @@ private List createAggregationAreas (Request req, Response res) } } - if (GEOJSON.equals(source.sourceFormat)) { + if (GEOJSON.equals(resource.sourceFormat)) { // TODO implement } List finalFeatures = features; - taskScheduler.enqueue(Task.create("Aggregation area creation: " + source.name) + taskScheduler.enqueue(Task.create("Aggregation area creation: " + resource.name) .forUser(userPermissions) .setHeavy(true) - .withWorkProduct(source) + .withWorkProduct(resource) .withAction(progressListener -> { progressListener.beginTask("Processing request", 1); Map areas = new HashMap<>(); @@ -142,7 +137,7 @@ private List createAggregationAreas (Request req, Response res) ); UnaryUnionOp union = new UnaryUnionOp(geometries); // Name the area using the name in the request directly - areas.put(source.name, union.union()); + areas.put(resource.name, union.union()); } else { // Don't union. Name each area by looking up its value for the name property in the request. finalFeatures.forEach(f -> areas.put( @@ -165,7 +160,7 @@ private List createAggregationAreas (Request req, Response res) }); AggregationArea aggregationArea = AggregationArea.create(userPermissions, name) - .withSource(source); + .withSource(resource); try { File gridFile = FileUtils.createScratchFile("grid"); @@ -176,7 +171,7 @@ private List createAggregationAreas (Request req, Response res) aggregationAreaCollection.insert(aggregationArea); aggregationAreas.add(aggregationArea); - fileStorage.moveIntoStorage(getStoragePath(aggregationArea), gridFile); + fileStorage.moveIntoStorage(aggregationArea.getStorageKey(), gridFile); } catch (IOException e) { throw new AnalysisServerException("Error processing/uploading aggregation area"); } @@ -201,19 +196,17 @@ private String readProperty (SimpleFeature feature, String propertyName) { private Collection getAggregationAreas (Request req, Response res) { return aggregationAreaCollection.findPermitted( - and(eq("regionId", req.queryParams("regionId"))), req.attribute(USER_GROUP_ATTRIBUTE) + and(eq("regionId", req.queryParams("regionId"))), getAccessGroup(req) ); } - private Object getAggregationArea (Request req, Response res) { - AggregationArea aggregationArea = (AggregationArea) aggregationAreaCollection.findPermitted( - eq("_id", req.params("maskId")), req.attribute(USER_GROUP_ATTRIBUTE) + private JSONObject getAggregationArea (Request req, Response res) { + AggregationArea aggregationArea = aggregationAreaCollection.findByIdIfPermitted( + req.params("maskId"), getAccessGroup(req) ); - - String url = fileStorage.getURL(getStoragePath(aggregationArea)); + String url = fileStorage.getURL(aggregationArea.getStorageKey()); JSONObject wrappedUrl = new JSONObject(); wrappedUrl.put("url", url); - return wrappedUrl; } diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java index afddbf157..d3443d01c 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java @@ -57,7 +57,7 @@ public SpatialResourceController ( SeamlessCensusGridExtractor extractor ) { this.fileStorage = fileStorage; - this.spatialResourceCollection = database.getAnalysisCollection("spatialSources", SpatialResource.class); + this.spatialResourceCollection = database.getAnalysisCollection("spatialResources", SpatialResource.class); this.taskScheduler = taskScheduler; this.extractor = extractor; } diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index a1ee8d794..544cfa892 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -1,8 +1,11 @@ package com.conveyal.analysis.models; import com.conveyal.analysis.UserPermissions; +import com.conveyal.file.FileStorageKey; import com.fasterxml.jackson.annotation.JsonIgnore; +import static com.conveyal.file.FileCategory.GRIDS; + /** * An aggregation area defines a set of origin points to be averaged together to produce an aggregate accessibility figure. * It is defined by a geometry that is rasterized and stored as a grid, with pixels with values between 0 and 100,000 @@ -33,4 +36,11 @@ public AggregationArea withSource (SpatialResource source) { public String getS3Key () { return String.format("%s/mask/%s.grid", regionId, _id); } + + @JsonIgnore + public FileStorageKey getStorageKey () { + // These in the GRIDS file storage category because aggregation areas are masks represented as binary grids. + return new FileStorageKey(GRIDS, getS3Key()); + } + } diff --git a/src/main/java/com/conveyal/analysis/models/SpatialResource.java b/src/main/java/com/conveyal/analysis/models/SpatialResource.java index 8f993c124..adaca7dd2 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialResource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialResource.java @@ -23,9 +23,10 @@ import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; /** - * Record of a spatial source file (e.g. shapefile, CSV) that has been validated and can be processed into specific - * Conveyal formats (e.g. grids and other spatial layers). Eventually a general Resource class could be extended by - * SpatialResource, GtfsResource, and OsmResource? + * A Resource is a database record associating metadata with an Upload (a raw file in FileStorage uploaded by the user). + * A SpatialResource is such a record for a spatial source file (e.g. shapefile, GeoJSON, CSV) that has been validated + * and is ready to be processed into specific Conveyal formats (e.g. grids and other spatial layers). Eventually a + * general Resource class could be extended by SpatialResource, GtfsResource, and OsmResource? */ public class SpatialResource extends BaseModel { public String regionId; @@ -41,12 +42,8 @@ private SpatialResource (UserPermissions userPermissions, String sourceName) { super(userPermissions, sourceName); } - /** - * No-arg constructor required for Mongo POJO serialization - */ - public SpatialResource () { - super(); - } + /** No-arg constructor required for Mongo POJO deserialization. */ + public SpatialResource () { } public static SpatialResource create (UserPermissions userPermissions, String sourceName) { return new SpatialResource(userPermissions, sourceName); diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 6fc6f7085..5421c009e 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.persistence; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.BaseModel; import com.conveyal.analysis.util.JsonUtil; import com.mongodb.client.MongoCollection; @@ -16,6 +17,7 @@ import java.util.ArrayList; import java.util.List; +import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -24,8 +26,13 @@ public class AnalysisCollection { public MongoCollection collection; private Class type; - private String getAccessGroup (Request req) { - return req.attribute("accessGroup"); + /** + * Helper method to extract accessGroup from the UserPermissions set by the authentication component on a request. + * The redundant request attributes for email and accessGroup may be removed once methods like this are in use. + */ + public static String getAccessGroup (Request req) { + UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + return userPermissions.accessGroup; } private AnalysisServerException invalidAccessGroup() { @@ -62,6 +69,17 @@ public T findById(ObjectId _id) { return collection.find(eq("_id", _id)).first(); } + public T findByIdIfPermitted (String _id, String accessGroup) { + T item = findById(_id); + if (item.accessGroup.equals(accessGroup)) { + return item; + } else { + // TODO: To simplify stack traces this should be refactored to "throw new InvalidAccessGroupException()" + // which should be a subtype of AnalysisServerException with methods like getHttpCode(). + throw invalidAccessGroup(); + } + } + public T create(T newModel, String accessGroup, String creatorEmail) { newModel.accessGroup = accessGroup; newModel.createdBy = creatorEmail; @@ -114,6 +132,9 @@ public T update(T value, String accessGroup) { return value; } + // TODO should all below be static helpers on HttpController? Passing the whole request in seems to defy encapsulation. + // On the other hand, making them instance methods reduces the number of parameters and gives access to Class. + /** * Controller creation helper. */ diff --git a/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java b/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java index afe4e19f5..721a0411f 100644 --- a/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java +++ b/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java @@ -7,6 +7,11 @@ import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; +/** + * Records the geometry type (polygon, point, or line) and number of features in a particular SpatialResource. + * Types are very general, so line type includes things like multilinestrings which must be unwrapped. + * Could we just flatten this into SpatialResource? + */ public class FeatureSummary { public int count; public Type type; diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java index e14301de9..6076c31c0 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java @@ -3,14 +3,21 @@ import org.locationtech.jts.geom.Geometry; import org.opengis.feature.type.AttributeType; -/** Groups the original names and user-friendly fields from shapefile attributes, CSV columns, etc. */ +/** + * In OpenGIS terminology, SpatialResources contain features, each of which has attributes. This class represents a + * single attribute present on all the features in a resource - it's basically the schema metadata for a GIS layer. + * Users can specify their own name for any attribute in the source file, so this also associates these user-specified + * names with the original attribute name. + */ public class SpatialAttribute { - /** Name in source file */ + + /** The name of the attribute (CSV column, Shapefile attribute, etc.) in the uploaded source file. */ public String name; - /** Editable by end users */ + /** The editable label specified by the end user. */ public String label; + /** The data type of the attribute - for our purposes primarily distinguishing between numbers and text. */ public Type type; private enum Type { @@ -29,11 +36,7 @@ public SpatialAttribute(String name, AttributeType type) { else this.type = Type.ERROR; } - /** - * No-arg constructor required for Mongo POJO serialization - */ - public SpatialAttribute () { - - } + /** No-arg constructor required for Mongo POJO deserialization. */ + public SpatialAttribute () { } } From f91216f4dd0501dba73b7903a5d0b59baa573254 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 13 Aug 2021 16:56:25 +0800 Subject: [PATCH 066/187] strongly typed UserPermissions Eliminate all use of literal strings to fetch accessGroup and email Replace String-typed email/accessGroup arguments with UserPermissions Encapsulate all calls to req.attribute(String) to increase type safety Replace related string literals for Mongo properties with constants Removed some dead code and whitespace --- .../conveyal/analysis/UserPermissions.java | 14 +++++ .../conveyal/analysis/components/HttpApi.java | 2 +- .../components/broker/WorkerTags.java | 5 ++ .../AggregationAreaController.java | 10 +--- .../controllers/BrokerController.java | 17 +++--- .../controllers/BundleController.java | 10 ++-- .../controllers/FileStorageController.java | 5 +- .../controllers/GTFSGraphQLController.java | 7 ++- .../analysis/controllers/HttpController.java | 3 + .../controllers/ModificationController.java | 5 +- .../OpportunityDatasetController.java | 26 ++++----- .../controllers/ProjectController.java | 17 +++--- .../RegionalAnalysisController.java | 42 +++++++------- .../SpatialResourceController.java | 10 ++-- .../controllers/UserActivityController.java | 3 +- .../controllers/WorkerProxyController.java | 5 +- .../analysis/models/AnalysisRequest.java | 11 ++-- .../persistence/AnalysisCollection.java | 56 +++++++------------ .../analysis/persistence/MongoMap.java | 48 +++++++--------- .../conveyal/analysis/spatial/Polygons.java | 9 --- .../analysis/spatial/SpatialLayers.java | 4 +- 21 files changed, 144 insertions(+), 165 deletions(-) delete mode 100644 src/main/java/com/conveyal/analysis/spatial/Polygons.java diff --git a/src/main/java/com/conveyal/analysis/UserPermissions.java b/src/main/java/com/conveyal/analysis/UserPermissions.java index b755495e8..695d1639b 100644 --- a/src/main/java/com/conveyal/analysis/UserPermissions.java +++ b/src/main/java/com/conveyal/analysis/UserPermissions.java @@ -1,5 +1,9 @@ package com.conveyal.analysis; +import spark.Request; + +import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; + /** * Groups together all information about what a user is allowed to do. * Currently all such information is known from the group ID. @@ -19,6 +23,16 @@ public UserPermissions (String email, boolean admin, String accessGroup) { this.accessGroup = accessGroup; } + /** + * From an HTTP request object, extract a strongly typed UserPermissions object containing the user's email and + * access group. This should be used almost everywhere instead of String email and accessGroup variables. Use this + * method to encapsulate all calls to req.attribute(String) because those calls are not typesafe (they cast an Object + * to whatever type seems appropriate in the context, or is supplied by the "req.attribute(String)" syntax). + */ + public static UserPermissions from (Request req) { + return req.attribute(USER_PERMISSIONS_ATTRIBUTE); + } + @Override public String toString () { return "UserPermissions{" + diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 6149ade60..1cb2d9d39 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -109,7 +109,7 @@ private spark.Service configureSparkService () { Instant requestStartTime = req.attribute(REQUEST_START_TIME_ATTRIBUTE); Duration elapsed = Duration.between(requestStartTime, Instant.now()); eventBus.send(new HttpApiEvent(req.requestMethod(), res.status(), req.pathInfo(), elapsed.toMillis()) - .forUser(req.attribute(USER_PERMISSIONS_ATTRIBUTE))); + .forUser(UserPermissions.from(req))); }); // Handle CORS preflight requests (which are OPTIONS requests). diff --git a/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java b/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java index ceb564694..565f8981c 100644 --- a/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java +++ b/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.components.broker; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.RegionalAnalysis; /** @@ -21,6 +22,10 @@ public class WorkerTags { /** The UUID for the project. */ public final String regionId; + public WorkerTags (UserPermissions userPermissions, String projectId, String regionId) { + this(userPermissions.accessGroup, userPermissions.email, projectId, regionId); + } + public WorkerTags (String group, String user, String projectId, String regionId) { this.group = group; this.user = user; diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 88b4445bd..7c0b2af32 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -8,7 +8,6 @@ import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.file.FileStorage; -import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.progress.Task; @@ -36,11 +35,8 @@ import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; -import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; -import static com.conveyal.analysis.persistence.AnalysisCollection.getAccessGroup; import static com.conveyal.analysis.spatial.FeatureSummary.Type.POLYGON; import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; @@ -87,7 +83,7 @@ public AggregationAreaController ( */ private List createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + UserPermissions userPermissions = UserPermissions.from(req); String resourceId = req.params("resourceId"); String nameProperty = req.queryParams("nameProperty"); final int zoom = parseZoom(req.queryParams("zoom")); @@ -196,13 +192,13 @@ private String readProperty (SimpleFeature feature, String propertyName) { private Collection getAggregationAreas (Request req, Response res) { return aggregationAreaCollection.findPermitted( - and(eq("regionId", req.queryParams("regionId"))), getAccessGroup(req) + and(eq("regionId", req.queryParams("regionId"))), UserPermissions.from(req) ); } private JSONObject getAggregationArea (Request req, Response res) { AggregationArea aggregationArea = aggregationAreaCollection.findByIdIfPermitted( - req.params("maskId"), getAccessGroup(req) + req.params("maskId"), UserPermissions.from(req) ); String url = fileStorage.getURL(aggregationArea.getStorageKey()); JSONObject wrappedUrl = new JSONObject(); diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 39f9cd18f..e9444e44a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.mongodb.QueryBuilder; -import com.sun.net.httpserver.Headers; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -129,13 +128,13 @@ private Object singlePoint(Request request, Response response) { // The accessgroup stuff is copypasta from the old single point controller. // We already know the user is authenticated, and we need not check if they have access to the graphs etc, // as they're all coded with UUIDs which contain significantly more entropy than any human's account password. - final String accessGroup = request.attribute("accessGroup"); - final String userEmail = request.attribute("email"); + UserPermissions userPermissions = UserPermissions.from(request); final long startTimeMsec = System.currentTimeMillis(); AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); - Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, accessGroup); + Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, userPermissions); // Transform the analysis UI/backend task format into a slightly different type for R5 workers. - TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest.populateTask(new TravelTimeSurfaceTask(), project); + TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest + .populateTask(new TravelTimeSurfaceTask(), project, userPermissions); // If destination opportunities are supplied, prepare to calculate accessibility worker-side if (notNullOrEmpty(analysisRequest.destinationPointSetIds)){ // Look up all destination opportunity data sets from the database and derive their storage keys. @@ -146,7 +145,7 @@ private Object singlePoint(Request request, Response response) { for (String destinationPointSetId : analysisRequest.destinationPointSetIds) { OpportunityDataset opportunityDataset = Persistence.opportunityDatasets.findByIdIfPermitted( destinationPointSetId, - accessGroup + userPermissions ); checkNotNull(opportunityDataset, "Opportunity dataset could not be found in database."); opportunityDatasets.add(opportunityDataset); @@ -170,7 +169,7 @@ private Object singlePoint(Request request, Response response) { String address = broker.getWorkerAddress(workerCategory); if (address == null) { // There are no workers that can handle this request. Request some. - WorkerTags workerTags = new WorkerTags(accessGroup, userEmail, project._id, project.regionId); + WorkerTags workerTags = new WorkerTags(userPermissions, project._id, project.regionId); broker.createOnDemandWorkerInCategory(workerCategory, workerTags); // No workers exist. Kick one off and return "service unavailable". response.header("Retry-After", "30"); @@ -211,7 +210,7 @@ private Object singlePoint(Request request, Response response) { analysisRequest.projectId, analysisRequest.variantIndex, durationMsec - ).forUser(userEmail, accessGroup) + ).forUser(userPermissions) ); } // If you return a stream to the Spark Framework, its SerializerChain will copy that stream out to the @@ -366,7 +365,7 @@ private static T objectFromRequestBody (Request request, Class classe) { } private static void enforceAdmin (Request request) { - if (!request.attribute("permissions").admin) { + if (!UserPermissions.from(request).admin) { throw AnalysisServerException.forbidden("You do not have access."); } } diff --git a/src/main/java/com/conveyal/analysis/controllers/BundleController.java b/src/main/java/com/conveyal/analysis/controllers/BundleController.java index 68d4d28a7..d45a47d78 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BundleController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BundleController.java @@ -43,7 +43,6 @@ import java.util.stream.Collectors; import java.util.zip.ZipFile; -import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.r5.analyst.progress.WorkProductType.BUNDLE; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.BUNDLES; @@ -134,8 +133,9 @@ private Bundle create (Request req, Response res) { bundle.feedsComplete = bundleWithFeed.feedsComplete; bundle.totalFeeds = bundleWithFeed.totalFeeds; } - bundle.accessGroup = req.attribute("accessGroup"); - bundle.createdBy = req.attribute("email"); + UserPermissions userPermissions = UserPermissions.from(req); + bundle.accessGroup = userPermissions.accessGroup; + bundle.createdBy = userPermissions.email; } catch (Exception e) { throw AnalysisServerException.badRequest(ExceptionUtils.stackTraceString(e)); } @@ -146,7 +146,7 @@ private Bundle create (Request req, Response res) { // Submit all slower work for asynchronous processing on the backend, then immediately return the partially // constructed bundle from the HTTP handler. Process OSM first, then each GTFS feed sequentially. - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + final UserPermissions userPermissions = UserPermissions.from(req); taskScheduler.enqueue(Task.create("Processing bundle " + bundle.name) .forUser(userPermissions) .setHeavy(true) @@ -265,7 +265,7 @@ private void writeManifestToCache (Bundle bundle) throws IOException { } private Bundle deleteBundle (Request req, Response res) throws IOException { - Bundle bundle = Persistence.bundles.removeIfPermitted(req.params("_id"), req.attribute("accessGroup")); + Bundle bundle = Persistence.bundles.removeIfPermitted(req.params("_id"), UserPermissions.from(req)); FileStorageKey key = new FileStorageKey(BUNDLES, bundle._id + ".zip"); fileStorage.delete(key); diff --git a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java index d7724587b..d3c9e64f7 100644 --- a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java +++ b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.controllers; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.FileInfo; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; @@ -59,7 +60,7 @@ public void registerEndpoints (Service sparkService) { * Find all associated FileInfo records for a region. */ private List findAllForRegion(Request req, Response res) { - return fileCollection.findPermitted(and(eq("regionId", req.queryParams("regionId"))), req.attribute("accessGroup")); + return fileCollection.findPermitted(and(eq("regionId", req.queryParams("regionId"))), UserPermissions.from(req)); } /** @@ -116,7 +117,7 @@ private FileInfo uploadFile(Request req, Response res) throws Exception { // Set status to ready fileInfo.isReady = true; - fileInfo.updatedBy = req.attribute("email"); + fileInfo.updatedBy = UserPermissions.from(req).email; // Store changes to the file info fileCollection.update(fileInfo); diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java index a3ba0a264..bb9d053df 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSGraphQLController.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.controllers; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.Bundle; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.JsonUtil; @@ -66,7 +67,7 @@ private Object handleQuery (Request req, Response res) throws IOException { }); QueryContext context = new QueryContext(); - context.accessGroup = req.attribute("accessGroup"); + context.userPermissions = UserPermissions.from(req); ExecutionResult er = graphql.execute(req.queryParams("query"), null, context, variables); @@ -153,7 +154,7 @@ private Collection fetchBundle(DataFetchingEnvironment environment) { QueryContext context = (QueryContext) environment.getContext(); return Persistence.bundles.findPermitted( QueryBuilder.start("_id").in(environment.getArgument("bundle_id")).get(), - context.accessGroup + context.userPermissions ); } @@ -202,7 +203,7 @@ public void registerEndpoints (spark.Service sparkService) { /** Context for a graphql query. Currently contains authorization info. */ public static class QueryContext extends ExecutionContext { - public String accessGroup; + public UserPermissions userPermissions; } } diff --git a/src/main/java/com/conveyal/analysis/controllers/HttpController.java b/src/main/java/com/conveyal/analysis/controllers/HttpController.java index 41843990e..a6a7e1c47 100644 --- a/src/main/java/com/conveyal/analysis/controllers/HttpController.java +++ b/src/main/java/com/conveyal/analysis/controllers/HttpController.java @@ -1,5 +1,8 @@ package com.conveyal.analysis.controllers; +import com.conveyal.analysis.UserPermissions; +import spark.Request; + /** * All of our classes defining HTTP API endpoints implement this interface. * It has a single method that registers all the endpoints. diff --git a/src/main/java/com/conveyal/analysis/controllers/ModificationController.java b/src/main/java/com/conveyal/analysis/controllers/ModificationController.java index b4b3c68e8..8990bd66b 100644 --- a/src/main/java/com/conveyal/analysis/controllers/ModificationController.java +++ b/src/main/java/com/conveyal/analysis/controllers/ModificationController.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.controllers; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.AbstractTimetable; import com.conveyal.analysis.models.AddTripPattern; import com.conveyal.analysis.models.ConvertToFrequency; @@ -36,7 +37,7 @@ private Modification update (Request request, Response response) throws IOExcept } private Modification deleteModification (Request req, Response res) { - return Persistence.modifications.removeIfPermitted(req.params("_id"), req.attribute("accessGroup")); + return Persistence.modifications.removeIfPermitted(req.params("_id"), UserPermissions.from(req)); } private void mapPhaseIds (List timetables, String oldModificationId, String newModificationId) { @@ -74,7 +75,7 @@ private Modification copyModification (Request req, Response res) { clone.name = clone.name + " (copy)"; // Set `updateBy` manually, `createdBy` stays with the original modification author - clone.updatedBy = req.attribute("email"); + clone.updatedBy = UserPermissions.from(req).email; // Update the clone return Persistence.modifications.put(clone); diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 1c5e42c54..1b13ff6f4 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -54,7 +54,6 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.analysis.spatial.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; @@ -106,7 +105,7 @@ private void addStatusAndRemoveOldStatuses(OpportunityDatasetUploadStatus status private Collection getRegionDatasets(Request req, Response res) { return Persistence.opportunityDatasets.findPermitted( QueryBuilder.start("regionId").is(req.params("regionId")).get(), - req.attribute("accessGroup") + UserPermissions.from(req) ); } @@ -140,9 +139,8 @@ private boolean clearStatus(Request req, Response res) { private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) { final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); - - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - final Region region = Persistence.regions.findByIdIfPermitted(regionId, userPermissions.accessGroup); + final UserPermissions userPermissions = UserPermissions.from(req); + final Region region = Persistence.regions.findByIdIfPermitted(regionId, userPermissions); // Common UUID for all LODES datasets created in this download (e.g. so they can be grouped together and // deleted as a batch using deleteSourceSet) // The bucket name contains the specific lodes data set and year so works as an appropriate name @@ -323,7 +321,7 @@ private String getFormField(Map> formFields, String field * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. */ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Response res) { - final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + final UserPermissions userPermissions = UserPermissions.from(req); final Map> formFields; try { ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); @@ -449,26 +447,23 @@ private OpportunityDataset editOpportunityDataset(Request request, Response resp private Collection deleteSourceSet(Request request, Response response) { String sourceId = request.params("sourceId"); - String accessGroup = request.attribute("accessGroup"); + UserPermissions userPermissions = UserPermissions.from(request); Collection datasets = Persistence.opportunityDatasets.findPermitted( - QueryBuilder.start("sourceId").is(sourceId).get(), accessGroup); - - datasets.forEach(dataset -> deleteDataset(dataset._id, accessGroup)); - + QueryBuilder.start("sourceId").is(sourceId).get(), userPermissions); + datasets.forEach(dataset -> deleteDataset(dataset._id, userPermissions)); return datasets; } private OpportunityDataset deleteOpportunityDataset(Request request, Response response) { String opportunityDatasetId = request.params("_id"); - return deleteDataset(opportunityDatasetId, request.attribute("accessGroup")); + return deleteDataset(opportunityDatasetId, UserPermissions.from(request)); } /** * Delete an Opportunity Dataset from the database and all formats from the file store. */ - private OpportunityDataset deleteDataset(String id, String accessGroup) { - OpportunityDataset dataset = Persistence.opportunityDatasets.removeIfPermitted(id, accessGroup); - + private OpportunityDataset deleteDataset(String id, UserPermissions userPermissions) { + OpportunityDataset dataset = Persistence.opportunityDatasets.removeIfPermitted(id, userPermissions); if (dataset == null) { throw AnalysisServerException.notFound("Opportunity dataset could not be found."); } else { @@ -476,7 +471,6 @@ private OpportunityDataset deleteDataset(String id, String accessGroup) { fileStorage.delete(dataset.getStorageKey(FileStorageFormat.PNG)); fileStorage.delete(dataset.getStorageKey(FileStorageFormat.TIFF)); } - return dataset; } diff --git a/src/main/java/com/conveyal/analysis/controllers/ProjectController.java b/src/main/java/com/conveyal/analysis/controllers/ProjectController.java index cd7cb0440..060c3392c 100644 --- a/src/main/java/com/conveyal/analysis/controllers/ProjectController.java +++ b/src/main/java/com/conveyal/analysis/controllers/ProjectController.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.controllers; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.AddTripPattern; import com.conveyal.analysis.models.ConvertToFrequency; import com.conveyal.analysis.models.Modification; @@ -32,7 +33,7 @@ private Project findById(Request req, Response res) { private Collection getAllProjects (Request req, Response res) { return Persistence.projects.findPermitted( QueryBuilder.start("regionId").is(req.params("region")).get(), - req.attribute("accessGroup") + UserPermissions.from(req) ); } @@ -47,16 +48,16 @@ private Project update(Request req, Response res) throws IOException { private Collection modifications (Request req, Response res) { return Persistence.modifications.findPermitted( QueryBuilder.start("projectId").is(req.params("_id")).get(), - req.attribute("accessGroup") + UserPermissions.from(req) ); } private Collection importModifications (Request req, Response res) { final String importId = req.params("_importId"); final String newId = req.params("_id"); - final String accessGroup = req.attribute("accessGroup"); - final Project project = Persistence.projects.findByIdIfPermitted(newId, accessGroup); - final Project importProject = Persistence.projects.findByIdIfPermitted(importId, accessGroup); + final UserPermissions userPermissions = UserPermissions.from(req); + final Project project = Persistence.projects.findByIdIfPermitted(newId, userPermissions); + final Project importProject = Persistence.projects.findByIdIfPermitted(importId, userPermissions); final boolean bundlesAreNotEqual = !project.bundleId.equals(importProject.bundleId); QueryBuilder query = QueryBuilder.start("projectId").is(importId); @@ -64,7 +65,7 @@ private Collection importModifications (Request req, Response res) // Different bundle? Only copy add trip modifications query = query.and("type").is("add-trip-pattern"); } - final Collection modifications = Persistence.modifications.findPermitted(query.get(), accessGroup); + final Collection modifications = Persistence.modifications.findPermitted(query.get(), userPermissions); // This would be a lot easier if we just used the actual `_id`s and dealt with it elsewhere when searching. They // should be unique anyways. Hmmmmmmmmmmmm. Trade offs. @@ -86,7 +87,7 @@ private Collection importModifications (Request req, Response res) clone.name = clone.name + " (import)"; // Set `updatedBy` by manually, `createdBy` stays with the original author - clone.updatedBy = req.attribute("email"); + clone.updatedBy = UserPermissions.from(req).email; // Matched up the phased entries and timetables if (modification.getType().equals(AddTripPattern.type)) { @@ -148,7 +149,7 @@ private Collection importModifications (Request req, Response res) } private Project deleteProject (Request req, Response res) { - return Persistence.projects.removeIfPermitted(req.params("_id"), req.attribute("accessGroup")); + return Persistence.projects.removeIfPermitted(req.params("_id"), UserPermissions.from(req)); } public Collection getProjects (Request req, Response res) { diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 9035fa61d..ec9bc2383 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -2,6 +2,7 @@ import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.SelectingGridReducer; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.broker.Broker; import com.conveyal.analysis.components.broker.JobStatus; import com.conveyal.analysis.models.AnalysisRequest; @@ -73,24 +74,24 @@ public RegionalAnalysisController (Broker broker, FileStorage fileStorage) { this.fileStorage = fileStorage; } - private Collection getRegionalAnalysesForRegion(String regionId, String accessGroup) { + private Collection getRegionalAnalysesForRegion(String regionId, UserPermissions userPermissions) { return Persistence.regionalAnalyses.findPermitted( QueryBuilder.start().and( QueryBuilder.start("regionId").is(regionId).get(), QueryBuilder.start("deleted").is(false).get() ).get(), DBProjection.exclude("request.scenario.modifications"), - accessGroup + userPermissions ); } private Collection getRegionalAnalysesForRegion(Request req, Response res) { - return getRegionalAnalysesForRegion(req.params("regionId"), req.attribute("accessGroup")); + return getRegionalAnalysesForRegion(req.params("regionId"), UserPermissions.from(req)); } // Note: this includes the modifications object which can be very large private RegionalAnalysis getRegionalAnalysis(Request req, Response res) { - return Persistence.regionalAnalyses.findByIdIfPermitted(req.params("_id"), req.attribute("accessGroup")); + return Persistence.regionalAnalyses.findByIdIfPermitted(req.params("_id"), UserPermissions.from(req)); } /** @@ -99,7 +100,7 @@ private RegionalAnalysis getRegionalAnalysis(Request req, Response res) { * @return JobStatues with associated regional analysis embedded */ private Collection getRunningAnalyses(Request req, Response res) { - Collection allAnalysesInRegion = getRegionalAnalysesForRegion(req.params("regionId"), req.attribute("accessGroup")); + Collection allAnalysesInRegion = getRegionalAnalysesForRegion(req.params("regionId"), UserPermissions.from(req)); List runningStatusesForRegion = new ArrayList<>(); Collection allJobStatuses = broker.getAllJobStatuses(); for (RegionalAnalysis ra : allAnalysesInRegion) { @@ -114,19 +115,17 @@ private Collection getRunningAnalyses(Request req, Response res) { } private RegionalAnalysis deleteRegionalAnalysis (Request req, Response res) { - String accessGroup = req.attribute("accessGroup"); - String email = req.attribute("email"); - + UserPermissions userPermissions = UserPermissions.from(req); RegionalAnalysis analysis = Persistence.regionalAnalyses.findPermitted( QueryBuilder.start().and( QueryBuilder.start("_id").is(req.params("_id")).get(), QueryBuilder.start("deleted").is(false).get() ).get(), DBProjection.exclude("request.scenario.modifications"), - accessGroup + userPermissions ).iterator().next(); analysis.deleted = true; - Persistence.regionalAnalyses.updateByUserIfPermitted(analysis, email, accessGroup); + Persistence.regionalAnalyses.updateByUserIfPermitted(analysis, userPermissions); // clear it from the broker if (!analysis.complete) { @@ -173,7 +172,7 @@ private Object getRegionalResults (Request req, Response res) throws IOException RegionalAnalysis analysis = Persistence.regionalAnalyses.findPermitted( QueryBuilder.start("_id").is(req.params("_id")).get(), DBProjection.exclude("request.scenario.modifications"), - req.attribute("accessGroup") + UserPermissions.from(req) ).iterator().next(); if (analysis == null || analysis.deleted) { @@ -333,7 +332,7 @@ private String getCsvResults (Request req, Response res) { RegionalAnalysis analysis = Persistence.regionalAnalyses.findPermitted( QueryBuilder.start("_id").is(regionalAnalysisId).get(), DBProjection.exclude("request.scenario.modifications"), - req.attribute("accessGroup") + UserPermissions.from(req) ).iterator().next(); if (analysis == null || analysis.deleted) { @@ -357,8 +356,7 @@ private String getCsvResults (Request req, Response res) { * in the body of the HTTP response. */ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) throws IOException { - final String accessGroup = req.attribute("accessGroup"); - final String email = req.attribute("email"); + final UserPermissions userPermissions = UserPermissions.from(req); AnalysisRequest analysisRequest = JsonUtil.objectMapper.readValue(req.body(), AnalysisRequest.class); @@ -378,10 +376,10 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro } // Create an internal RegionalTask and RegionalAnalysis from the AnalysisRequest sent by the client. - Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, accessGroup); + Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, userPermissions); // TODO now this is setting cutoffs and percentiles in the regional (template) task. // why is some stuff set in this populate method, and other things set here in the caller? - RegionalTask task = (RegionalTask) analysisRequest.populateTask(new RegionalTask(), project); + RegionalTask task = (RegionalTask) analysisRequest.populateTask(new RegionalTask(), project, userPermissions); // Set the destination PointSets, which are required for all non-Taui regional requests. if (! analysisRequest.makeTauiSite) { @@ -395,7 +393,7 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro String destinationPointSetId = analysisRequest.destinationPointSetIds[i]; OpportunityDataset opportunityDataset = Persistence.opportunityDatasets.findByIdIfPermitted( destinationPointSetId, - accessGroup + userPermissions ); checkNotNull(opportunityDataset, "Opportunity dataset could not be found in database."); opportunityDatasets.add(opportunityDataset); @@ -430,7 +428,7 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro // Also load this freeform origin pointset instance itself, so broker can see point coordinates, ids etc. if (analysisRequest.originPointSetId != null) { task.originPointSetKey = Persistence.opportunityDatasets - .findByIdIfPermitted(analysisRequest.originPointSetId, accessGroup).storageLocation(); + .findByIdIfPermitted(analysisRequest.originPointSetId, userPermissions).storageLocation(); task.originPointSet = PointSetCache.readFreeFormFromFileStore(task.originPointSetKey); } @@ -478,9 +476,9 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro regionalAnalysis.west = task.west; regionalAnalysis.width = task.width; - regionalAnalysis.accessGroup = accessGroup; + regionalAnalysis.accessGroup = userPermissions.accessGroup; regionalAnalysis.bundleId = project.bundleId; - regionalAnalysis.createdBy = email; + regionalAnalysis.createdBy = userPermissions.email; regionalAnalysis.destinationPointSetIds = analysisRequest.destinationPointSetIds; regionalAnalysis.name = analysisRequest.name; regionalAnalysis.projectId = analysisRequest.projectId; @@ -534,10 +532,8 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro } private RegionalAnalysis updateRegionalAnalysis(Request request, Response response) throws IOException { - final String accessGroup = request.attribute("accessGroup"); - final String email = request.attribute("email"); RegionalAnalysis regionalAnalysis = JsonUtil.objectMapper.readValue(request.body(), RegionalAnalysis.class); - return Persistence.regionalAnalyses.updateByUserIfPermitted(regionalAnalysis, email, accessGroup); + return Persistence.regionalAnalyses.updateByUserIfPermitted(regionalAnalysis, UserPermissions.from(request)); } @Override diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java index d3443d01c..018aa03f6 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.StringJoiner; -import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.analysis.spatial.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.RESOURCES; @@ -65,7 +64,7 @@ public SpatialResourceController ( private List getRegionResources (Request req, Response res) { return spatialResourceCollection.findPermitted( and(eq("regionId", req.params("regionId"))), - req.attribute("accessGroup") + UserPermissions.from(req) ); } @@ -76,7 +75,7 @@ private Object getResource (Request req, Response res) { private SpatialResource downloadLODES(Request req, Response res) { final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + UserPermissions userPermissions = UserPermissions.from(req); SpatialResource source = SpatialResource.create(userPermissions, extractor.sourceName) .withRegion(regionId); @@ -118,7 +117,7 @@ private String getFormField(Map> formFields, String field * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. */ private SpatialResource handleUpload(Request req, Response res) { - final UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + final UserPermissions userPermissions = UserPermissions.from(req); final Map> formFields = HttpUtils.getRequestFiles(req.raw()); // Parse required fields. Will throw a ServerException on failure. @@ -170,10 +169,9 @@ private Collection deleteResource (Request request, Response re // TODO delete files from storage // TODO delete referencing database records spatialResourceCollection.delete(source); - return spatialResourceCollection.findPermitted( and(eq("regionId", request.params("regionId"))), - request.attribute("accessGroup") + UserPermissions.from(request) ); } diff --git a/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java b/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java index c72220079..e09d450a3 100644 --- a/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java +++ b/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java @@ -9,7 +9,6 @@ import java.util.List; -import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.conveyal.analysis.util.JsonUtil.toJson; /** @@ -41,7 +40,7 @@ public void registerEndpoints (Service sparkService) { } private ResponseModel getActivity (Request req, Response res) { - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); + UserPermissions userPermissions = UserPermissions.from(req); ResponseModel responseModel = new ResponseModel(); responseModel.systemStatusMessages = List.of(); responseModel.taskBacklog = taskScheduler.getBacklog(); diff --git a/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java b/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java index 84b0d34e9..1a18cd1d2 100644 --- a/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java +++ b/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java @@ -1,5 +1,6 @@ package com.conveyal.analysis.controllers; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.broker.Broker; import com.conveyal.analysis.components.broker.WorkerTags; import com.conveyal.analysis.models.Bundle; @@ -60,9 +61,7 @@ private Object proxyGet (Request request, Response response) { if (address == null) { Bundle bundle = null; // There are no workers that can handle this request. Request one and ask the UI to retry later. - final String accessGroup = request.attribute("accessGroup"); - final String userEmail = request.attribute("email"); - WorkerTags workerTags = new WorkerTags(accessGroup, userEmail, "anyProjectId", bundle.regionId); + WorkerTags workerTags = new WorkerTags(UserPermissions.from(request), "anyProjectId", bundle.regionId); broker.createOnDemandWorkerInCategory(workerCategory, workerTags); response.status(HttpStatus.ACCEPTED_202); response.header("Retry-After", "30"); diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 7eb39b629..7d63bd947 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.models; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; @@ -153,12 +154,12 @@ public class AnalysisRequest { * corresponding r5 mod */ private static List modificationsForProject ( - String accessGroup, + UserPermissions userPermissions, String projectId, int variantIndex) { return Persistence.modifications - .findPermitted(QueryBuilder.start("projectId").is(projectId).get(), accessGroup) + .findPermitted(QueryBuilder.start("projectId").is(projectId).get(), userPermissions) .stream() .filter(m -> variantIndex < m.variants.length && m.variants[variantIndex]) .map(com.conveyal.analysis.models.Modification::toR5) @@ -178,7 +179,7 @@ private static List modificationsForProject ( * TODO arguably this should be done by a method on the task classes themselves, with common parts factored out * to the same method on the superclass. */ - public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project) { + public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project, UserPermissions userPermissions) { // Fetch the modifications associated with this project, filtering for the selected scenario // (denoted here as "variant"). There are no modifications in the baseline scenario @@ -189,7 +190,7 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project if (variantIndex >= project.variants.length) { throw AnalysisServerException.badRequest("Scenario does not exist. Please select a new scenario."); } - modifications = modificationsForProject(project.accessGroup, projectId, variantIndex); + modifications = modificationsForProject(userPermissions, projectId, variantIndex); scenarioName = project.variants[variantIndex]; } else { scenarioName = "Baseline"; @@ -220,7 +221,7 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, Project project Bounds bounds = this.bounds; if (bounds == null) { // If no bounds were specified, fall back on the bounds of the entire region. - Region region = Persistence.regions.findByIdIfPermitted(project.regionId, project.accessGroup); + Region region = Persistence.regions.findByIdIfPermitted(project.regionId, new UserPermissions("UNKNOWN", false, project.accessGroup)); bounds = region.bounds; } diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 5421c009e..671dc89d6 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -17,23 +17,15 @@ import java.util.ArrayList; import java.util.List; -import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; public class AnalysisCollection { - public MongoCollection collection; - private Class type; + public static final String MONGO_PROP_ACCESS_GROUP = "accessGroup"; - /** - * Helper method to extract accessGroup from the UserPermissions set by the authentication component on a request. - * The redundant request attributes for email and accessGroup may be removed once methods like this are in use. - */ - public static String getAccessGroup (Request req) { - UserPermissions userPermissions = req.attribute(USER_PERMISSIONS_ATTRIBUTE); - return userPermissions.accessGroup; - } + public final MongoCollection collection; + private final Class type; private AnalysisServerException invalidAccessGroup() { return AnalysisServerException.forbidden("Permission denied. Invalid access group."); @@ -48,8 +40,8 @@ public DeleteResult delete (T value) { return collection.deleteOne(eq("_id", value._id)); } - public List findPermitted(Bson query, String accessGroup) { - return find(and(eq("accessGroup", accessGroup), query)); + public List findPermitted(Bson query, UserPermissions userPermissions) { + return find(and(eq(MONGO_PROP_ACCESS_GROUP, userPermissions.accessGroup), query)); } public List find(Bson query) { @@ -69,9 +61,9 @@ public T findById(ObjectId _id) { return collection.find(eq("_id", _id)).first(); } - public T findByIdIfPermitted (String _id, String accessGroup) { + public T findByIdIfPermitted (String _id, UserPermissions userPermissions) { T item = findById(_id); - if (item.accessGroup.equals(accessGroup)) { + if (item.accessGroup.equals(userPermissions.accessGroup)) { return item; } else { // TODO: To simplify stack traces this should be refactored to "throw new InvalidAccessGroupException()" @@ -80,10 +72,10 @@ public T findByIdIfPermitted (String _id, String accessGroup) { } } - public T create(T newModel, String accessGroup, String creatorEmail) { - newModel.accessGroup = accessGroup; - newModel.createdBy = creatorEmail; - newModel.updatedBy = creatorEmail; + public T create(T newModel, UserPermissions userPermissions) { + newModel.accessGroup = userPermissions.accessGroup; + newModel.createdBy = userPermissions.email; + newModel.updatedBy = userPermissions.email; // This creates the `_id` automatically if it is missing collection.insertOne(newModel); @@ -112,7 +104,7 @@ public T update(T value, String accessGroup) { UpdateResult result = collection.replaceOne(and( eq("_id", value._id), eq("nonce", oldNonce), - eq("accessGroup", accessGroup) + eq(MONGO_PROP_ACCESS_GROUP, accessGroup) ), value); // If no documents were modified try to find the document to find out why @@ -140,24 +132,19 @@ public T update(T value, String accessGroup) { */ public T create(Request req, Response res) throws IOException { T value = JsonUtil.objectMapper.readValue(req.body(), type); - - String accessGroup = getAccessGroup(req); - String email = req.attribute("email"); - return create(value, accessGroup, email); + return create(value, UserPermissions.from(req)); } /** * Controller find by id helper. */ public T findPermittedByRequestParamId(Request req, Response res) { - String accessGroup = getAccessGroup(req); + UserPermissions user = UserPermissions.from(req); T value = findById(req.params("_id")); - // Throw if or does not have permission - if (!value.accessGroup.equals(accessGroup)) { + if (!value.accessGroup.equals(user.accessGroup)) { throw invalidAccessGroup(); } - return value; } @@ -166,12 +153,11 @@ public T findPermittedByRequestParamId(Request req, Response res) { */ public T update(Request req, Response res) throws IOException { T value = JsonUtil.objectMapper.readValue(req.body(), type); - - String accessGroup = getAccessGroup(req); - value.updatedBy = req.attribute("email"); - - if (!value.accessGroup.equals(accessGroup)) throw invalidAccessGroup(); - - return update(value, accessGroup); + final UserPermissions user = UserPermissions.from(req); + value.updatedBy = user.email; + if (!value.accessGroup.equals(user.accessGroup)) { + throw invalidAccessGroup(); + } + return update(value, user.accessGroup); } } diff --git a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java index b88bddac4..6001a82d2 100644 --- a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java +++ b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.persistence; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.Model; import com.conveyal.r5.common.JsonUtilities; import com.mongodb.BasicDBObject; @@ -18,6 +19,8 @@ import java.io.IOException; import java.util.Collection; +import static com.conveyal.analysis.persistence.AnalysisCollection.MONGO_PROP_ACCESS_GROUP; + /** * An attempt at simulating a MapDB-style interface, for storing Java objects in MongoDB. * Note this used to implement Map, but the Map interface predates generics in Java, so it is more typesafe not @@ -31,10 +34,6 @@ public class MongoMap { private JacksonDBCollection wrappedCollection; private Class type; - private String getAccessGroup (Request req) { - return req.attribute("accessGroup"); - } - public MongoMap (JacksonDBCollection wrappedCollection, Class type) { this.type = type; this.wrappedCollection = wrappedCollection; @@ -45,17 +44,17 @@ public int size() { } public V findByIdFromRequestIfPermitted(Request request) { - return findByIdIfPermitted(request.params("_id"), getAccessGroup(request)); + return findByIdIfPermitted(request.params("_id"), UserPermissions.from(request)); } - public V findByIdIfPermitted(String id, String accessGroup) { + public V findByIdIfPermitted(String id, UserPermissions userPermissions) { V result = wrappedCollection.findOneById(id); if (result == null) { throw AnalysisServerException.notFound(String.format( "The resource you requested (_id %s) could not be found. Has it been deleted?", id )); - } else if (!accessGroup.equals(result.accessGroup)) { + } else if (!userPermissions.accessGroup.equals(result.accessGroup)) { throw AnalysisServerException.forbidden("You do not have permission to access this data."); } else { return result; @@ -67,27 +66,27 @@ public V get(String key) { } public Collection findAllForRequest(Request req) { - return find(QueryBuilder.start("accessGroup").is(getAccessGroup(req)).get()).toArray(); + return find(QueryBuilder.start(MONGO_PROP_ACCESS_GROUP).is(UserPermissions.from(req).accessGroup).get()).toArray(); } /** * Helper function that adds the `accessGroup` to the query if the user is not an admin. If you want to query using * the `accessGroup` as an admin it must be added to the query. */ - public Collection findPermitted(DBObject query, String accessGroup) { + public Collection findPermitted(DBObject query, UserPermissions userPermissions) { DBCursor cursor = find(QueryBuilder.start().and( query, - QueryBuilder.start("accessGroup").is(accessGroup).get() + QueryBuilder.start(MONGO_PROP_ACCESS_GROUP).is(userPermissions.accessGroup).get() ).get()); return cursor.toArray(); } // See comments for `findPermitted` above. This helper also adds a projection. - public Collection findPermitted(DBObject query, DBObject project, String accessGroup) { + public Collection findPermitted(DBObject query, DBObject project, UserPermissions userPermissions) { DBCursor cursor = find(QueryBuilder.start().and( query, - QueryBuilder.start("accessGroup").is(accessGroup).get() + QueryBuilder.start(MONGO_PROP_ACCESS_GROUP).is(userPermissions.accessGroup).get() ).get(), project); return cursor.toArray(); @@ -102,8 +101,7 @@ public Collection findPermittedForQuery (Request req) { req.queryParams().forEach(name -> { query.and(name).is(req.queryParams(name)); }); - - return findPermitted(query.get(), getAccessGroup(req)); + return findPermitted(query.get(), UserPermissions.from(req)); } /** @@ -124,13 +122,9 @@ public Collection getByProperty (String property, Object value) { public V createFromJSONRequest(Request request) throws IOException { V json = JsonUtilities.objectMapper.readValue(request.body(), this.type); - - // Set access group - json.accessGroup = getAccessGroup(request); - - // Set `createdBy` from the user's email - json.createdBy = request.attribute("email"); - + UserPermissions userPermissions = UserPermissions.from(request); + json.accessGroup = userPermissions.accessGroup; + json.createdBy = userPermissions.email; return create(json); } @@ -159,14 +153,14 @@ public V create(V value) { public V updateFromJSONRequest(Request request) throws IOException { V json = JsonUtilities.objectMapper.readValue(request.body(), this.type); // Add the additional check for the same access group - return updateByUserIfPermitted(json, request.attribute("email"), getAccessGroup(request)); + return updateByUserIfPermitted(json, UserPermissions.from(request)); } - public V updateByUserIfPermitted(V value, String updatedBy, String accessGroup) { + public V updateByUserIfPermitted(V value, UserPermissions userPermissions) { // Set `updatedBy` - value.updatedBy = updatedBy; + value.updatedBy = userPermissions.email; - return put(value, QueryBuilder.start("accessGroup").is(accessGroup).get()); + return put(value, QueryBuilder.start(MONGO_PROP_ACCESS_GROUP).is(userPermissions.accessGroup).get()); } public V put(String key, V value) { @@ -228,10 +222,10 @@ public V modifiyWithoutUpdatingLock (V value) { return value; } - public V removeIfPermitted(String key, String accessGroup) { + public V removeIfPermitted(String key, UserPermissions userPermissions) { DBObject query = QueryBuilder.start().and( QueryBuilder.start("_id").is(key).get(), - QueryBuilder.start("accessGroup").is(accessGroup).get() + QueryBuilder.start(MONGO_PROP_ACCESS_GROUP).is(userPermissions.accessGroup).get() ).get(); V result = wrappedCollection.findAndRemove(query); diff --git a/src/main/java/com/conveyal/analysis/spatial/Polygons.java b/src/main/java/com/conveyal/analysis/spatial/Polygons.java deleted file mode 100644 index 32fe52c60..000000000 --- a/src/main/java/com/conveyal/analysis/spatial/Polygons.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.conveyal.analysis.spatial; - -public class Polygons { - - // TODO toGrid from shapefile and geojson - - // TODO modification polygon - -} diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java b/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java index 7a46a02aa..139b4b176 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java +++ b/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java @@ -12,9 +12,9 @@ import java.util.Set; /** - * Utility class with common methods for validating and processing uploaded spatial data files. + * Utility class with common static methods for validating and processing uploaded spatial data files. */ -public class SpatialLayers { +public abstract class SpatialLayers { /** * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary From a6f7055ec0ae5275a839b5505a375d7df06a56fc Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 13 Aug 2021 20:18:39 +0800 Subject: [PATCH 067/187] remove apparent no-op single-operand "and"s --- .../analysis/controllers/AggregationAreaController.java | 2 +- .../analysis/controllers/FileStorageController.java | 4 +++- .../analysis/controllers/SpatialResourceController.java | 6 ++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 7c0b2af32..d575bb5c5 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -192,7 +192,7 @@ private String readProperty (SimpleFeature feature, String propertyName) { private Collection getAggregationAreas (Request req, Response res) { return aggregationAreaCollection.findPermitted( - and(eq("regionId", req.queryParams("regionId"))), UserPermissions.from(req) + eq("regionId", req.queryParams("regionId")), UserPermissions.from(req) ); } diff --git a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java index d3c9e64f7..b69e37795 100644 --- a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java +++ b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java @@ -60,7 +60,9 @@ public void registerEndpoints (Service sparkService) { * Find all associated FileInfo records for a region. */ private List findAllForRegion(Request req, Response res) { - return fileCollection.findPermitted(and(eq("regionId", req.queryParams("regionId"))), UserPermissions.from(req)); + return fileCollection.findPermitted( + eq("regionId", req.queryParams("regionId")), UserPermissions.from(req) + ); } /** diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java index 018aa03f6..16599d066 100644 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java @@ -63,8 +63,7 @@ public SpatialResourceController ( private List getRegionResources (Request req, Response res) { return spatialResourceCollection.findPermitted( - and(eq("regionId", req.params("regionId"))), - UserPermissions.from(req) + eq("regionId", req.params("regionId")), UserPermissions.from(req) ); } @@ -170,8 +169,7 @@ private Collection deleteResource (Request request, Response re // TODO delete referencing database records spatialResourceCollection.delete(source); return spatialResourceCollection.findPermitted( - and(eq("regionId", request.params("regionId"))), - UserPermissions.from(request) + eq("regionId", request.params("regionId")), UserPermissions.from(request) ); } From cfc13590b51ed6fe1a7470e8a06b5f688feda1b9 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 13 Aug 2021 20:23:19 +0800 Subject: [PATCH 068/187] remove BSON annotations, use basic codec registry according to documentation, Mongo driver will properly handle enums. if not, the problem might be solved by manually registering a codec. --- .../analysis/persistence/AnalysisDB.java | 18 +++++++++++------- .../com/conveyal/file/FileStorageFormat.java | 4 +--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 8777e2d37..3dac06403 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -5,12 +5,14 @@ import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; +import org.bson.codecs.configuration.CodecProvider; import org.bson.codecs.configuration.CodecRegistry; import org.bson.codecs.pojo.Conventions; import org.bson.codecs.pojo.PojoCodecProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry; import static org.bson.codecs.configuration.CodecRegistries.fromProviders; import static org.bson.codecs.configuration.CodecRegistries.fromRegistries; @@ -29,14 +31,16 @@ public AnalysisDB (Config config) { mongo = MongoClients.create(); } - // Create codec registry for POJOs + // Create a codec registry that has all the default codecs (dates, geojson, etc.) and falls back to a provider + // that automatically generates codecs for any other Java class it encounters, based on their public getter and + // setter methods and public fields, skipping any properties whose underlying fields are transient or static. + // These classes must have an empty public or protected zero-argument constructor. + // In the documentation a "discriminator" refers to a field that identifies the Java type, like @JsonTypes. + CodecProvider automaticPojoCodecProvider = PojoCodecProvider.builder().automatic(true).build(); CodecRegistry pojoCodecRegistry = fromRegistries( - MongoClientSettings.getDefaultCodecRegistry(), - fromProviders(PojoCodecProvider.builder() - .conventions(Conventions.DEFAULT_CONVENTIONS) - .automatic(true) - .build() - )); + getDefaultCodecRegistry(), + fromProviders(automaticPojoCodecProvider) + ); database = mongo.getDatabase(config.databaseName()).withCodecRegistry(pojoCodecRegistry); diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index fae6cae68..2a52dfe90 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -18,10 +18,8 @@ public enum FileStorageFormat { SHP("shp", "application/octet-stream"), GEOJSON("json", "application/json"); - - @BsonIgnore + // These should not be serialized into Mongo. Default Enum codec uses String name() and valueOf(String). public final String extension; - @BsonIgnore public final String mimeType; FileStorageFormat (String extension, String mimeType) { From 16fb2d9fd15a82a4a2b542961654c35c5539003f Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 18 Aug 2021 22:36:32 +0800 Subject: [PATCH 069/187] WIP refactoring data source upload --- .../components/BackendComponents.java | 4 +- .../AggregationAreaController.java | 19 +- .../controllers/DataSourceController.java | 151 ++++++++++++++ .../OpportunityDatasetController.java | 17 +- .../SpatialResourceController.java | 186 ------------------ .../datasource/DataSourceIngester.java | 153 ++++++++++++++ .../{spatial => datasource}/Lines.java | 2 +- .../{spatial => datasource}/Points.java | 2 +- .../ShapefileDataSourceIngester.java | 71 +++++++ .../SpatialAttribute.java | 2 +- .../SpatialLayers.java | 5 +- .../analysis/models/AggregationArea.java | 2 +- .../conveyal/analysis/models/BaseModel.java | 9 +- .../com/conveyal/analysis/models/Bounds.java | 8 + .../conveyal/analysis/models/DataSource.java | 46 +++++ .../models/DataSourceValidationIssue.java | 13 ++ .../analysis/models/SpatialDataSource.java | 65 ++++++ .../analysis/models/SpatialResource.java | 103 ---------- .../analysis/spatial/FeatureSummary.java | 40 ---- .../com/conveyal/analysis/util/HttpUtils.java | 5 +- .../java/com/conveyal/file/FileCategory.java | 2 +- .../com/conveyal/file/FileStorageFormat.java | 10 +- .../java/com/conveyal/r5/analyst/Grid.java | 2 +- .../conveyal/r5/analyst/progress/Task.java | 2 +- .../r5/analyst/progress/WorkProductType.java | 10 +- .../com/conveyal/r5/util/ShapefileReader.java | 27 ++- 26 files changed, 579 insertions(+), 377 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/controllers/DataSourceController.java delete mode 100644 src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java create mode 100644 src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java rename src/main/java/com/conveyal/analysis/{spatial => datasource}/Lines.java (62%) rename src/main/java/com/conveyal/analysis/{spatial => datasource}/Points.java (85%) create mode 100644 src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java rename src/main/java/com/conveyal/analysis/{spatial => datasource}/SpatialAttribute.java (97%) rename src/main/java/com/conveyal/analysis/{spatial => datasource}/SpatialLayers.java (94%) create mode 100644 src/main/java/com/conveyal/analysis/models/DataSource.java create mode 100644 src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java create mode 100644 src/main/java/com/conveyal/analysis/models/SpatialDataSource.java delete mode 100644 src/main/java/com/conveyal/analysis/models/SpatialResource.java delete mode 100644 src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 408a8356d..330386ce2 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -14,7 +14,7 @@ import com.conveyal.analysis.controllers.OpportunityDatasetController; import com.conveyal.analysis.controllers.ProjectController; import com.conveyal.analysis.controllers.RegionalAnalysisController; -import com.conveyal.analysis.controllers.SpatialResourceController; +import com.conveyal.analysis.controllers.DataSourceController; import com.conveyal.analysis.controllers.TimetableController; import com.conveyal.analysis.controllers.UserActivityController; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; @@ -104,7 +104,7 @@ public List standardHttpControllers () { new BrokerController(broker, eventBus), new UserActivityController(taskScheduler), new GtfsTileController(gtfsCache), - new SpatialResourceController(fileStorage, database, taskScheduler, censusExtractor) + new DataSourceController(fileStorage, database, taskScheduler, censusExtractor) ); } diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index d575bb5c5..6c5981d85 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -4,7 +4,8 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.models.AggregationArea; -import com.conveyal.analysis.models.SpatialResource; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.file.FileStorage; @@ -35,11 +36,11 @@ import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; -import static com.conveyal.analysis.spatial.FeatureSummary.Type.POLYGON; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -60,7 +61,7 @@ public class AggregationAreaController implements HttpController { private final FileStorage fileStorage; private final TaskScheduler taskScheduler; private final AnalysisCollection aggregationAreaCollection; - private final AnalysisCollection spatialResourceCollection; + private final AnalysisCollection dataSourceCollection; public AggregationAreaController ( FileStorage fileStorage, @@ -70,7 +71,7 @@ public AggregationAreaController ( this.fileStorage = fileStorage; this.taskScheduler = taskScheduler; this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); - this.spatialResourceCollection = database.getAnalysisCollection("spatialSources", SpatialResource.class); + this.dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); } /** @@ -84,18 +85,18 @@ public AggregationAreaController ( private List createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); UserPermissions userPermissions = UserPermissions.from(req); - String resourceId = req.params("resourceId"); + String dataSourceId = req.params("dataSourceId"); String nameProperty = req.queryParams("nameProperty"); final int zoom = parseZoom(req.queryParams("zoom")); // 1. Get file from storage and read its features. ============================================================= - SpatialResource resource = spatialResourceCollection.findById(resourceId); - Preconditions.checkArgument(POLYGON.equals(resource.features.type), + SpatialDataSource resource = dataSourceCollection.findById(dataSourceId); + Preconditions.checkArgument(POLYGON.equals(resource.geometryType), "Only polygons can be converted to aggregation areas."); File sourceFile; List features = null; - if (SHP.equals(resource.sourceFormat)) { + if (SHP.equals(resource.fileFormat)) { sourceFile = fileStorage.getFile(resource.storageKey()); ShapefileReader reader = null; try { @@ -106,7 +107,7 @@ private List createAggregationAreas (Request req, Response res) } } - if (GEOJSON.equals(resource.sourceFormat)) { + if (GEOJSON.equals(resource.fileFormat)) { // TODO implement } diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java new file mode 100644 index 000000000..7c46b1c76 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -0,0 +1,151 @@ +package com.conveyal.analysis.controllers; + +import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.components.TaskScheduler; +import com.conveyal.analysis.datasource.DataSourceIngester; +import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.analysis.persistence.AnalysisDB; +import com.conveyal.analysis.util.HttpUtils; +import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; +import com.conveyal.r5.analyst.progress.Task; +import com.conveyal.r5.analyst.progress.WorkProduct; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.StringJoiner; + +import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; +import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; +import static com.conveyal.analysis.util.JsonUtil.toJson; +import static com.conveyal.file.FileCategory.DATASOURCES; +import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.conveyal.r5.analyst.progress.WorkProductType.DATA_SOURCE; +import static com.mongodb.client.model.Filters.and; +import static com.mongodb.client.model.Filters.eq; + +/** + * Controller that handles CRUD of DataSources, which are Mongo metadata about user-uploaded files. + * Unlike some Mongo documents, these are mostly created and updated by backend validation and processing methods. + * Currently this handles only one subtype: SpatialDataSource, which represents GIS-like geospatial feature data. + */ +public class DataSourceController implements HttpController { + + private static final Logger LOG = LoggerFactory.getLogger(DataSourceController.class); + + // Component Dependencies + private final FileStorage fileStorage; + private final TaskScheduler taskScheduler; + private final SeamlessCensusGridExtractor extractor; + + // Collection in the database holding all our DataSources, which can be of several subtypes. + private final AnalysisCollection dataSourceCollection; + + public DataSourceController ( + FileStorage fileStorage, + AnalysisDB database, + TaskScheduler taskScheduler, + SeamlessCensusGridExtractor extractor + ) { + this.fileStorage = fileStorage; + this.taskScheduler = taskScheduler; + this.extractor = extractor; + // We don't hold on to the AnalysisDB Component, just get one collection from it. + this.dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); + } + + private List getAllDataSourcesForRegion (Request req, Response res) { + return dataSourceCollection.findPermitted( + eq("regionId", req.params("regionId")), UserPermissions.from(req) + ); + } + + private Object getOneDataSourceById (Request req, Response res) { + return dataSourceCollection.findPermittedByRequestParamId(req, res); + } + + private SpatialDataSource downloadLODES(Request req, Response res) { + final String regionId = req.params("regionId"); + final int zoom = parseZoom(req.queryParams("zoom")); + UserPermissions userPermissions = UserPermissions.from(req); + SpatialDataSource source = new SpatialDataSource(userPermissions, extractor.sourceName); + source.regionId = regionId; + + taskScheduler.enqueue(Task.create("Extracting LODES data") + .forUser(userPermissions) + .setHeavy(true) + .withWorkProduct(source) + .withAction((progressListener) -> { + // TODO implement + })); + + return source; + } + + /** + * A file is posted to this endpoint to create a new DataSource. It is validated and metadata are extracted. + * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. + * In a standard REST API, a post would return the ID of the newly created DataSource. Here we're starting an async + * background process, so we return the task or its work product? + */ + private WorkProduct handleUpload (Request req, Response res) { + final UserPermissions userPermissions = UserPermissions.from(req); + final Map> formFields = HttpUtils.getRequestFiles(req.raw()); + + DataSourceIngester ingester = DataSourceIngester.forFormFields( + fileStorage, dataSourceCollection, formFields, userPermissions + ); + + Task backgroundTask = Task.create("Processing uploaded files: " + ingester.getDataSourceName()) + .forUser(userPermissions) + //.withWorkProduct(dataSource) + // or should TaskActions have a method to return their work product? + // Or a WorkProductDescriptor, with type, region, and ID? + // TaskActions could define methods to return a title, workProductDescriptor, etc. + // Then we just have taskScheduler.enqueue(Task.forAction(user, ingester)); + .withWorkProduct(DATA_SOURCE, ingester.getDataSourceId(), ingester.getRegionId()) + .withAction(ingester); + + taskScheduler.enqueue(backgroundTask); + return backgroundTask.workProduct; + } + + private Collection deleteOneDataSourceById (Request request, Response response) { + DataSource source = dataSourceCollection.findPermittedByRequestParamId(request, response); + // TODO delete files from storage + // TODO delete referencing database records + // Shouldn't this be deleting by ID instead of sending the whole document? + dataSourceCollection.delete(source); + // TODO why do our delete methods return a list of documents? Can we just return the ID or HTTP status code? + // Isn't this going to fail since the document was just deleted? + return dataSourceCollection.findPermitted( + eq("regionId", request.params("regionId")), UserPermissions.from(request) + ); + } + + @Override + public void registerEndpoints (spark.Service sparkService) { + sparkService.path("/api/datasource", () -> { + sparkService.get("/:_id", this::getOneDataSourceById, toJson); + sparkService.get("/region/:regionId", this::getAllDataSourcesForRegion, toJson); + sparkService.delete("/:_id", this::deleteOneDataSourceById, toJson); + sparkService.post("", this::handleUpload, toJson); + sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); + }); + } +} diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 1b13ff6f4..c4fa6aea8 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -6,7 +6,7 @@ import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.Region; -import com.conveyal.analysis.models.SpatialResource; +import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.FileItemInputStreamProvider; import com.conveyal.file.FileStorage; @@ -54,7 +54,7 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import static com.conveyal.analysis.spatial.SpatialLayers.detectUploadFormatAndValidate; +import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; @@ -147,8 +147,8 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) final OpportunityDatasetUploadStatus status = new OpportunityDatasetUploadStatus(regionId, extractor.sourceName); addStatusAndRemoveOldStatuses(status); - SpatialResource source = SpatialResource.create(userPermissions, extractor.sourceName) - .withRegion(regionId); + SpatialDataSource source = new SpatialDataSource(userPermissions, extractor.sourceName); + source.regionId = regionId; taskScheduler.enqueue(Task.create("Extracting LODES data") .forUser(userPermissions) @@ -172,7 +172,7 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) * Given a list of new PointSets, serialize each PointSet and save it to S3, then create a metadata object about * that PointSet and store it in Mongo. */ - private void updateAndStoreDatasets (SpatialResource source, + private void updateAndStoreDatasets (SpatialDataSource source, OpportunityDatasetUploadStatus status, List pointSets) { status.status = Status.UPLOADING; @@ -297,8 +297,9 @@ private List createFreeFormPointSetsFromCsv(FileItem csvFileIt /** * Get the specified field from a map representing a multipart/form-data POST request, as a UTF-8 String. * FileItems represent any form item that was received within a multipart/form-data POST request, not just files. + * This is a static utility method that should be reusable across different HttpControllers. */ - private String getFormField(Map> formFields, String fieldName, boolean required) { + public static String getFormField(Map> formFields, String fieldName, boolean required) { try { List fileItems = formFields.get(fieldName); if (fileItems == null || fileItems.isEmpty()) { @@ -404,8 +405,8 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res // Create a single unique ID string that will be referenced by all opportunity datasets produced by // this upload. This allows us to group together datasets from the same source and associate them with // the file(s) that produced them. - SpatialResource source = SpatialResource.create(userPermissions, sourceName) - .withRegion(regionId); + SpatialDataSource source = new SpatialDataSource(userPermissions, sourceName); + source.regionId = regionId; updateAndStoreDatasets(source, status, pointsets); } catch (Exception e) { status.completeWithError(e); diff --git a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java b/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java deleted file mode 100644 index 16599d066..000000000 --- a/src/main/java/com/conveyal/analysis/controllers/SpatialResourceController.java +++ /dev/null @@ -1,186 +0,0 @@ -package com.conveyal.analysis.controllers; - -import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.UserPermissions; -import com.conveyal.analysis.components.TaskScheduler; -import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; -import com.conveyal.analysis.models.SpatialResource; -import com.conveyal.analysis.persistence.AnalysisCollection; -import com.conveyal.analysis.persistence.AnalysisDB; -import com.conveyal.analysis.util.HttpUtils; -import com.conveyal.file.FileStorage; -import com.conveyal.file.FileStorageFormat; -import com.conveyal.file.FileStorageKey; -import com.conveyal.r5.analyst.progress.Task; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.disk.DiskFileItem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; - -import java.io.File; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.StringJoiner; - -import static com.conveyal.analysis.spatial.SpatialLayers.detectUploadFormatAndValidate; -import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.conveyal.file.FileCategory.RESOURCES; -import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; -import static com.conveyal.r5.analyst.progress.WorkProductType.RESOURCE; -import static com.mongodb.client.model.Filters.and; -import static com.mongodb.client.model.Filters.eq; - -/** - * Controller that handles CRUD of spatial resources. - */ -public class SpatialResourceController implements HttpController { - - private static final Logger LOG = LoggerFactory.getLogger(SpatialResourceController.class); - - // Component Dependencies - private final FileStorage fileStorage; - private final AnalysisCollection spatialResourceCollection; - private final TaskScheduler taskScheduler; - private final SeamlessCensusGridExtractor extractor; - - public SpatialResourceController ( - FileStorage fileStorage, - AnalysisDB database, - TaskScheduler taskScheduler, - SeamlessCensusGridExtractor extractor - ) { - this.fileStorage = fileStorage; - this.spatialResourceCollection = database.getAnalysisCollection("spatialResources", SpatialResource.class); - this.taskScheduler = taskScheduler; - this.extractor = extractor; - } - - private List getRegionResources (Request req, Response res) { - return spatialResourceCollection.findPermitted( - eq("regionId", req.params("regionId")), UserPermissions.from(req) - ); - } - - private Object getResource (Request req, Response res) { - return spatialResourceCollection.findPermittedByRequestParamId(req, res); - } - - private SpatialResource downloadLODES(Request req, Response res) { - final String regionId = req.params("regionId"); - final int zoom = parseZoom(req.queryParams("zoom")); - UserPermissions userPermissions = UserPermissions.from(req); - SpatialResource source = SpatialResource.create(userPermissions, extractor.sourceName) - .withRegion(regionId); - - taskScheduler.enqueue(Task.create("Extracting LODES data") - .forUser(userPermissions) - .setHeavy(true) - .withWorkProduct(source) - .withAction((progressListener) -> { - // TODO implement - })); - - return source; - } - - /** - * Get the specified field from a map representing a multipart/form-data POST request, as a UTF-8 String. - * FileItems represent any form item that was received within a multipart/form-data POST request, not just files. - */ - private String getFormField(Map> formFields, String fieldName, boolean required) { - try { - List fileItems = formFields.get(fieldName); - if (fileItems == null || fileItems.isEmpty()) { - if (required) { - throw AnalysisServerException.badRequest("Missing required field: " + fieldName); - } else { - return null; - } - } - String value = fileItems.get(0).getString("UTF-8"); - return value; - } catch (UnsupportedEncodingException e) { - throw AnalysisServerException.badRequest(String.format("Multipart form field '%s' had unsupported encoding", - fieldName)); - } - } - - /** - * Handle many types of spatial upload. - * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. - */ - private SpatialResource handleUpload(Request req, Response res) { - final UserPermissions userPermissions = UserPermissions.from(req); - final Map> formFields = HttpUtils.getRequestFiles(req.raw()); - - // Parse required fields. Will throw a ServerException on failure. - final String sourceName = getFormField(formFields, "sourceName", true); - final String regionId = getFormField(formFields, "regionId", true); - - // Initialize model object - SpatialResource source = SpatialResource.create(userPermissions, sourceName).withRegion(regionId); - - taskScheduler.enqueue(Task.create("Uploading spatial files: " + sourceName) - .forUser(userPermissions) - .withWorkProduct(RESOURCE, source._id.toString(), regionId) - .withAction(progressListener -> { - - // Loop through uploaded files, registering the extensions and writing to storage (with filenames that - // correspond to the source id) - List files = new ArrayList<>(); - StringJoiner fileNames = new StringJoiner(", "); - final List fileItems = formFields.get("sourceFiles"); - for (FileItem fileItem : fileItems) { - DiskFileItem dfi = (DiskFileItem) fileItem; - String filename = fileItem.getName(); - fileNames.add(filename); - String extension = filename.substring(filename.lastIndexOf(".") + 1).toUpperCase(Locale.ROOT); - FileStorageKey key = new FileStorageKey(RESOURCES, source._id.toString(), extension); - fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); - files.add(fileStorage.getFile(key)); - } - - progressListener.beginTask("Detecting format", 1); - final FileStorageFormat uploadFormat; - try { - // Validate inputs, which will throw an exception if there's anything wrong with them. - uploadFormat = detectUploadFormatAndValidate(fileItems); - LOG.info("Handling uploaded {} file", uploadFormat); - } catch (Exception e) { - throw AnalysisServerException.fileUpload("Problem reading uploaded spatial files" + e.getMessage()); - } - progressListener.beginTask("Validating files", 1); - source.description = "From uploaded files: " + fileNames; - source.validateAndSetDetails(uploadFormat, files); - spatialResourceCollection.insert(source); - })); - return source; - } - - private Collection deleteResource (Request request, Response response) { - SpatialResource source = spatialResourceCollection.findPermittedByRequestParamId(request, response); - // TODO delete files from storage - // TODO delete referencing database records - spatialResourceCollection.delete(source); - return spatialResourceCollection.findPermitted( - eq("regionId", request.params("regionId")), UserPermissions.from(request) - ); - } - - @Override - public void registerEndpoints (spark.Service sparkService) { - sparkService.path("/api/spatial", () -> { - sparkService.post("", this::handleUpload, toJson); - sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); - sparkService.get("/region/:regionId", this::getRegionResources, toJson); - sparkService.delete("/source/:_id", this::deleteResource, toJson); - sparkService.get("/:_id", this::getResource, toJson); - }); - } -} diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java new file mode 100644 index 000000000..9ce50a143 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -0,0 +1,153 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.controllers.DataSourceController; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.analyst.progress.TaskAction; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.apache.commons.io.FilenameUtils; +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; +import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; +import static com.conveyal.file.FileCategory.DATASOURCES; + +/** + * Given a batch of uploaded files, put them into FileStorage, categorize and validate them, and record metadata. + * This implements TaskAction so it can be run in the background without blocking the HTTP request and handler thread. + */ +public abstract class DataSourceIngester implements TaskAction { + + private static final Logger LOG = LoggerFactory.getLogger(DataSourceIngester.class); + + // Components used by this bacground task - formerly captured by an anonymous closure. + // By explicitly representing these fields, dependencies and data flow are clearer. + private final FileStorage fileStorage; + private final AnalysisCollection dataSourceCollection; + + protected final List fileItems; + + /** + * The concrete DataSource instance being filled in by this ingester instance. + * It should be created immediately by the subclass constructors. + */ + protected D dataSource; + + /** One File instance per source file, after being moved into storage - they should all have the same name. */ + // TODO Is this necessary? Can't we just process the uploaded files before moving them into storage? + // clarify this in the Javadoc of DataSourceIngester. + // I guess we can't be sure they are all in the same directory which could confuse the SHP reader. + protected List files; + + public String getDataSourceId () { + return dataSource._id.toString(); + }; + + public String getRegionId () { + return dataSource.regionId; + }; + + public String getDataSourceName () { + return dataSource.name; + } + + public DataSourceIngester ( + FileStorage fileStorage, + AnalysisCollection dataSourceCollection, + List fileItems + ) { + this.fileStorage = fileStorage; + this.dataSourceCollection = dataSourceCollection; + this.fileItems = fileItems; + } + + @Override + public final void action (ProgressListener progressListener) throws Exception { + // Call shared logic to move all files into cloud storage from temp upload location. + moveFilesIntoStorage(progressListener); + // Call ingestion logic specific to the detected file format. + ingest(progressListener); + dataSourceCollection.insert(dataSource); + } + + /** + * Implement on concrete subclasses to provide logic for interpreting a single file type. + * This is potentially the slowest part so is called asynchronously (in a background task). + */ + public abstract void ingest (ProgressListener progressListener); + + /** + * Called asynchronously (in a background task) because when using cloud storage, this transfer could be slow. + * FIXME should we do this after processing, and also move any other created files into storage? + * Or shouod we make it clear that ingestion never produces additional files (what about mapDBs?) + */ + private final void moveFilesIntoStorage (ProgressListener progressListener) { + // Loop through uploaded files, registering the extensions and writing to storage (with filenames that + // correspond to the source id) + progressListener.beginTask("Moving files into storage...", 1); + files = new ArrayList<>(fileItems.size()); + for (FileItem fileItem : fileItems) { + DiskFileItem dfi = (DiskFileItem) fileItem; + // TODO use canonical extensions from filetype enum + // TODO upper case? should we be using lower case? + String extension = FilenameUtils.getExtension(fileItem.getName()).toUpperCase(Locale.ROOT); + FileStorageKey key = new FileStorageKey(DATASOURCES, getDataSourceId(), extension); + fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); + files.add(fileStorage.getFile(key)); + } + } + + /*** + * Given the HTTP post form fields from our data source creation endpoint, return a concrete DataSourceIngester + * instance set up to process the uploaded data in the background. This also allows us to fail fast on data files + * that we can't recognize or have obvious problems. Care should be taken that this method contains no slow actions. + */ + public static DataSourceIngester forFormFields ( + FileStorage fileStorage, + AnalysisCollection dataSourceCollection, + Map> formFields, + UserPermissions userPermissions + ) { + // Extract values of required fields. Throws AnalysisServerException on failure, e.g. if a field is missing. + final String sourceName = getFormField(formFields, "sourceName", true); + final String regionId = getFormField(formFields, "regionId", true); + final List fileItems = formFields.get("sourceFiles"); + + FileStorageFormat format = detectUploadFormatAndValidate(fileItems); + DataSourceIngester dataSourceIngester; + if (format == FileStorageFormat.SHP) { + dataSourceIngester = new ShapefileDataSourceIngester( + fileStorage, dataSourceCollection, fileItems, userPermissions + ); + } else { + throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); + } + // Arrgh no-arg constructors are used for deserialization so they don't create an _id or nonce ObjectId(); + // There has to be a better way to get all this garbage into the subclasses. + // I think we need to go with composition instead of subclassing - specific format ingester does not need + // access to fileStorage or database collection. + dataSourceIngester.dataSource.name = sourceName; + dataSourceIngester.dataSource.regionId = regionId; + String fileNames = fileItems.stream().map(FileItem::getName).collect(Collectors.joining(", ")); + dataSourceIngester.dataSource.description = "From uploaded files: " + fileNames; + return dataSourceIngester; + } + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/Lines.java b/src/main/java/com/conveyal/analysis/datasource/Lines.java similarity index 62% rename from src/main/java/com/conveyal/analysis/spatial/Lines.java rename to src/main/java/com/conveyal/analysis/datasource/Lines.java index beaba4248..144c5eeed 100644 --- a/src/main/java/com/conveyal/analysis/spatial/Lines.java +++ b/src/main/java/com/conveyal/analysis/datasource/Lines.java @@ -1,4 +1,4 @@ -package com.conveyal.analysis.spatial; +package com.conveyal.analysis.datasource; public class Lines { diff --git a/src/main/java/com/conveyal/analysis/spatial/Points.java b/src/main/java/com/conveyal/analysis/datasource/Points.java similarity index 85% rename from src/main/java/com/conveyal/analysis/spatial/Points.java rename to src/main/java/com/conveyal/analysis/datasource/Points.java index 0abbb9d94..abb699af4 100644 --- a/src/main/java/com/conveyal/analysis/spatial/Points.java +++ b/src/main/java/com/conveyal/analysis/datasource/Points.java @@ -1,4 +1,4 @@ -package com.conveyal.analysis.spatial; +package com.conveyal.analysis.datasource; import com.conveyal.analysis.models.FileInfo; diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java new file mode 100644 index 000000000..e2f1b8e8f --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -0,0 +1,71 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.models.Bounds; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.util.ShapefileReader; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.apache.commons.io.FilenameUtils; +import org.locationtech.jts.geom.Envelope; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.StringJoiner; + +import static com.conveyal.file.FileCategory.DATASOURCES; +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; + +public class ShapefileDataSourceIngester extends DataSourceIngester { + + public ShapefileDataSourceIngester ( + FileStorage fileStorage, + AnalysisCollection dataSourceCollection, + List fileItems, + UserPermissions userPermissions + ) { + super(fileStorage, dataSourceCollection, fileItems); + dataSource = new SpatialDataSource(userPermissions, "NONE"); + dataSource.fileFormat = FileStorageFormat.SHP; + } + + @Override + public void ingest (ProgressListener progressListener) { + progressListener.beginTask("Validating files", 1); + // In the caller, we should have already verified that all files have the same base name and have an extension. + // Extract the relevant files: .shp, .prj, .dbf, and .shx. + Map filesByExtension = new HashMap<>(); + for (File file : files) { + filesByExtension.put(FilenameUtils.getExtension(file.getName()).toUpperCase(), file); + } + try { + ShapefileReader reader = new ShapefileReader(filesByExtension.get("SHP")); + Envelope envelope = reader.wgs84Bounds(); + checkWgsEnvelopeSize(envelope); + dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); + dataSource.attributes = reader.attributes(); + dataSource.geometryType = reader.geometryType(); + dataSource.featureCount = reader.featureCount(); + } catch (FactoryException | TransformException e) { + throw new RuntimeException("Shapefile transform error. Try uploading an unprojected (EPSG:4326) file.", e); + } catch (Exception e) { + // Must catch because ShapefileReader throws a checked IOException. + throw new RuntimeException("Error parsing shapefile. Ensure the files you uploaded are valid.", e); + } + } + +} diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java similarity index 97% rename from src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java rename to src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java index 6076c31c0..4e7cef853 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java @@ -1,4 +1,4 @@ -package com.conveyal.analysis.spatial; +package com.conveyal.analysis.datasource; import org.locationtech.jts.geom.Geometry; import org.opengis.feature.type.AttributeType; diff --git a/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java similarity index 94% rename from src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java rename to src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java index 139b4b176..e9d9d8966 100644 --- a/src/main/java/com/conveyal/analysis/spatial/SpatialLayers.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java @@ -1,4 +1,4 @@ -package com.conveyal.analysis.spatial; +package com.conveyal.analysis.datasource; import com.conveyal.analysis.AnalysisServerException; @@ -17,10 +17,13 @@ public abstract class SpatialLayers { /** + * FIXME Originally used for OpportunityDataset upload, moved to SpatialLayers but should be named DataSource * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary * grids. In the process we validate the list of uploaded files, making sure certain preconditions are met. * Some kinds of uploads must contain multiple files (.shp) or can contain multiple files (.grid) while others * must have only a single file (.csv). Scan the list of uploaded files to ensure it makes sense before acting. + * Note that this does not validate the contents of the files semantically, just the high-level characteristics of + * the set of files. * @throws AnalysisServerException if the type of the upload can't be detected or preconditions are violated. * @return the expected type of the uploaded file or files, never null. */ diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index 544cfa892..8c7d9c43c 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -26,7 +26,7 @@ public static AggregationArea create (UserPermissions user, String name) { return new AggregationArea(user, name); } - public AggregationArea withSource (SpatialResource source) { + public AggregationArea withSource (SpatialDataSource source) { this.regionId = source.regionId; this.sourceId = source._id.toString(); return this; diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index e3acb73e1..10a1bf951 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -29,10 +29,7 @@ public class BaseModel { this.name = name; } - /** - * No-arg constructor required for Mongo POJO serialization - */ - public BaseModel () { - - } + /** Zero argument constructor required for MongoDB driver automatic POJO deserialization. */ + public BaseModel () { } + } diff --git a/src/main/java/com/conveyal/analysis/models/Bounds.java b/src/main/java/com/conveyal/analysis/models/Bounds.java index fce7673fd..87240befe 100644 --- a/src/main/java/com/conveyal/analysis/models/Bounds.java +++ b/src/main/java/com/conveyal/analysis/models/Bounds.java @@ -26,4 +26,12 @@ public Envelope envelope () { return new Envelope(this.west, this.east, this.south, this.north); } + public static Bounds fromWgsEnvelope (Envelope envelope) { + Bounds bounds = new Bounds(); + bounds.west = envelope.getMinX(); + bounds.east = envelope.getMaxX(); + bounds.south = envelope.getMinY(); + bounds.north = envelope.getMaxY(); + return bounds; + } } diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java new file mode 100644 index 000000000..ca21044c0 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -0,0 +1,46 @@ +package com.conveyal.analysis.models; + +import com.conveyal.analysis.UserPermissions; +import com.conveyal.file.FileStorageFormat; +import org.bson.codecs.pojo.annotations.BsonDiscriminator; + +// Do we get any advantages from a DataSource class hierarchy as opposed to a class with some fields left null? +// Handling different classes may not be worth additional effort unless we take advantage of polymorphic methods. +// We'll have Mongo collections returning the supertype, leaving any specialized fields inaccessible. +// Usefulness will depend on how different the different subtypes are, and we don't know that yet. +// The main action we take on DataSources is to process them into derived data. That can't easily be polymorphic. +// Perhaps we should just have a DataSourceType enum with corresponding field, but we're then ignoring Java types. + +/** + * This represents a file which was uploaded by the user and validated by the backend. Instances are persisted to Mongo. + * DataSources can be processed into derived products like aggregation areas, destination grids, and transport networks. + * Subtypes exist to allow additional fields on certain kinds of data sources. The attribute "type" of instances + * serialized into Mongo is a "discriminator" which determines the corresponding Java class on deserialization. + */ +@BsonDiscriminator(key="type") +public abstract class DataSource extends BaseModel { + + public String regionId; + + /** Description editable by end users */ + public String description; + + /** + * Internally we store all files with the same ID as their database entry, but we retain the file name to help the + * user recognize files they uploaded. We could also just put that info in the description. + */ + public String originalFileName; + + public FileStorageFormat fileFormat; + + // This type uses (north, south, east, west), ideally we'd use (minLon, minLat, maxLon, maxLat). + public Bounds wgsBounds; + + public DataSource (UserPermissions user, String name) { + super(user, name); + } + + /** Zero-argument constructor required for Mongo automatic POJO deserialization. */ + public DataSource () { } + +} diff --git a/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java b/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java new file mode 100644 index 000000000..208e0d050 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java @@ -0,0 +1,13 @@ +package com.conveyal.analysis.models; + +public abstract class DataSourceValidationIssue { + + public abstract String description(); + + public abstract Level level(); + + public enum Level { + ERROR, WARN, INFO + } + +} diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java new file mode 100644 index 000000000..d9c014423 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java @@ -0,0 +1,65 @@ +package com.conveyal.analysis.models; + +import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.datasource.SpatialAttribute; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; +import com.conveyal.r5.util.ShapefileReader; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.io.FilenameUtils; +import org.bson.codecs.pojo.annotations.BsonDiscriminator; +import org.locationtech.jts.geom.Envelope; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.conveyal.file.FileCategory.DATASOURCES; +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; + +/** + * A SpatialDataSource is metadata about a user-uploaded file containing geospatial features (e.g. shapefile, GeoJSON, + * or CSV containing point coordinates) that has been validated and is ready to be processed into specific Conveyal + * formats (e.g. grids and other spatial layers). + * The defining characteristic of a SpatialDataSource is that it contains a set of "features" which all share a schema: + * they all have the same set of attributes (named and typed fields), and one of these attributes is a geometry of a + * dataset-wide type (polygon, linestring etc.) in a coordinate system referencing geographic space. + */ +@BsonDiscriminator(key="type", value="spatial") +public class SpatialDataSource extends DataSource { + + /** The number of features in this SpatialDataSource. */ + public int featureCount; + + /** All features in this SpatialDataSource have an attached geometry of this type. */ + public ShapefileReader.GeometryType geometryType; + + /** Every feature has this set of Attributes - this is essentially a schema. */ + public List attributes; + + public SpatialDataSource (UserPermissions userPermissions, String name) { + super(userPermissions, name); + } + + /** Zero-argument constructor required for Mongo automatic POJO deserialization. */ + public SpatialDataSource () { } + + /** + * Fluent methods to avoid constructors with lots of positional paramters. + * Not really a builder pattern since it doesn't construct immutable objects, + * but due to automatic POJO Mongo storage we can't have immutable objects anyway. + * Unfortunately these can't be placed on the superclass because they won't return the concrete subclass. + * Hence more absurd Java verbosity for things that should be built in language features like named parameters. + */ + // Given these restrictions I think I'd rather go with classic factory methods here instead of builders. + + public FileStorageKey storageKey() { + return new FileStorageKey(DATASOURCES, this._id.toString(), fileFormat.toString()); + } + +} diff --git a/src/main/java/com/conveyal/analysis/models/SpatialResource.java b/src/main/java/com/conveyal/analysis/models/SpatialResource.java deleted file mode 100644 index adaca7dd2..000000000 --- a/src/main/java/com/conveyal/analysis/models/SpatialResource.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.conveyal.analysis.models; - -import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.UserPermissions; -import com.conveyal.analysis.spatial.FeatureSummary; -import com.conveyal.analysis.spatial.SpatialAttribute; -import com.conveyal.file.FileStorageFormat; -import com.conveyal.file.FileStorageKey; -import com.conveyal.r5.util.ShapefileReader; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.io.FilenameUtils; -import org.locationtech.jts.geom.Envelope; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.operation.TransformException; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.conveyal.file.FileCategory.RESOURCES; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; - -/** - * A Resource is a database record associating metadata with an Upload (a raw file in FileStorage uploaded by the user). - * A SpatialResource is such a record for a spatial source file (e.g. shapefile, GeoJSON, CSV) that has been validated - * and is ready to be processed into specific Conveyal formats (e.g. grids and other spatial layers). Eventually a - * general Resource class could be extended by SpatialResource, GtfsResource, and OsmResource? - */ -public class SpatialResource extends BaseModel { - public String regionId; - /** Description editable by end users */ - public String description; - public FileStorageFormat sourceFormat; - /** General geometry type */ - public FeatureSummary features; - /** Attributes, set only after validation (e.g. appropriate format for each feature's attributes) */ - public List attributes; - - private SpatialResource (UserPermissions userPermissions, String sourceName) { - super(userPermissions, sourceName); - } - - /** No-arg constructor required for Mongo POJO deserialization. */ - public SpatialResource () { } - - public static SpatialResource create (UserPermissions userPermissions, String sourceName) { - return new SpatialResource(userPermissions, sourceName); - } - - public SpatialResource withRegion (String regionId) { - this.regionId = regionId; - return this; - } - - public void validateAndSetDetails (FileStorageFormat uploadFormat, List files) { - this.sourceFormat = uploadFormat; - if (uploadFormat == FileStorageFormat.GRID) { - // TODO source.fromGrids(fileItems); - } else if (uploadFormat == FileStorageFormat.SHP) { - this.fromShapefile(files); - } else if (uploadFormat == FileStorageFormat.CSV) { - // TODO source.fromCsv(fileItems); - } else if (uploadFormat == FileStorageFormat.GEOJSON) { - // TODO source.fromGeojson(fileItems); - } - } - - private void fromShapefile (List files) { - // In the caller, we should have already verified that all files have the same base name and have an extension. - // Extract the relevant files: .shp, .prj, .dbf, and .shx. - // We need the SHX even though we're looping over every feature as they might be sparse. - Map filesByExtension = new HashMap<>(); - for (File file : files) { - filesByExtension.put(FilenameUtils.getExtension(file.getName()).toUpperCase(), file); - } - - try { - ShapefileReader reader = new ShapefileReader(filesByExtension.get("SHP")); - Envelope envelope = reader.wgs84Bounds(); - checkWgsEnvelopeSize(envelope); - this.attributes = reader.getAttributes(); - this.features = reader.featureSummary(); - } catch (IOException e) { - throw AnalysisServerException.fileUpload("Shapefile parsing error. Ensure the files you are trying to " + - "upload are valid."); - } catch (FactoryException | TransformException e) { - throw AnalysisServerException.fileUpload("Shapefile transform error. Try uploading an unprojected " + - "(EPSG:4326) file." + e.getMessage()); - } - } - - public SpatialResource fromFiles (List fileItemList) { - // TODO this.files from fileItemList; - return this; - } - - public FileStorageKey storageKey() { - return new FileStorageKey(RESOURCES, this._id.toString(), sourceFormat.toString()); - } - -} diff --git a/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java b/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java deleted file mode 100644 index 721a0411f..000000000 --- a/src/main/java/com/conveyal/analysis/spatial/FeatureSummary.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.conveyal.analysis.spatial; - -import org.geotools.feature.FeatureCollection; -import org.locationtech.jts.geom.Lineal; -import org.locationtech.jts.geom.Polygonal; -import org.locationtech.jts.geom.Puntal; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; - -/** - * Records the geometry type (polygon, point, or line) and number of features in a particular SpatialResource. - * Types are very general, so line type includes things like multilinestrings which must be unwrapped. - * Could we just flatten this into SpatialResource? - */ -public class FeatureSummary { - public int count; - public Type type; - - public enum Type { - POLYGON, - POINT, - LINE; - } - - public FeatureSummary (FeatureCollection features) { - Class geometryType = features.getSchema().getGeometryDescriptor().getType().getBinding(); - if (Polygonal.class.isAssignableFrom(geometryType)) this.type = Type.POLYGON; - if (Puntal.class.isAssignableFrom(geometryType)) this.type = Type.POINT; - if (Lineal.class.isAssignableFrom(geometryType)) this.type = Type.LINE; - // TODO throw exception if geometryType is not one of the above - this.count = features.size(); - } - - /** - * No-arg constructor for Mongo serialization - */ - public FeatureSummary () { - } - -} diff --git a/src/main/java/com/conveyal/analysis/util/HttpUtils.java b/src/main/java/com/conveyal/analysis/util/HttpUtils.java index 6226c7d8d..5073a00ab 100644 --- a/src/main/java/com/conveyal/analysis/util/HttpUtils.java +++ b/src/main/java/com/conveyal/analysis/util/HttpUtils.java @@ -18,7 +18,10 @@ public static Map> getRequestFiles (HttpServletRequest re // all looks threadsafe. But also very lightweight to instantiate, so in this code run by multiple threads // we play it safe and always create a new factory. // Setting a size threshold of 0 causes all files to be written to disk, which allows processing them in a - // uniform way in other threads, after the request handler has returned. + // uniform way in other threads, after the request handler has returned. This does however cause some very + // small form fields to be written to disk files. Ideally we'd identify the smallest actual file we'll ever + // handle and set the threshold a little higher. The downside is that if a tiny file is actually uploaded even + // by accident, our code will not be able to get a file handle for it and fail. FileItemFactory fileItemFactory = new DiskFileItemFactory(0, null); ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); try { diff --git a/src/main/java/com/conveyal/file/FileCategory.java b/src/main/java/com/conveyal/file/FileCategory.java index 7533a5110..90fc41068 100644 --- a/src/main/java/com/conveyal/file/FileCategory.java +++ b/src/main/java/com/conveyal/file/FileCategory.java @@ -8,7 +8,7 @@ */ public enum FileCategory { - BUNDLES, GRIDS, RESULTS, RESOURCES, POLYGONS, TAUI; + BUNDLES, GRIDS, RESULTS, DATASOURCES, POLYGONS, TAUI; /** @return a String for the directory or sub-bucket name containing all files in this category. */ public String directoryName () { diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index 2a52dfe90..0bd132107 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -10,15 +10,17 @@ public enum FileStorageFormat { TIFF("tiff", "image/tiff"), CSV("csv", "text/csv"), - // These are not currently used but plan to be in the future. Exact types need to be determined - // GTFS("zip", "application/zip"), - // PBF("pbf", "application/octet-stream"), - // SHP implies .dbf and .prj, and optionally .shx SHP("shp", "application/octet-stream"), + // These final ones are not yet used. + // In our internal storage, we may want to force less ambiguous .gtfs.zip .osm.pbf and .geo.json. + GTFS("zip", "application/zip"), + OSMPBF("pbf", "application/octet-stream"), GEOJSON("json", "application/json"); + // These should not be serialized into Mongo. Default Enum codec uses String name() and valueOf(String). + // TODO clarify whether the extension is used for backend storage, or for detecting type up uploaded files. public final String extension; public final String mimeType; diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 6f186f185..9525c8a5e 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -679,7 +679,7 @@ public static List fromShapefile (File shapefile, int zoom, ProgressListen } checkPixelCount(extents, numericAttributes.size()); - int total = reader.getFeatureCount(); + int total = reader.featureCount(); if (progressListener != null) { progressListener.setTotalItems(total); } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index a321a24cb..262952c13 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -32,7 +32,7 @@ public static enum State { QUEUED, ACTIVE, DONE, ERROR } - /** Every has an ID so the UI can update tasks it already knows about with new information after polling. */ + /** Every Task has an ID so the UI can update tasks it already knows about with new information after polling. */ public final UUID id = UUID.randomUUID(); // User and group are only relevant on the backend. On workers, we want to show network or cost table build progress diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java index 05ce4fbc4..360abce08 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProductType.java @@ -2,25 +2,27 @@ import com.conveyal.analysis.models.AggregationArea; import com.conveyal.analysis.models.Bundle; +import com.conveyal.analysis.models.DataSource; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.RegionalAnalysis; -import com.conveyal.analysis.models.SpatialResource; +import com.conveyal.analysis.models.SpatialDataSource; /** * There is some implicit and unenforced correspondence between these values and those in FileCategory, as well - * as the tables in Mongo. We should probably clearly state and enforce this parallelism. No background work is + * as the tables in Mongo. We should probably clearly state and enforce this correspondance. No background work is * done creating regions, projects, or modifications so they don't need to be represented here. */ public enum WorkProductType { - BUNDLE, REGIONAL_ANALYSIS, AGGREGATION_AREA, OPPORTUNITY_DATASET, RESOURCE; + BUNDLE, REGIONAL_ANALYSIS, AGGREGATION_AREA, OPPORTUNITY_DATASET, DATA_SOURCE; public static WorkProductType forModel (Object model) { if (model instanceof Bundle) return BUNDLE; if (model instanceof OpportunityDataset) return OPPORTUNITY_DATASET; if (model instanceof RegionalAnalysis) return REGIONAL_ANALYSIS; if (model instanceof AggregationArea) return AGGREGATION_AREA; - if (model instanceof SpatialResource) return RESOURCE; // TODO switch to spatial dataset source + if (model instanceof DataSource) return DATA_SOURCE; throw new IllegalArgumentException("Unrecognized work product type."); } + } diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 817aadb2b..a4ebc1e72 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -1,8 +1,7 @@ package com.conveyal.r5.util; import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.spatial.FeatureSummary; -import com.conveyal.analysis.spatial.SpatialAttribute; +import com.conveyal.analysis.datasource.SpatialAttribute; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; import org.geotools.data.FeatureSource; @@ -14,6 +13,9 @@ import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Lineal; +import org.locationtech.jts.geom.Polygonal; +import org.locationtech.jts.geom.Puntal; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.AttributeDescriptor; @@ -38,6 +40,10 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.LINE; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.POINT; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; + /** * Encapsulate Shapefile reading logic */ @@ -139,11 +145,11 @@ public void close () { store.dispose(); } - public int getFeatureCount() throws IOException { + public int featureCount () throws IOException { return source.getCount(Query.ALL); } - public List getAttributes () { + public List attributes () { List attributes = new ArrayList<>(); HashSet uniqueAttributes = new HashSet<>(); features.getSchema() @@ -162,7 +168,16 @@ public List getAttributes () { return attributes; } - public FeatureSummary featureSummary () { - return new FeatureSummary(features); + /** These are very broad. For example, line includes linestring and multilinestring. */ + public enum GeometryType { + POLYGON, POINT, LINE; + } + + public GeometryType geometryType () { + Class geometryClass = features.getSchema().getGeometryDescriptor().getType().getBinding(); + if (Polygonal.class.isAssignableFrom(geometryClass)) return POLYGON; + if (Puntal.class.isAssignableFrom(geometryClass)) return POINT; + if (Lineal.class.isAssignableFrom(geometryClass)) return LINE; + throw new IllegalArgumentException("Could not determine geometry type of features in DataSource."); } } From e1177760f39aedae9fac82a5ede195fd8d3a1cf2 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 13:54:31 +0800 Subject: [PATCH 070/187] Disable FileStorageController Confirmed in recent call that we don't use this HTTP API, even though we heavily use the underlying FileStorage interface. Keeping the HttpController around though as documentation, since it was written around the same time as the underlying component. --- .../com/conveyal/analysis/components/BackendComponents.java | 2 -- .../conveyal/analysis/controllers/FileStorageController.java | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 330386ce2..c0e2d02bc 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -6,7 +6,6 @@ import com.conveyal.analysis.controllers.AggregationAreaController; import com.conveyal.analysis.controllers.BrokerController; import com.conveyal.analysis.controllers.BundleController; -import com.conveyal.analysis.controllers.FileStorageController; import com.conveyal.analysis.controllers.GTFSGraphQLController; import com.conveyal.analysis.controllers.GtfsTileController; import com.conveyal.analysis.controllers.HttpController; @@ -95,7 +94,6 @@ public List standardHttpControllers () { new RegionalAnalysisController(broker, fileStorage), new AggregationAreaController(fileStorage, database, taskScheduler), new TimetableController(), - new FileStorageController(fileStorage, database), // This broker controller registers at least one handler at URL paths beginning with /internal, which // is exempted from authentication and authorization, but should be hidden from the world // outside the cluster by the reverse proxy. Perhaps we should serve /internal on a separate diff --git a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java index b69e37795..905d12e5c 100644 --- a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java +++ b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java @@ -22,6 +22,8 @@ /** * HTTP request handler methods allowing users to upload/download files from FileStorage implementations and CRUDing * metadata about those files in the database. + * NOTE: THIS CLASS IS UNUSED AND IS RETAINED FOR DOCUMENTATION PURPOSES - TO DEMONSTRATE HOW FILESTORAGE IS USED. + * In practice we don't need direct HTTP API access to FileStorage - it's always used in some more complex process. */ public class FileStorageController implements HttpController { From ad5b37493cd84590a2ec2f9d5f222e0d1aa0b0b6 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 18:08:05 +0800 Subject: [PATCH 071/187] register subclass discriminators in AnalysisDB --- .../analysis/persistence/AnalysisDB.java | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 3dac06403..409838510 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -1,5 +1,7 @@ package com.conveyal.analysis.persistence; +import com.conveyal.analysis.models.BaseModel; +import com.conveyal.analysis.models.SpatialDataSource; import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; @@ -30,19 +32,7 @@ public AnalysisDB (Config config) { LOG.info("Connecting to local MongoDB instance..."); mongo = MongoClients.create(); } - - // Create a codec registry that has all the default codecs (dates, geojson, etc.) and falls back to a provider - // that automatically generates codecs for any other Java class it encounters, based on their public getter and - // setter methods and public fields, skipping any properties whose underlying fields are transient or static. - // These classes must have an empty public or protected zero-argument constructor. - // In the documentation a "discriminator" refers to a field that identifies the Java type, like @JsonTypes. - CodecProvider automaticPojoCodecProvider = PojoCodecProvider.builder().automatic(true).build(); - CodecRegistry pojoCodecRegistry = fromRegistries( - getDefaultCodecRegistry(), - fromProviders(automaticPojoCodecProvider) - ); - - database = mongo.getDatabase(config.databaseName()).withCodecRegistry(pojoCodecRegistry); + database = mongo.getDatabase(config.databaseName()).withCodecRegistry(makeCodecRegistry()); // Request that the JVM clean up database connections in all cases - exiting cleanly or by being terminated. // We should probably register such hooks for other components to shut down more cleanly. @@ -52,12 +42,42 @@ public AnalysisDB (Config config) { })); } - public AnalysisCollection getAnalysisCollection (String name, Class clazz) { - return new AnalysisCollection<>(database.getCollection(name, clazz), clazz); + /** + * Create a codec registry that has all the default codecs (dates, geojson, etc.) and falls back to a provider + * that automatically generates codecs for any other Java class it encounters, based on their public getter and + * setter methods and public fields, skipping any properties whose underlying fields are transient or static. + * These classes must have an empty public or protected zero-argument constructor. + * An automatic PojoCodecProvider can create class models and codecs on the fly as it encounters the classes + * during writing. However, upon restart it will need to re-register those same classes before it can decode + * them. This is apparently done automatically when calling database.getCollection(), but gets a little tricky + * when decoding subclasses whose discriminators are not fully qualified class names with package. See Javadoc + * on getAnalysisCollection() for how we register such subclasses. + * We could register all these subclasses here via the PojoCodecProvider.Builder, but that separates their + * registration from the place they're used. The builder has methods for registering whole packages, but these + * methods do not auto-scan, they just provide the same behavior as automatic() but limited to specific packages. + */ + private CodecRegistry makeCodecRegistry () { + CodecProvider automaticPojoCodecProvider = PojoCodecProvider.builder().automatic(true).build(); + CodecRegistry pojoCodecRegistry = fromRegistries( + getDefaultCodecRegistry(), + fromProviders(automaticPojoCodecProvider) + ); + return pojoCodecRegistry; } - public MongoCollection getMongoCollection (String name, Class clazz) { - return database.getCollection(name, clazz); + /** + * If the optional subclasses are supplied, the codec registry will be hit to cause it to build class models and + * codecs for them. This is necessary when these subclasses specify short discriminators, as opposed to the + * verbose default discriminator of a fully qualified class name, because the Mongo driver does not auto-scan for + * classes it has not encountered in a write operation or in a request for a collection. + */ + public AnalysisCollection getAnalysisCollection ( + String name, Class clazz, Class... subclasses + ){ + for (Class subclass : subclasses) { + database.getCodecRegistry().get(subclass); + } + return new AnalysisCollection(database.getCollection(name, clazz), clazz); } /** Interface to supply configuration to this component. */ From b894d904303cae80a3369f6fbc77f9c5c311334e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 18:13:01 +0800 Subject: [PATCH 072/187] update DataSourceController - revise API URL paths as decided in recent meeting - register subclass discriminators of DataSource - new utility method on AnalysisCollection to delete - return task ID instead of work product on create - placeholders for other DataSource types - add comments --- .../controllers/DataSourceController.java | 61 +++++++++---------- .../analysis/models/GtfsDataSource.java | 11 ++++ .../analysis/models/OsmDataSource.java | 11 ++++ .../persistence/AnalysisCollection.java | 8 +++ .../conveyal/r5/analyst/progress/Task.java | 1 + 5 files changed, 60 insertions(+), 32 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/models/GtfsDataSource.java create mode 100644 src/main/java/com/conveyal/analysis/models/OsmDataSource.java diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 7c46b1c76..351dc853a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -1,39 +1,30 @@ package com.conveyal.analysis.controllers; -import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.datasource.DataSourceIngester; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.GtfsDataSource; +import com.conveyal.analysis.models.OsmDataSource; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; -import com.conveyal.file.FileStorageFormat; -import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.analyst.progress.WorkProduct; import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.disk.DiskFileItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; -import java.io.File; -import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.StringJoiner; -import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; -import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.conveyal.file.FileCategory.DATASOURCES; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; import static com.conveyal.r5.analyst.progress.WorkProductType.DATA_SOURCE; import static com.mongodb.client.model.Filters.and; @@ -66,19 +57,34 @@ public DataSourceController ( this.taskScheduler = taskScheduler; this.extractor = extractor; // We don't hold on to the AnalysisDB Component, just get one collection from it. - this.dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); + // Register all the subclasses so the Mongo driver will recognize their discriminators. + this.dataSourceCollection = database.getAnalysisCollection( + "dataSources", DataSource.class, SpatialDataSource.class, OsmDataSource.class, GtfsDataSource.class + ); } + /** HTTP GET: Retrieve all DataSource records, filtered by the (required) regionId query parameter. */ private List getAllDataSourcesForRegion (Request req, Response res) { return dataSourceCollection.findPermitted( - eq("regionId", req.params("regionId")), UserPermissions.from(req) + eq("regionId", req.queryParams("regionId")), UserPermissions.from(req) ); } - private Object getOneDataSourceById (Request req, Response res) { + /** HTTP GET: Retrieve a single DataSource record by the ID supplied in the URL path parameter. */ + private DataSource getOneDataSourceById (Request req, Response res) { return dataSourceCollection.findPermittedByRequestParamId(req, res); } + /** HTTP DELETE: Delete a single DataSource record and associated files in FileStorage by supplied ID parameter. */ + private String deleteOneDataSourceById (Request request, Response response) { + dataSourceCollection.deleteByIdParamIfPermitted(request).getDeletedCount(); + return "DELETE"; + // TODO delete files from storage + // TODO delete referencing database records + // Shouldn't this be deleting by ID instead of sending the whole document? + // TODO why do our delete methods return a list of documents? Can we just return the ID or HTTP status code? + } + private SpatialDataSource downloadLODES(Request req, Response res) { final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); @@ -101,9 +107,9 @@ private SpatialDataSource downloadLODES(Request req, Response res) { * A file is posted to this endpoint to create a new DataSource. It is validated and metadata are extracted. * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. * In a standard REST API, a post would return the ID of the newly created DataSource. Here we're starting an async - * background process, so we return the task or its work product? + * background process, so we return the task ID or the ID its work product (the DataSource)? */ - private WorkProduct handleUpload (Request req, Response res) { + private String handleUpload (Request req, Response res) { final UserPermissions userPermissions = UserPermissions.from(req); final Map> formFields = HttpUtils.getRequestFiles(req.raw()); @@ -118,34 +124,25 @@ private WorkProduct handleUpload (Request req, Response res) { // Or a WorkProductDescriptor, with type, region, and ID? // TaskActions could define methods to return a title, workProductDescriptor, etc. // Then we just have taskScheduler.enqueue(Task.forAction(user, ingester)); + // To the extent that TaskActions are named types instead of lambdas, they can create their workproduct + // upon instantiation and return it via a method. + // ProgressListener could also have a setWorkProduct method. .withWorkProduct(DATA_SOURCE, ingester.getDataSourceId(), ingester.getRegionId()) .withAction(ingester); taskScheduler.enqueue(backgroundTask); - return backgroundTask.workProduct; - } - - private Collection deleteOneDataSourceById (Request request, Response response) { - DataSource source = dataSourceCollection.findPermittedByRequestParamId(request, response); - // TODO delete files from storage - // TODO delete referencing database records - // Shouldn't this be deleting by ID instead of sending the whole document? - dataSourceCollection.delete(source); - // TODO why do our delete methods return a list of documents? Can we just return the ID or HTTP status code? - // Isn't this going to fail since the document was just deleted? - return dataSourceCollection.findPermitted( - eq("regionId", request.params("regionId")), UserPermissions.from(request) - ); + return backgroundTask.id.toString(); } @Override public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/datasource", () -> { + sparkService.get("/", this::getAllDataSourcesForRegion, toJson); sparkService.get("/:_id", this::getOneDataSourceById, toJson); - sparkService.get("/region/:regionId", this::getAllDataSourcesForRegion, toJson); sparkService.delete("/:_id", this::deleteOneDataSourceById, toJson); sparkService.post("", this::handleUpload, toJson); - sparkService.post("/region/:regionId/download", this::downloadLODES, toJson); + // regionId will be in query parameter + sparkService.post("/addLodesDataSource", this::downloadLODES, toJson); }); } } diff --git a/src/main/java/com/conveyal/analysis/models/GtfsDataSource.java b/src/main/java/com/conveyal/analysis/models/GtfsDataSource.java new file mode 100644 index 000000000..7b40e0053 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/GtfsDataSource.java @@ -0,0 +1,11 @@ +package com.conveyal.analysis.models; + +import org.bson.codecs.pojo.annotations.BsonDiscriminator; + +/** + * Placeholder for representing uploaded GTFS data. + */ +@BsonDiscriminator(key="type", value="gtfs") +public class GtfsDataSource extends DataSource { + +} diff --git a/src/main/java/com/conveyal/analysis/models/OsmDataSource.java b/src/main/java/com/conveyal/analysis/models/OsmDataSource.java new file mode 100644 index 000000000..e7abca37f --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/OsmDataSource.java @@ -0,0 +1,11 @@ +package com.conveyal.analysis.models; + +import org.bson.codecs.pojo.annotations.BsonDiscriminator; + +/** + * Placeholder for representing uploaded OSM data. + */ +@BsonDiscriminator(key="type", value="osm") +public class OsmDataSource extends DataSource { + +} diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 671dc89d6..519165714 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -40,6 +40,12 @@ public DeleteResult delete (T value) { return collection.deleteOne(eq("_id", value._id)); } + public DeleteResult deleteByIdParamIfPermitted (Request request) { + String _id = request.params("_id"); + UserPermissions user = UserPermissions.from(request); + return collection.deleteOne(and(eq("_id", _id), eq("accessGroup", user.accessGroup))); + } + public List findPermitted(Bson query, UserPermissions userPermissions) { return find(and(eq(MONGO_PROP_ACCESS_GROUP, userPermissions.accessGroup), query)); } @@ -137,6 +143,7 @@ public T create(Request req, Response res) throws IOException { /** * Controller find by id helper. + * TODO remove unused second parameter. */ public T findPermittedByRequestParamId(Request req, Response res) { UserPermissions user = UserPermissions.from(req); @@ -160,4 +167,5 @@ public T update(Request req, Response res) throws IOException { } return update(value, user.accessGroup); } + } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index 262952c13..a96c41ca4 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -204,6 +204,7 @@ public void increment (int n) { } // Methods for reporting elapsed times over API + // Durations are reported instead of times to avoid problems with clock skew between backend and client. public Duration durationInQueue () { Instant endTime = (began == null) ? Instant.now() : began; From 59e8cb04d07e0744aaf6ac72d67b4d6689132b03 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 21:36:50 +0800 Subject: [PATCH 073/187] Composition not inheritance for DataSourceIngester - supply a single file to DataSourceIngesters - fix delete method (ObjectID type not String) - report work products via ProgressListener --- .../controllers/DataSourceController.java | 25 +-- .../datasource/DataSourceIngester.java | 152 ++--------------- .../datasource/DataSourceUploadAction.java | 159 ++++++++++++++++++ .../ShapefileDataSourceIngester.java | 49 ++---- .../conveyal/analysis/models/DataSource.java | 20 ++- .../persistence/AnalysisCollection.java | 2 +- .../r5/analyst/progress/ProgressListener.java | 9 + .../conveyal/r5/analyst/progress/Task.java | 11 +- 8 files changed, 231 insertions(+), 196 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 351dc853a..8c4a9abaf 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -2,7 +2,7 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; -import com.conveyal.analysis.datasource.DataSourceIngester; +import com.conveyal.analysis.datasource.DataSourceUploadAction; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; import com.conveyal.analysis.models.DataSource; import com.conveyal.analysis.models.GtfsDataSource; @@ -13,14 +13,12 @@ import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; import com.conveyal.r5.analyst.progress.Task; -import com.conveyal.r5.analyst.progress.WorkProduct; import org.apache.commons.fileupload.FileItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -77,8 +75,8 @@ private DataSource getOneDataSourceById (Request req, Response res) { /** HTTP DELETE: Delete a single DataSource record and associated files in FileStorage by supplied ID parameter. */ private String deleteOneDataSourceById (Request request, Response response) { - dataSourceCollection.deleteByIdParamIfPermitted(request).getDeletedCount(); - return "DELETE"; + long nDeleted = dataSourceCollection.deleteByIdParamIfPermitted(request).getDeletedCount(); + return "DELETE " + nDeleted; // TODO delete files from storage // TODO delete referencing database records // Shouldn't this be deleting by ID instead of sending the whole document? @@ -112,23 +110,12 @@ private SpatialDataSource downloadLODES(Request req, Response res) { private String handleUpload (Request req, Response res) { final UserPermissions userPermissions = UserPermissions.from(req); final Map> formFields = HttpUtils.getRequestFiles(req.raw()); - - DataSourceIngester ingester = DataSourceIngester.forFormFields( + DataSourceUploadAction uploadAction = DataSourceUploadAction.forFormFields( fileStorage, dataSourceCollection, formFields, userPermissions ); - - Task backgroundTask = Task.create("Processing uploaded files: " + ingester.getDataSourceName()) + Task backgroundTask = Task.create("Processing uploaded files: " + uploadAction.getDataSourceName()) .forUser(userPermissions) - //.withWorkProduct(dataSource) - // or should TaskActions have a method to return their work product? - // Or a WorkProductDescriptor, with type, region, and ID? - // TaskActions could define methods to return a title, workProductDescriptor, etc. - // Then we just have taskScheduler.enqueue(Task.forAction(user, ingester)); - // To the extent that TaskActions are named types instead of lambdas, they can create their workproduct - // upon instantiation and return it via a method. - // ProgressListener could also have a setWorkProduct method. - .withWorkProduct(DATA_SOURCE, ingester.getDataSourceId(), ingester.getRegionId()) - .withAction(ingester); + .withAction(uploadAction); taskScheduler.enqueue(backgroundTask); return backgroundTask.id.toString(); diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index 9ce50a143..e9ec07e3b 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -1,153 +1,33 @@ package com.conveyal.analysis.datasource; -import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.UserPermissions; -import com.conveyal.analysis.controllers.DataSourceController; import com.conveyal.analysis.models.DataSource; -import com.conveyal.analysis.persistence.AnalysisCollection; -import com.conveyal.file.FileStorage; -import com.conveyal.file.FileStorageFormat; -import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.ProgressListener; -import com.conveyal.r5.analyst.progress.TaskAction; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.disk.DiskFileItem; -import org.apache.commons.io.FilenameUtils; -import org.bson.types.ObjectId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.StringJoiner; -import java.util.stream.Collectors; - -import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; -import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; -import static com.conveyal.file.FileCategory.DATASOURCES; /** - * Given a batch of uploaded files, put them into FileStorage, categorize and validate them, and record metadata. - * This implements TaskAction so it can be run in the background without blocking the HTTP request and handler thread. + * Logic for loading and validating a specific kind of input file, yielding a specific subclass of DataSource. + * This plugs into DataSourceUploadAction, which handles the general parts of processing any new DataSource. */ -public abstract class DataSourceIngester implements TaskAction { - - private static final Logger LOG = LoggerFactory.getLogger(DataSourceIngester.class); - - // Components used by this bacground task - formerly captured by an anonymous closure. - // By explicitly representing these fields, dependencies and data flow are clearer. - private final FileStorage fileStorage; - private final AnalysisCollection dataSourceCollection; - - protected final List fileItems; - - /** - * The concrete DataSource instance being filled in by this ingester instance. - * It should be created immediately by the subclass constructors. - */ - protected D dataSource; - - /** One File instance per source file, after being moved into storage - they should all have the same name. */ - // TODO Is this necessary? Can't we just process the uploaded files before moving them into storage? - // clarify this in the Javadoc of DataSourceIngester. - // I guess we can't be sure they are all in the same directory which could confuse the SHP reader. - protected List files; - - public String getDataSourceId () { - return dataSource._id.toString(); - }; - - public String getRegionId () { - return dataSource.regionId; - }; - - public String getDataSourceName () { - return dataSource.name; - } - - public DataSourceIngester ( - FileStorage fileStorage, - AnalysisCollection dataSourceCollection, - List fileItems - ) { - this.fileStorage = fileStorage; - this.dataSourceCollection = dataSourceCollection; - this.fileItems = fileItems; - } - - @Override - public final void action (ProgressListener progressListener) throws Exception { - // Call shared logic to move all files into cloud storage from temp upload location. - moveFilesIntoStorage(progressListener); - // Call ingestion logic specific to the detected file format. - ingest(progressListener); - dataSourceCollection.insert(dataSource); - } +public abstract class DataSourceIngester { /** - * Implement on concrete subclasses to provide logic for interpreting a single file type. - * This is potentially the slowest part so is called asynchronously (in a background task). + * An accessor method that gives the general purpose DataSourceUploadAction a view of the DataSource being + * constructed. This allows to DataSourceUploadAction to set all the shared general properties of a DataSource, + * leaving the DataSourceIngester to handle only the details specific to its input format and DataSource subclass. + * Concrete subclasses should ensure that this method can return an object immediately after they're constructed. + * Or maybe only after ingest() returns? */ - public abstract void ingest (ProgressListener progressListener); + public abstract D dataSource (); /** - * Called asynchronously (in a background task) because when using cloud storage, this transfer could be slow. - * FIXME should we do this after processing, and also move any other created files into storage? - * Or shouod we make it clear that ingestion never produces additional files (what about mapDBs?) - */ - private final void moveFilesIntoStorage (ProgressListener progressListener) { - // Loop through uploaded files, registering the extensions and writing to storage (with filenames that - // correspond to the source id) - progressListener.beginTask("Moving files into storage...", 1); - files = new ArrayList<>(fileItems.size()); - for (FileItem fileItem : fileItems) { - DiskFileItem dfi = (DiskFileItem) fileItem; - // TODO use canonical extensions from filetype enum - // TODO upper case? should we be using lower case? - String extension = FilenameUtils.getExtension(fileItem.getName()).toUpperCase(Locale.ROOT); - FileStorageKey key = new FileStorageKey(DATASOURCES, getDataSourceId(), extension); - fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); - files.add(fileStorage.getFile(key)); - } - } - - /*** - * Given the HTTP post form fields from our data source creation endpoint, return a concrete DataSourceIngester - * instance set up to process the uploaded data in the background. This also allows us to fail fast on data files - * that we can't recognize or have obvious problems. Care should be taken that this method contains no slow actions. + * This method is implemented on concrete subclasses to provide logic for interpreting a particular file type. + * This is potentially the slowest part of DataSource creation so is called asynchronously (in a background task). + * A single File is passed in here (rather than in the subclass constructors) because the file is moved into + * storage before ingestion. Some supported formats (only shapefile for now) are made up of more than one file, + * which must all be in the same directory. Moving them into storage ensures they're all in the same directory with + * the same base name as required, and only one of their complete file names must be provided. */ - public static DataSourceIngester forFormFields ( - FileStorage fileStorage, - AnalysisCollection dataSourceCollection, - Map> formFields, - UserPermissions userPermissions - ) { - // Extract values of required fields. Throws AnalysisServerException on failure, e.g. if a field is missing. - final String sourceName = getFormField(formFields, "sourceName", true); - final String regionId = getFormField(formFields, "regionId", true); - final List fileItems = formFields.get("sourceFiles"); - - FileStorageFormat format = detectUploadFormatAndValidate(fileItems); - DataSourceIngester dataSourceIngester; - if (format == FileStorageFormat.SHP) { - dataSourceIngester = new ShapefileDataSourceIngester( - fileStorage, dataSourceCollection, fileItems, userPermissions - ); - } else { - throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); - } - // Arrgh no-arg constructors are used for deserialization so they don't create an _id or nonce ObjectId(); - // There has to be a better way to get all this garbage into the subclasses. - // I think we need to go with composition instead of subclassing - specific format ingester does not need - // access to fileStorage or database collection. - dataSourceIngester.dataSource.name = sourceName; - dataSourceIngester.dataSource.regionId = regionId; - String fileNames = fileItems.stream().map(FileItem::getName).collect(Collectors.joining(", ")); - dataSourceIngester.dataSource.description = "From uploaded files: " + fileNames; - return dataSourceIngester; - } + public abstract void ingest (File file, ProgressListener progressListener); } diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java new file mode 100644 index 000000000..6898d2f3f --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -0,0 +1,159 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.analyst.progress.TaskAction; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.apache.commons.io.FilenameUtils; +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; +import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; +import static com.conveyal.file.FileCategory.DATASOURCES; +import static com.conveyal.file.FileStorageFormat.SHP; +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Given a batch of uploaded files, put them into FileStorage, categorize and validate them, and record metadata as + * some specific subclass of DataSource. This implements TaskAction so it can be run in the background without blocking + * the HTTP request and handler thread. + */ +public class DataSourceUploadAction implements TaskAction { + + private static final Logger LOG = LoggerFactory.getLogger(DataSourceUploadAction.class); + + // The Components used by this background task, which were formerly captured by an anonymous closure. + // Using named and well-defined classes for these background actions makes data flow and depdendencies clearer. + private final FileStorage fileStorage; + private final AnalysisCollection dataSourceCollection; + + /** The files provided in the HTTP post form. These will be moved into storage. */ + private final List fileItems; + + /** + * This DataSourceIngester provides encapsulated loading and validation logic for a single format, by composition + * rather than subclassing. Format ingestion does not require access to the fileStorage or the database collection. + */ + private DataSourceIngester ingester; + + /** + * The file to be ingested, after it has been moved into storage. For Shapefiles and other such "sidecar" formats, + * this is the main file (.shp), with the same base name and in the same directory as all its sidecar files. + */ + private File file; + + // This method is a stopgaps - it seems like this should be done differently. + public String getDataSourceName () { + return ingester.dataSource().name; + } + + public DataSourceUploadAction ( + FileStorage fileStorage, + AnalysisCollection dataSourceCollection, + List fileItems, + DataSourceIngester ingester + ) { + this.fileStorage = fileStorage; + this.dataSourceCollection = dataSourceCollection; + this.fileItems = fileItems; + this.ingester = ingester; + } + + /** + * Our no-arg BaseModel constructors are used for deserialization so they don't create an _id or nonce ObjectId(); + * This DataSourceUploadAction takes care of setting the fields common to all kinds of DataSource, with the specific + * DataSourceIngester taking care of the rest. There must be a better way to set all this, but this works for now. + */ + public void initializeDataSource (String name, String regionId, UserPermissions userPermissions) { + DataSource dataSource = ingester.dataSource(); + dataSource._id = new ObjectId(); + dataSource.nonce = new ObjectId(); + dataSource.name = name; + dataSource.regionId = regionId; + dataSource.createdBy = userPermissions.email; + dataSource.updatedBy = userPermissions.email; + dataSource.accessGroup = userPermissions.accessGroup; + String fileNames = fileItems.stream().map(FileItem::getName).collect(Collectors.joining(", ")); + dataSource.description = "From uploaded files: " + fileNames; + } + + @Override + public final void action (ProgressListener progressListener) throws Exception { + progressListener.setWorkProduct(ingester.dataSource().toWorkProduct()); + moveFilesIntoStorage(progressListener); + ingester.ingest(file, progressListener); + dataSourceCollection.insert(ingester.dataSource()); + } + + /** + * Move all files uploaded in the HTTP post form into (cloud) FileStorage from their temp upload location. + * Called asynchronously (in a background task) because when using cloud storage, this transfer could be slow. + * We could do this after processing instead of before, but consider the shapefile case: we can't be completely + * sure the source temp files are all in the same directory. Better to process them after moving into one directory. + * We should also consider whether preprocessing like conversion of GTFS to MapDBs should happen at this upload + * stage. If so, then this logic needs to change a bit. + */ + private final void moveFilesIntoStorage (ProgressListener progressListener) { + // Loop through uploaded files, registering the extensions and writing to storage (with filenames that + // correspond to the source id) + progressListener.beginTask("Moving files into storage...", 1); + final String dataSourceId = ingester.dataSource()._id.toString(); + for (FileItem fileItem : fileItems) { + DiskFileItem dfi = (DiskFileItem) fileItem; + // TODO use canonical extensions from filetype enum + // TODO upper case? should we be using lower case? + String extension = FilenameUtils.getExtension(fileItem.getName()).toUpperCase(Locale.ROOT); + FileStorageKey key = new FileStorageKey(DATASOURCES, dataSourceId, extension); + fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); + if (fileItems.size() == 1 || extension.equalsIgnoreCase(SHP.extension)) { + file = fileStorage.getFile(key); + } + } + checkNotNull(file); + } + + /** + * Given the HTTP post form fields from our data source creation endpoint, return a DataSourceUploadAction + * instance set up to process the uploaded data in the background. This will fail fast on data files that we can't + * recognize or have obvious problems. Care should be taken that this method contains no slow actions. + */ + public static DataSourceUploadAction forFormFields ( + FileStorage fileStorage, + AnalysisCollection dataSourceCollection, + Map> formFields, + UserPermissions userPermissions + ) { + // Extract required parameters. Throws AnalysisServerException on failure, e.g. if a field is missing. + final String sourceName = getFormField(formFields, "sourceName", true); + final String regionId = getFormField(formFields, "regionId", true); + final List fileItems = formFields.get("sourceFiles"); + + FileStorageFormat format = detectUploadFormatAndValidate(fileItems); + DataSourceIngester ingester; + if (format == SHP) { + ingester = new ShapefileDataSourceIngester(); + } else { + throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); + } + DataSourceUploadAction dataSourceUploadAction = + new DataSourceUploadAction(fileStorage, dataSourceCollection, fileItems, ingester); + + dataSourceUploadAction.initializeDataSource(sourceName, regionId, userPermissions); + return dataSourceUploadAction; + } + +} diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index e2f1b8e8f..c915adfe6 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -1,59 +1,42 @@ package com.conveyal.analysis.datasource; -import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.Bounds; -import com.conveyal.analysis.models.DataSource; import com.conveyal.analysis.models.SpatialDataSource; -import com.conveyal.analysis.persistence.AnalysisCollection; -import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; -import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.disk.DiskFileItem; -import org.apache.commons.io.FilenameUtils; import org.locationtech.jts.geom.Envelope; import org.opengis.referencing.FactoryException; import org.opengis.referencing.operation.TransformException; import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.StringJoiner; -import static com.conveyal.file.FileCategory.DATASOURCES; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +/** + * Logic to create SpatialDataSource metadata from a Shapefile. + */ public class ShapefileDataSourceIngester extends DataSourceIngester { - public ShapefileDataSourceIngester ( - FileStorage fileStorage, - AnalysisCollection dataSourceCollection, - List fileItems, - UserPermissions userPermissions - ) { - super(fileStorage, dataSourceCollection, fileItems); - dataSource = new SpatialDataSource(userPermissions, "NONE"); + private final SpatialDataSource dataSource; + + @Override + public SpatialDataSource dataSource () { + return dataSource; + } + + public ShapefileDataSourceIngester () { + // Note we're using the no-arg constructor creating a totally empty object. + // Its fields will be set later by the enclosing DataSourceUploadAction. + this.dataSource = new SpatialDataSource(); dataSource.fileFormat = FileStorageFormat.SHP; } @Override - public void ingest (ProgressListener progressListener) { + public void ingest (File file, ProgressListener progressListener) { progressListener.beginTask("Validating files", 1); - // In the caller, we should have already verified that all files have the same base name and have an extension. - // Extract the relevant files: .shp, .prj, .dbf, and .shx. - Map filesByExtension = new HashMap<>(); - for (File file : files) { - filesByExtension.put(FilenameUtils.getExtension(file.getName()).toUpperCase(), file); - } try { - ShapefileReader reader = new ShapefileReader(filesByExtension.get("SHP")); + ShapefileReader reader = new ShapefileReader(file); Envelope envelope = reader.wgs84Bounds(); checkWgsEnvelopeSize(envelope); dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java index ca21044c0..f0861ae89 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSource.java +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -2,20 +2,24 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.file.FileStorageFormat; +import com.conveyal.r5.analyst.progress.WorkProduct; import org.bson.codecs.pojo.annotations.BsonDiscriminator; -// Do we get any advantages from a DataSource class hierarchy as opposed to a class with some fields left null? -// Handling different classes may not be worth additional effort unless we take advantage of polymorphic methods. -// We'll have Mongo collections returning the supertype, leaving any specialized fields inaccessible. -// Usefulness will depend on how different the different subtypes are, and we don't know that yet. -// The main action we take on DataSources is to process them into derived data. That can't easily be polymorphic. -// Perhaps we should just have a DataSourceType enum with corresponding field, but we're then ignoring Java types. +import static com.conveyal.r5.analyst.progress.WorkProductType.DATA_SOURCE; /** * This represents a file which was uploaded by the user and validated by the backend. Instances are persisted to Mongo. * DataSources can be processed into derived products like aggregation areas, destination grids, and transport networks. * Subtypes exist to allow additional fields on certain kinds of data sources. The attribute "type" of instances * serialized into Mongo is a "discriminator" which determines the corresponding Java class on deserialization. + * + * Given the existence of descriminators in the Mongo driver, for now we're trying full Java typing with inheritance. + * It is debatable whether we get any advantages from this DataSource class hierarchy as opposed to a single class with + * some fields left null in certain cases (e.g. feature schema is null on OSM DataSources). Juggling different classes + * may not be worth the trouble unless we get some utility out of polymorphic methods. For example, Mongo collections + * will return the shared supertype, leaving any specialized fields inaccessible except via overridden methods. + * Usefulness will depend on how different the different subtypes are, and we haven't really seen that yet. + * The main action we take on DataSources is to process them into derived data. That can't easily use polymorphism. */ @BsonDiscriminator(key="type") public abstract class DataSource extends BaseModel { @@ -43,4 +47,8 @@ public DataSource (UserPermissions user, String name) { /** Zero-argument constructor required for Mongo automatic POJO deserialization. */ public DataSource () { } + public WorkProduct toWorkProduct () { + return new WorkProduct(DATA_SOURCE, _id.toString(), regionId); + }; + } diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 519165714..2bd54d544 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -43,7 +43,7 @@ public DeleteResult delete (T value) { public DeleteResult deleteByIdParamIfPermitted (Request request) { String _id = request.params("_id"); UserPermissions user = UserPermissions.from(request); - return collection.deleteOne(and(eq("_id", _id), eq("accessGroup", user.accessGroup))); + return collection.deleteOne(and(eq("_id", new ObjectId(_id)), eq("accessGroup", user.accessGroup))); } public List findPermitted(Bson query, UserPermissions userPermissions) { diff --git a/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java b/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java index 7fbb67b10..47fd4d1f2 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java @@ -25,4 +25,13 @@ default void increment () { increment(1); } + /** + * We want WorkProducts to be revealed by a TaskAction even in the case of exception or failure (to report complex + * structured error or validation information). Returning them from the action method will not work in case of an + * unexpected exception. Adding them to the background Task with a fluent method is also problematic as it requires + * the caller to construct or otherwise hold a reference to the product to get its ID before the action is run. It's + * preferable for the product to be fully encapsulated in the action, so it's reported as park of the task progress. + */ + default void setWorkProduct (WorkProduct workProduct) { /* Default is no-op */ } + } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index a96c41ca4..b8401849e 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -257,12 +257,21 @@ public Task withWorkProduct (BaseModel model) { return this; } - /** Ideally we'd just pass in a Model, but currently we have two base classes, also see WorkProduct.forModel(). */ + /** + * Ideally we'd just pass in a Model, but currently we have two base classes, also see WorkProduct.forModel(). + * This can now be reported via ProgressListener interface for better encapsulation, potentially only revealing the + * work product when it's acutally in the database. + */ public Task withWorkProduct (WorkProductType type, String id, String region) { this.workProduct = new WorkProduct(type, id, region); return this; } + @Override + public void setWorkProduct (WorkProduct workProduct) { + this.workProduct = workProduct; + } + public Task setHeavy (boolean heavy) { this.isHeavy = heavy; return this; From 53777c5f35aa38b9d59245c7c5fff535307a8a61 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 22:05:20 +0800 Subject: [PATCH 074/187] de-parameterize DataSourceIngester --- .../com/conveyal/analysis/datasource/DataSourceIngester.java | 4 ++-- .../analysis/datasource/ShapefileDataSourceIngester.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index e9ec07e3b..345dbc74c 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -9,7 +9,7 @@ * Logic for loading and validating a specific kind of input file, yielding a specific subclass of DataSource. * This plugs into DataSourceUploadAction, which handles the general parts of processing any new DataSource. */ -public abstract class DataSourceIngester { +public abstract class DataSourceIngester { /** * An accessor method that gives the general purpose DataSourceUploadAction a view of the DataSource being @@ -18,7 +18,7 @@ public abstract class DataSourceIngester { * Concrete subclasses should ensure that this method can return an object immediately after they're constructed. * Or maybe only after ingest() returns? */ - public abstract D dataSource (); + public abstract DataSource dataSource (); /** * This method is implemented on concrete subclasses to provide logic for interpreting a particular file type. diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index c915adfe6..41dc34837 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -16,7 +16,7 @@ /** * Logic to create SpatialDataSource metadata from a Shapefile. */ -public class ShapefileDataSourceIngester extends DataSourceIngester { +public class ShapefileDataSourceIngester extends DataSourceIngester { private final SpatialDataSource dataSource; From d16ab1704fb60bc4cb978c6eb944008b85b78532 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 22:06:32 +0800 Subject: [PATCH 075/187] validation issues in DataSource maybe these should instead be in the loader class, and saved in a json file in FileStorage instead of in Mongo (like with GTFS). --- .../com/conveyal/analysis/models/DataSource.java | 9 +++++++++ .../analysis/models/DataSourceValidationIssue.java | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java index f0861ae89..e4a99e90d 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSource.java +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -5,6 +5,8 @@ import com.conveyal.r5.analyst.progress.WorkProduct; import org.bson.codecs.pojo.annotations.BsonDiscriminator; +import java.util.List; + import static com.conveyal.r5.analyst.progress.WorkProductType.DATA_SOURCE; /** @@ -40,6 +42,9 @@ public abstract class DataSource extends BaseModel { // This type uses (north, south, east, west), ideally we'd use (minLon, minLat, maxLon, maxLat). public Bounds wgsBounds; + /** Problems encountered while loading. TODO should this be a separate json file in storage? */ + public List issues; + public DataSource (UserPermissions user, String name) { super(user, name); } @@ -51,4 +56,8 @@ public WorkProduct toWorkProduct () { return new WorkProduct(DATA_SOURCE, _id.toString(), regionId); }; + public void addIssue (DataSourceValidationIssue.Level level, String message) { + issues.add(new DataSourceValidationIssue(level, message)); + } + } diff --git a/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java b/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java index 208e0d050..515abc5a9 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java +++ b/src/main/java/com/conveyal/analysis/models/DataSourceValidationIssue.java @@ -1,13 +1,21 @@ package com.conveyal.analysis.models; -public abstract class DataSourceValidationIssue { +/** + * Represents problems encountered while validating a newly uploaded DataSource. + */ +public class DataSourceValidationIssue { - public abstract String description(); + public Level level; - public abstract Level level(); + public String description; public enum Level { ERROR, WARN, INFO } + public DataSourceValidationIssue (Level level, String description) { + this.level = level; + this.description = description; + } + } From 67fcb409fec3620d7cd2fd133c166275189e810f Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 20 Aug 2021 22:06:48 +0800 Subject: [PATCH 076/187] placeholder CSV ingester class --- .../datasource/CsvDataSourceIngester.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java diff --git a/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java new file mode 100644 index 000000000..11ad2f080 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java @@ -0,0 +1,56 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.models.Bounds; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.r5.analyst.FreeFormPointSet; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.util.ShapefileReader; +import org.locationtech.jts.geom.Envelope; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; + +import java.io.File; + +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; + +/** + * Logic to create SpatialDataSource metadata from a comma separated file. + * Eventually we may want to support other separators like semicolon, tab, vertical bar etc. + * Eventually this could also import non-spatial delimited text files. + */ +public class CsvDataSourceIngester extends DataSourceIngester { + + private final SpatialDataSource dataSource; + + @Override + public SpatialDataSource dataSource () { + return dataSource; + } + + public CsvDataSourceIngester () { + this.dataSource = new SpatialDataSource(); + dataSource.fileFormat = FileStorageFormat.CSV; + } + + @Override + public void ingest (File file, ProgressListener progressListener) { + progressListener.beginTask("Scanning CSV file", 1); + try { + // TODO logic based on FreeFormPointSet.fromCsv() and Grid.fromCsv() + ShapefileReader reader = new ShapefileReader(null); + Envelope envelope = reader.wgs84Bounds(); + checkWgsEnvelopeSize(envelope); + dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); + dataSource.attributes = reader.attributes(); + dataSource.geometryType = reader.geometryType(); + dataSource.featureCount = reader.featureCount(); + } catch (FactoryException | TransformException e) { + throw new RuntimeException("Shapefile transform error. Try uploading an unprojected (EPSG:4326) file.", e); + } catch (Exception e) { + // Must catch because ShapefileReader throws a checked IOException. + throw new RuntimeException("Error parsing shapefile. Ensure the files you uploaded are valid.", e); + } + } + +} From 9ce35a86288661d7ea18d6bbd7270793f438b18e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 24 Aug 2021 16:29:47 +0800 Subject: [PATCH 077/187] synchronousPreload after loading destinations The previous order worked for TAUI sites which don't have destintion pointsets, and don't need them to establish the linkage extents. But for normal regional analyses, we need to load the pointsets first so they can imply a linkage extent to the NetworkPreloader. --- .../r5/analyst/cluster/AnalysisWorker.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index fc5e0e6b6..c409151a9 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -456,13 +456,8 @@ protected void handleOneRegionalTask (RegionalTask task) throws Throwable { // Get the graph object for the ID given in the task, fetching inputs and building as needed. // All requests handled together are for the same graph, and this call is synchronized so the graph will - // only be built once. - // Record the currently loaded network ID so we "stick" to this same graph on subsequent polls. + // only be built once. Record the currently loaded network ID to remain on this same graph on subsequent polls. networkId = task.graphId; - // Note we're completely bypassing the async loader here and relying on the older nested LoadingCaches. - // If those are ever removed, the async loader will need a synchronous mode with per-path blocking (kind of - // reinventing the wheel of LoadingCache) or we'll need to make preparation for regional tasks async. - TransportNetwork transportNetwork = networkPreloader.synchronousPreload(task); // Static site tasks do not specify destinations, but all other regional tasks should. // Load the PointSets based on the IDs (actually, full storage keys including IDs) in the task. @@ -471,6 +466,13 @@ protected void handleOneRegionalTask (RegionalTask task) throws Throwable { task.loadAndValidateDestinationPointSets(pointSetCache); } + // Pull all necessary inputs into cache in a blocking fashion, unlike single-point tasks where prep is async. + // Avoids auto-shutdown while preloading. Must be done after loading destination pointsets to establish extents. + // Note we're completely bypassing the async loader here and relying on the older nested LoadingCaches. + // If those are ever removed, the async loader will need a synchronous mode with per-path blocking (kind of + // reinventing the wheel of LoadingCache) or we'll need to make preparation for regional tasks async. + TransportNetwork transportNetwork = networkPreloader.synchronousPreload(task); + // If we are generating a static site, there must be a single metadata file for an entire batch of results. // Arbitrarily we create this metadata as part of the first task in the job. if (task.makeTauiSite && task.taskId == 0) { From 88dc41e6f6d401c58878eaa69b7cc5274904fab0 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Wed, 25 Aug 2021 09:38:13 +0800 Subject: [PATCH 078/187] Use common base identifier for responses --- .../analysis/controllers/GTFSController.java | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index 3c8d9b0f0..9316f9d4e 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -28,29 +28,45 @@ public GTFSController (GTFSCache gtfsCache) { this.gtfsCache = gtfsCache; } + static class BaseIdentifier { + public final String _id; + public final String name; + + BaseIdentifier (String _id, String name) { + this._id = _id; + this.name = name; + } + } + private GTFSFeed getFeedFromRequest (Request req) { Bundle bundle = Persistence.bundles.findByIdFromRequestIfPermitted(req); String bundleScopedFeedId = Bundle.bundleScopeFeedId(req.params("feedId"), bundle.feedGroupId); return gtfsCache.get(bundleScopedFeedId); } - static class RouteAPIResponse { - public final String id; - public final String name; + static class RouteAPIResponse extends BaseIdentifier { public final int type; public final String color; - RouteAPIResponse(Route route) { - id = route.route_id; - color = route.route_color; + static String getRouteName (Route route) { String tempName = ""; if (route.route_short_name != null) tempName += route.route_short_name; if (route.route_long_name != null) tempName += " " + route.route_long_name; - name = tempName.trim(); + return tempName.trim(); + } + + RouteAPIResponse(Route route) { + super(route.route_id, getRouteName(route)); + color = route.route_color; type = route.route_type; } } + private RouteAPIResponse getRoute(Request req, Response res) { + GTFSFeed feed = getFeedFromRequest(req); + return new RouteAPIResponse(feed.routes.get(req.params("routeId"))); + } + private List getRoutes(Request req, Response res) { GTFSFeed feed = getFeedFromRequest(req); return feed.routes @@ -60,21 +76,19 @@ private List getRoutes(Request req, Response res) { .collect(Collectors.toList()); } - static class PatternAPIResponse { - public final String id; - public final String name; + static class PatternAPIResponse extends BaseIdentifier { public final GeoJSONLineString geometry; public final List orderedStopIds; public final List associatedTripIds; + PatternAPIResponse(Pattern pattern) { - id = pattern.pattern_id; - name = pattern.name; + super(pattern.pattern_id, pattern.name); geometry = serialize(pattern.geometry); orderedStopIds = pattern.orderedStops; associatedTripIds = pattern.associatedTrips; } - GeoJSONLineString serialize (com.vividsolutions.jts.geom.LineString geometry) { + static GeoJSONLineString serialize (com.vividsolutions.jts.geom.LineString geometry) { GeoJSONLineString ret = new GeoJSONLineString(); ret.coordinates = Stream.of(geometry.getCoordinates()) .map(c -> new double[] { c.x, c.y }) @@ -95,15 +109,12 @@ private List getPatternsForRoute (Request req, Response res) .collect(Collectors.toList()); } - static class StopAPIResponse { - public final String id; - public final String name; + static class StopAPIResponse extends BaseIdentifier { public final double lat; public final double lon; StopAPIResponse(Stop stop) { - id = stop.stop_id; - name = stop.stop_name; + super(stop.stop_id, stop.stop_name); lat = stop.stop_lat; lon = stop.stop_lon; } @@ -137,17 +148,14 @@ private List getBundleStops (Request req, Response res) { }).collect(Collectors.toList()); } - static class TripAPIResponse { - public final String id; - public final String name; + static class TripAPIResponse extends BaseIdentifier { public final String headsign; public final Integer startTime; public final Integer duration; public final int directionId; TripAPIResponse(GTFSFeed feed, Trip trip) { - id = trip.trip_id; - name = trip.trip_short_name; + super(trip.trip_id, trip.trip_short_name); headsign = trip.trip_headsign; directionId = trip.direction_id; @@ -179,6 +187,7 @@ private List getTripsForRoute (Request req, Response res) { public void registerEndpoints (spark.Service sparkService) { sparkService.get("/api/gtfs/:_id/stops", this::getBundleStops, toJson); sparkService.get("/api/gtfs/:_id/:feedId/routes", this::getRoutes, toJson); + sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId", this::getRoute, toJson); sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/patterns", this::getPatternsForRoute, toJson); sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/trips", this::getTripsForRoute, toJson); sparkService.get("/api/gtfs/:_id/:feedId/stops", this::getStops, toJson); From 81a14e1975b0f07510ff66a07b63fe26710e56e9 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Wed, 25 Aug 2021 15:41:12 +0800 Subject: [PATCH 079/187] Retrieve the correct modifications on regional creation --- .../analysis/controllers/RegionalAnalysisController.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index bd02e3a15..d0109c725 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -380,9 +380,8 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro analysisRequest.percentiles = DEFAULT_REGIONAL_PERCENTILES; } - final List modificationIds = new ArrayList<>(); List modifications = Persistence.modifications.findPermitted( - QueryBuilder.start("_id").in(modificationIds).get(), + QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(), accessGroup ); @@ -396,7 +395,7 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro ); // Set the destination PointSets, which are required for all non-Taui regional requests. - if (! analysisRequest.makeTauiSite) { + if (!analysisRequest.makeTauiSite) { checkNotNull(analysisRequest.destinationPointSetIds); checkState(analysisRequest.destinationPointSetIds.length > 0, "At least one destination pointset ID must be supplied."); From 635d76d4d57805c06bfcdd9f85f14fa2064816fd Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Thu, 26 Aug 2021 18:18:11 +0800 Subject: [PATCH 080/187] Use feedGroupId instead of passing the bundle --- .../analysis/controllers/GTFSController.java | 50 +++++-------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index 9316f9d4e..cef74e227 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -1,7 +1,6 @@ package com.conveyal.analysis.controllers; import com.conveyal.analysis.models.Bundle; -import com.conveyal.analysis.persistence.Persistence; import com.conveyal.gtfs.GTFSCache; import com.conveyal.gtfs.GTFSFeed; import com.conveyal.gtfs.model.Pattern; @@ -28,7 +27,7 @@ public GTFSController (GTFSCache gtfsCache) { this.gtfsCache = gtfsCache; } - static class BaseIdentifier { + private static class BaseIdentifier { public final String _id; public final String name; @@ -38,9 +37,13 @@ static class BaseIdentifier { } } + private static class GeoJSONLineString { + public final String type = "LineString"; + public double[][] coordinates; + } + private GTFSFeed getFeedFromRequest (Request req) { - Bundle bundle = Persistence.bundles.findByIdFromRequestIfPermitted(req); - String bundleScopedFeedId = Bundle.bundleScopeFeedId(req.params("feedId"), bundle.feedGroupId); + String bundleScopedFeedId = Bundle.bundleScopeFeedId(req.params("feedId"), req.params("feedGroupId")); return gtfsCache.get(bundleScopedFeedId); } @@ -125,29 +128,6 @@ private List getStops (Request req, Response res) { return feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); } - static class FeedStopsAPIResponse { - public final String feedId; - public final List stops; - - FeedStopsAPIResponse(String feedId, List stops) { - this.feedId = feedId; - this.stops = stops; - } - } - - private List getBundleStops (Request req, Response res) { - final Bundle bundle = Persistence.bundles.findByIdFromRequestIfPermitted(req); - return bundle.feeds.stream().map(f -> { - String bundleScopedFeedId = Bundle.bundleScopeFeedId(f.feedId, bundle.feedGroupId); - GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); - List stops = feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); - return new FeedStopsAPIResponse( - f.feedId, - stops - ); - }).collect(Collectors.toList()); - } - static class TripAPIResponse extends BaseIdentifier { public final String headsign; public final Integer startTime; @@ -185,16 +165,10 @@ private List getTripsForRoute (Request req, Response res) { @Override public void registerEndpoints (spark.Service sparkService) { - sparkService.get("/api/gtfs/:_id/stops", this::getBundleStops, toJson); - sparkService.get("/api/gtfs/:_id/:feedId/routes", this::getRoutes, toJson); - sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId", this::getRoute, toJson); - sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/patterns", this::getPatternsForRoute, toJson); - sparkService.get("/api/gtfs/:_id/:feedId/routes/:routeId/trips", this::getTripsForRoute, toJson); - sparkService.get("/api/gtfs/:_id/:feedId/stops", this::getStops, toJson); - } - - private static class GeoJSONLineString { - public final String type = "LineString"; - public double[][] coordinates; + sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes", this::getRoutes, toJson); + sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId", this::getRoute, toJson); + sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId/patterns", this::getPatternsForRoute, toJson); + sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId/trips", this::getTripsForRoute, toJson); + sparkService.get("/api/gtfs/:feedGroupId/:feedId/stops", this::getStops, toJson); } } From 519dfabf5a6d4b71497fdfd76555b49cd6a1fc67 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 27 Aug 2021 11:37:05 +0800 Subject: [PATCH 081/187] WIP on GeoJSON (GeoTools upgrade) --- build.gradle | 4 +- .../datasource/DataSourceIngester.java | 33 ++++- .../datasource/DataSourceUploadAction.java | 28 ++-- .../datasource/GeoJsonDataSourceIngester.java | 133 ++++++++++++++++++ .../datasource/GeoTiffDataSourceIngester.java | 31 ++++ .../ShapefileDataSourceIngester.java | 3 +- .../com/conveyal/r5/util/ShapefileReader.java | 42 +++--- 7 files changed, 227 insertions(+), 47 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java create mode 100644 src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java diff --git a/build.gradle b/build.gradle index 50390bd7d..b8e649ffb 100644 --- a/build.gradle +++ b/build.gradle @@ -154,13 +154,13 @@ dependencies { implementation 'com.beust:jcommander:1.30' // GeoTools provides GIS functionality on top of JTS topology suite. - def geotoolsVersion = '21.2' + def geotoolsVersion = '25.2' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-main' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-opengis' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-referencing' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-shapefile' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-coverage' - implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojson' + implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojsondatastore' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geotiff' // Provides the EPSG coordinate reference system catalog as an HSQL database. implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-epsg-hsql' diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index 345dbc74c..f51f913f1 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -1,9 +1,13 @@ package com.conveyal.analysis.datasource; +import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.DataSource; import com.conveyal.r5.analyst.progress.ProgressListener; +import org.apache.commons.fileupload.FileItem; +import org.bson.types.ObjectId; import java.io.File; +import java.util.stream.Collectors; /** * Logic for loading and validating a specific kind of input file, yielding a specific subclass of DataSource. @@ -12,13 +16,13 @@ public abstract class DataSourceIngester { /** - * An accessor method that gives the general purpose DataSourceUploadAction a view of the DataSource being - * constructed. This allows to DataSourceUploadAction to set all the shared general properties of a DataSource, - * leaving the DataSourceIngester to handle only the details specific to its input format and DataSource subclass. - * Concrete subclasses should ensure that this method can return an object immediately after they're constructed. - * Or maybe only after ingest() returns? + * An accessor method that gives the general purpose DataSourceUploadAction and DataSourceIngester code a view of + * the DataSource being constructed. This allows to DataSourceUploadAction to set all the shared general properties + * of a DataSource and insert it into the database, leaving the DataSourceIngester to handle only the details + * specific to its input format and DataSource subclass. Concrete subclasses should ensure that this method can + * return an object immediately after they're constructed. */ - public abstract DataSource dataSource (); + protected abstract DataSource dataSource (); /** * This method is implemented on concrete subclasses to provide logic for interpreting a particular file type. @@ -30,4 +34,21 @@ public abstract class DataSourceIngester { */ public abstract void ingest (File file, ProgressListener progressListener); + /** + * This method takes care of setting the fields common to all kinds of DataSource, with the specific concrete + * DataSourceIngester taking care of the rest. + * Our no-arg BaseModel constructors are used for deserialization so they don't create an _id or nonce ObjectId(); + */ + public void initializeDataSource (String name, String description, String regionId, UserPermissions userPermissions) { + DataSource dataSource = dataSource(); + dataSource._id = new ObjectId(); + dataSource.nonce = new ObjectId(); + dataSource.name = name; + dataSource.regionId = regionId; + dataSource.createdBy = userPermissions.email; + dataSource.updatedBy = userPermissions.email; + dataSource.accessGroup = userPermissions.accessGroup; + dataSource.description = description; + } + } diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 6898d2f3f..46f701106 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -24,7 +24,9 @@ import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.file.FileCategory.DATASOURCES; +import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; +import static com.conveyal.file.FileStorageFormat.TIFF; import static com.google.common.base.Preconditions.checkNotNull; /** @@ -73,24 +75,6 @@ public DataSourceUploadAction ( this.ingester = ingester; } - /** - * Our no-arg BaseModel constructors are used for deserialization so they don't create an _id or nonce ObjectId(); - * This DataSourceUploadAction takes care of setting the fields common to all kinds of DataSource, with the specific - * DataSourceIngester taking care of the rest. There must be a better way to set all this, but this works for now. - */ - public void initializeDataSource (String name, String regionId, UserPermissions userPermissions) { - DataSource dataSource = ingester.dataSource(); - dataSource._id = new ObjectId(); - dataSource.nonce = new ObjectId(); - dataSource.name = name; - dataSource.regionId = regionId; - dataSource.createdBy = userPermissions.email; - dataSource.updatedBy = userPermissions.email; - dataSource.accessGroup = userPermissions.accessGroup; - String fileNames = fileItems.stream().map(FileItem::getName).collect(Collectors.joining(", ")); - dataSource.description = "From uploaded files: " + fileNames; - } - @Override public final void action (ProgressListener progressListener) throws Exception { progressListener.setWorkProduct(ingester.dataSource().toWorkProduct()); @@ -146,13 +130,19 @@ public static DataSourceUploadAction forFormFields ( DataSourceIngester ingester; if (format == SHP) { ingester = new ShapefileDataSourceIngester(); + } else if (format == GEOJSON) { + ingester = new GeoJsonDataSourceIngester(); + } else if (format == TIFF) { // really this enum value should be GEOTIFF rather than just TIFF. + ingester = new GeoTiffDataSourceIngester(); } else { throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); } + String description = "From uploaded files: " + fileItems.stream() + .map(FileItem::getName).collect(Collectors.joining(", ")); + ingester.initializeDataSource(sourceName, description, regionId, userPermissions); DataSourceUploadAction dataSourceUploadAction = new DataSourceUploadAction(fileStorage, dataSourceCollection, fileItems, ingester); - dataSourceUploadAction.initializeDataSource(sourceName, regionId, userPermissions); return dataSourceUploadAction; } diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java new file mode 100644 index 000000000..024f5ebad --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -0,0 +1,133 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.models.Bounds; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.geojson.GeoJsonModule; +import com.conveyal.r5.analyst.progress.ProgressInputStream; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.util.ShapefileReader; +import org.geotools.data.FeatureReader; +import org.geotools.data.geojson.GeoJSONDataStore; +import org.geotools.data.simple.SimpleFeatureSource; +import org.geotools.feature.FeatureCollection; +import org.geotools.feature.FeatureIterator; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.index.strtree.STRtree; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.feature.type.FeatureType; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; + +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.util.ShapefileReader.geometryType; + +/** + * Logic to create SpatialDataSource metadata from an uploaded GeoJSON file and perform validation. + * We are using the (unsupported) GeoTools module for loading GeoJSON into a FeatureCollection of OpenGIS Features. + * However, GeoJSON deviates significantly from usual GIS concepts. In a GeoJSON feature collection, + * every single object can have a different geometry type and different properties. + * GeoJSON is always in WGS84, which is also how we handle things internally, so we can avoid any CRS transform logic. + * GeoJSON geometries are JSON objects with a type property (Point, LineString, Polygon, MultiPoint, MultiPolygon, + * or MultiLineString) and an array of coordinates. The "multi" types simply have another level of nested arrays. + * Geometries are usually nested into objects of type "Feature", which allows attaching properties. Features can be + * further nested into a top-level object of type FeatureCollection. We only support GeoJSON whose top level object is + * a FeatureCollection (not a single Feature or a single Geometry), and where every geometry is of the same type. + * + * Section 4 of the GeoJSON RFC at https://datatracker.ietf.org/doc/html/rfc7946#section-4 defines the only acceptable + * coordinate reference system as WGS84. You may notice some versions of the GeoTools GeoJSON handler have CRS parsing + * capabilities. This is just support for an obsolete feature and should not be invoked. We instead range check all + * incoming coordinates (via a total bounding box check) to ensure they look reasonable in WGS84. + * + * Current stable Geotools documentation (version 25?) shows a GeoJSONFeatureSource and CSVFeatureSource. + * We're using 21.2 which does not have these. + * + * In GeoTools FeatureSource is a read-only mechanism but it can apparently only return FeatureCollections, which load + * everything into memory. FeatureReader provides iterator-style access, but seems quite low-level and not intended + * for regular use. Because we limit the size of file uploads we can be fairly sure it will be harmless for the backend + * to load any data fully into memory. Streaming capabilities can be added later if the need arises. + * This is explained well at: https://docs.geotools.org/stable/userguide/tutorial/datastore/read.html + * The datastore.getFeatureReader() idiom used in our ShapefileReader class seems to be the right way to stream. + * But it seems unecessary to go through the steps we do steps - our ShapfileReader creates a FeatureSource and FeatureCollection + * in memory. Actually we're doing the same thing in ShapefileMain but worse - supplying a query when there is a + * parameter-less method to call. + * + * As of summer 2021, the unsupported module gt-geojson (package org.geotools.geojson) is deprecated and has been + * replaced with gt-geojsondatastore (package org.geotools.data.geojson), which is on track to supported module status. + * The newer module uses Jackson instead of an abandoned JSON library, and uses standard GeoTools DataStore interfaces. + * We also have our own com.conveyal.geojson.GeoJsonModule which should be phased out if GeoTools support is sufficient. + * + * They've also got flatbuf and geobuf modules - can we replace our custom one? + * + * Note that GeoTools featureReader queries have setCoordinateSystemReproject method - we don't need to do the + * manual reprojection in our ShapefileReader. + * + * Allow Attributes to be of "AMBIGUOUS" or null type, or just drop them if they're ambiguous. + * Flag them as hasMissingValues (or hasNoDataValues). + */ +public class GeoJsonDataSourceIngester extends DataSourceIngester { + + private final SpatialDataSource dataSource; + + @Override + public DataSource dataSource () { + return dataSource; + } + + public GeoJsonDataSourceIngester () { + // Note we're using the no-arg constructor creating a totally empty object. + // Its ID and other general fields will be set later by the enclosing DataSourceUploadAction. + this.dataSource = new SpatialDataSource(); + dataSource.fileFormat = FileStorageFormat.SHP; + } + + @Override + public void ingest (File file, ProgressListener progressListener) { + try { +// InputStream inputStream = new ProgressInputStream( +// progressListener, new BufferedInputStream(new FileInputStream(file)) +// ); + GeoJSONDataStore dataStore = new GeoJSONDataStore(file); + FeatureReader featureReader = dataStore.getFeatureReader(); + while (featureReader.hasNext()) { + SimpleFeature feature = featureReader.next(); + feature.getFeatureType().getGeometryDescriptor().getType(); + } + + SimpleFeatureSource featureSource = dataStore.getFeatureSource(); + featureSource.getBounds(); + featureSource.getSchema(); + dataStore.getSchema(); + dataStore.getBbox(); + + // This loads the whole thing into memory. That should be harmless given our file upload size limits. + // We could also use streamFeatureCollection, which might accommodate more custom validation anyway. + // As an example, internally readFeatureCollection just calls streamFeatureCollection. + // This produces a DefaultFeatureCollection, whose schema is the schema of the first element. That's not + // quite what we want for GeoJSON. So again, we might be better off defining our own streaming logic. + // By streaming over the features, we avoid loading them all into memory but imitate a lot of the logic + // in DefaultFeatureCollection. + FeatureCollection featureCollection = + featureJSON.readFeatureCollection(inputStream); + + // As a GeoJSON file, this is known to be in WGS84. ReferencedEnvelope is a subtype of Envelope. + ReferencedEnvelope envelope = featureCollection.getBounds(); + checkWgsEnvelopeSize(envelope); + + dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); + dataSource.attributes = ShapefileReader.attributes(featureCollection.getSchema()); + dataSource.geometryType = geometryType(featureCollection); + dataSource.featureCount = featureCollection.size(); + } catch (Exception e) { + throw new RuntimeException("Error parsing GeoJSON. Ensure the files you uploaded are valid.", e); + } + } +} diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java new file mode 100644 index 000000000..97b72969f --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java @@ -0,0 +1,31 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.r5.analyst.progress.ProgressListener; + +import java.io.File; + +/** + * GoeTIFFs are used as inputs in network building as digital elevation profiles, and eventually expected to + * serve as impedance or cost fields (e.g. shade bonus and pollution malus). + */ +public class GeoTiffDataSourceIngester extends DataSourceIngester { + + private final SpatialDataSource dataSource; + + @Override + protected DataSource dataSource () { + return dataSource; + } + + @Override + public void ingest (File file, ProgressListener progressListener) { + throw new UnsupportedOperationException(); + } + + public GeoTiffDataSourceIngester () { + this.dataSource = new SpatialDataSource(); + } + +} diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index 41dc34837..d2eaac06c 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.datasource; import com.conveyal.analysis.models.Bounds; +import com.conveyal.analysis.models.DataSource; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; @@ -21,7 +22,7 @@ public class ShapefileDataSourceIngester extends DataSourceIngester { private final SpatialDataSource dataSource; @Override - public SpatialDataSource dataSource () { + public DataSource dataSource () { return dataSource; } diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index a4ebc1e72..48c13b484 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -79,7 +79,7 @@ public Stream stream () throws IOException { FeatureIterator wrapped = features.features(); @Override - public boolean hasNext() { + public boolean hasNext () { boolean hasNext = wrapped.hasNext(); if (!hasNext) { // Prevent keeping a lock on the shapefile. @@ -91,7 +91,7 @@ public boolean hasNext() { } @Override - public SimpleFeature next() { + public SimpleFeature next () { return wrapped.next(); } }; @@ -104,12 +104,7 @@ public ReferencedEnvelope getBounds () throws IOException { } public List numericAttributes () { - return features.getSchema() - .getAttributeDescriptors() - .stream() - .filter(d -> Number.class.isAssignableFrom(d.getType().getBinding())) - .map(AttributeDescriptor::getLocalName) - .collect(Collectors.toList()); + return features.getSchema().getAttributeDescriptors().stream().filter(d -> Number.class.isAssignableFrom(d.getType().getBinding())).map(AttributeDescriptor::getLocalName).collect(Collectors.toList()); } public double getAreaSqKm () throws IOException, TransformException, FactoryException { @@ -150,18 +145,21 @@ public int featureCount () throws IOException { } public List attributes () { + return attributes(features.getSchema()); + } + + /** Static utility method for reuse in other classes importing GeoTools FeatureCollections. */ + public static List attributes (SimpleFeatureType schema) { List attributes = new ArrayList<>(); HashSet uniqueAttributes = new HashSet<>(); - features.getSchema() - .getAttributeDescriptors() - .forEach(d -> { - String attributeName = d.getLocalName(); - AttributeType type = d.getType(); - if (type != null) { - attributes.add(new SpatialAttribute(attributeName, type)); - uniqueAttributes.add(attributeName); - } - }); + schema.getAttributeDescriptors().forEach(d -> { + String attributeName = d.getLocalName(); + AttributeType type = d.getType(); + if (type != null) { + attributes.add(new SpatialAttribute(attributeName, type)); + uniqueAttributes.add(attributeName); + } + }); if (attributes.size() != uniqueAttributes.size()) { throw new AnalysisServerException("Shapefile has duplicate attributes."); } @@ -174,10 +172,16 @@ public enum GeometryType { } public GeometryType geometryType () { - Class geometryClass = features.getSchema().getGeometryDescriptor().getType().getBinding(); + return geometryType(features); + } + + /** Static utility method for reuse in other classes importing GeoTools FeatureCollections. */ + public static GeometryType geometryType (FeatureCollection featureCollection) { + Class geometryClass = featureCollection.getSchema().getGeometryDescriptor().getType().getBinding(); if (Polygonal.class.isAssignableFrom(geometryClass)) return POLYGON; if (Puntal.class.isAssignableFrom(geometryClass)) return POINT; if (Lineal.class.isAssignableFrom(geometryClass)) return LINE; throw new IllegalArgumentException("Could not determine geometry type of features in DataSource."); } + } From 5dd418710df6f22d5a96e604f8ace9072e730cf8 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Sat, 28 Aug 2021 23:01:42 +0800 Subject: [PATCH 082/187] continue work on SpatialDataSource ingestion draft geojson and geopackage ingesters bumped geotools to use a new geojson module --- build.gradle | 7 ++ .../OpportunityDatasetController.java | 1 + .../datasource/DataSourceIngester.java | 20 +++ .../datasource/DataSourceUploadAction.java | 14 +-- .../datasource/GeoJsonDataSourceIngester.java | 114 +++++++++++++----- .../GeoPackageDataSourceIngester.java | 62 ++++++++++ .../ShapefileDataSourceIngester.java | 6 + .../analysis/datasource/SpatialAttribute.java | 28 +++-- .../conveyal/analysis/models/DataSource.java | 8 +- .../com/conveyal/file/FileStorageFormat.java | 11 +- .../java/com/conveyal/r5/analyst/Grid.java | 2 +- .../conveyal/r5/labeling/SpeedLabeler.java | 5 +- .../com/conveyal/r5/util/ShapefileReader.java | 14 ++- .../GeoJsonDataSourceIngesterTest.java | 40 ++++++ .../ShapefileDataSourceIngesterTest.java | 30 +++++ .../datasource/TestingProgressListener.java | 37 ++++++ 16 files changed, 339 insertions(+), 60 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java create mode 100644 src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java create mode 100644 src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java create mode 100644 src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java diff --git a/build.gradle b/build.gradle index b8e649ffb..a49ba2d2b 100644 --- a/build.gradle +++ b/build.gradle @@ -125,6 +125,11 @@ repositories { maven { url 'https://nexus.axiomalaska.com/nexus/content/repositories/public-releases' } } +// Exclude all JUnit 4 transitive dependencies - IntelliJ bug causes it to think we're using Junit 4 instead of 5. +configurations.all { + exclude group: "junit", module: "junit" +} + dependencies { // Provides our logging API implementation 'org.slf4j:slf4j-api:1.7.30' @@ -160,7 +165,9 @@ dependencies { implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-referencing' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-shapefile' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-coverage' + implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojson' // TODO REMOVE DEPRECATED implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojsondatastore' + implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geopkg' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geotiff' // Provides the EPSG coordinate reference system catalog as an HSQL database. implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-epsg-hsql' diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index c4fa6aea8..72c3d0b84 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -409,6 +409,7 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res source.regionId = regionId; updateAndStoreDatasets(source, status, pointsets); } catch (Exception e) { + e.printStackTrace(); status.completeWithError(e); } }); diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index f51f913f1..a74a38e6a 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -2,6 +2,7 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.DataSource; +import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; import org.apache.commons.fileupload.FileItem; import org.bson.types.ObjectId; @@ -9,6 +10,10 @@ import java.io.File; import java.util.stream.Collectors; +import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static com.conveyal.file.FileStorageFormat.SHP; +import static com.conveyal.file.FileStorageFormat.TIFF; + /** * Logic for loading and validating a specific kind of input file, yielding a specific subclass of DataSource. * This plugs into DataSourceUploadAction, which handles the general parts of processing any new DataSource. @@ -51,4 +56,19 @@ public void initializeDataSource (String name, String description, String region dataSource.description = description; } + /** + * Factory method to return an instance of the appropriate concrete subclass for the given file format. + */ + public static DataSourceIngester forFormat (FileStorageFormat format) { + if (format == SHP) { + return new ShapefileDataSourceIngester(); + } else if (format == GEOJSON) { + return new GeoJsonDataSourceIngester(); + } else if (format == TIFF) { // really this enum value should be GEOTIFF rather than just TIFF. + return new GeoTiffDataSourceIngester(); + } else { + throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); + } + + } } diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 46f701106..92880df9d 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -28,6 +28,7 @@ import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.file.FileStorageFormat.TIFF; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; /** * Given a batch of uploaded files, put them into FileStorage, categorize and validate them, and record metadata as @@ -108,6 +109,7 @@ private final void moveFilesIntoStorage (ProgressListener progressListener) { } } checkNotNull(file); + checkState(file.exists()); } /** @@ -127,16 +129,8 @@ public static DataSourceUploadAction forFormFields ( final List fileItems = formFields.get("sourceFiles"); FileStorageFormat format = detectUploadFormatAndValidate(fileItems); - DataSourceIngester ingester; - if (format == SHP) { - ingester = new ShapefileDataSourceIngester(); - } else if (format == GEOJSON) { - ingester = new GeoJsonDataSourceIngester(); - } else if (format == TIFF) { // really this enum value should be GEOTIFF rather than just TIFF. - ingester = new GeoTiffDataSourceIngester(); - } else { - throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); - } + DataSourceIngester ingester = DataSourceIngester.forFormat(format); + String description = "From uploaded files: " + fileItems.stream() .map(FileItem::getName).collect(Collectors.joining(", ")); ingester.initializeDataSource(sourceName, description, regionId, userPermissions); diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 024f5ebad..4f3ea231d 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -2,6 +2,7 @@ import com.conveyal.analysis.models.Bounds; import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.DataSourceValidationIssue; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.file.FileStorageFormat; import com.conveyal.geojson.GeoJsonModule; @@ -10,23 +11,35 @@ import com.conveyal.r5.util.ShapefileReader; import org.geotools.data.FeatureReader; import org.geotools.data.geojson.GeoJSONDataStore; +import org.geotools.data.geojson.GeoJSONFeatureSource; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.CRS; +import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.index.strtree.STRtree; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.feature.type.AttributeDescriptor; +import org.opengis.feature.type.AttributeType; import org.opengis.feature.type.FeatureType; import org.opengis.referencing.FactoryException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.operation.TransformException; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import static com.conveyal.analysis.models.DataSourceValidationIssue.Level.ERROR; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; import static com.conveyal.r5.util.ShapefileReader.geometryType; @@ -68,13 +81,21 @@ * They've also got flatbuf and geobuf modules - can we replace our custom one? * * Note that GeoTools featureReader queries have setCoordinateSystemReproject method - we don't need to do the - * manual reprojection in our ShapefileReader. + * manual reprojection in our ShapefileReader as we currently are. * * Allow Attributes to be of "AMBIGUOUS" or null type, or just drop them if they're ambiguous. - * Flag them as hasMissingValues (or hasNoDataValues). + * Flag them as hasMissingValues, or the quantity of missing values. + * + * Be careful, QGIS will happily export GeoJSON with a CRS property which is no longer considered valid: + * "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::3857" } } + * If a CRS is present, make sure it matches one of the names for WGS84. Throw a warning if the field is present at all. + * + * See also: com.conveyal.r5.analyst.scenario.IndexedPolygonCollection#loadFromS3GeoJson() */ public class GeoJsonDataSourceIngester extends DataSourceIngester { + public static final int MIN_GEOJSON_FILE_LENGTH = "{'type':'GeometryCollection','features':[]}".length(); + private final SpatialDataSource dataSource; @Override @@ -86,48 +107,77 @@ public GeoJsonDataSourceIngester () { // Note we're using the no-arg constructor creating a totally empty object. // Its ID and other general fields will be set later by the enclosing DataSourceUploadAction. this.dataSource = new SpatialDataSource(); - dataSource.fileFormat = FileStorageFormat.SHP; + dataSource.fileFormat = FileStorageFormat.GEOJSON; } + @Override public void ingest (File file, ProgressListener progressListener) { + // Check that file exists and is not empty. + // Geotools GeoJson reader fails with stack overflow on empty/missing file. TODO: File GeoTools issue. + if (!file.exists()) { + throw new IllegalArgumentException("File does not exist: " + file.getPath()); + } + if (file.length() < MIN_GEOJSON_FILE_LENGTH) { + throw new IllegalArgumentException("File is too short to be GeoJSON, length is: " + file.length()); + } try { -// InputStream inputStream = new ProgressInputStream( -// progressListener, new BufferedInputStream(new FileInputStream(file)) -// ); + // Note that most of this logic is identical to Shapefile and GeoPackage, extract common code. GeoJSONDataStore dataStore = new GeoJSONDataStore(file); - FeatureReader featureReader = dataStore.getFeatureReader(); - while (featureReader.hasNext()) { - SimpleFeature feature = featureReader.next(); - feature.getFeatureType().getGeometryDescriptor().getType(); - } - SimpleFeatureSource featureSource = dataStore.getFeatureSource(); - featureSource.getBounds(); - featureSource.getSchema(); - dataStore.getSchema(); - dataStore.getBbox(); - // This loads the whole thing into memory. That should be harmless given our file upload size limits. - // We could also use streamFeatureCollection, which might accommodate more custom validation anyway. - // As an example, internally readFeatureCollection just calls streamFeatureCollection. - // This produces a DefaultFeatureCollection, whose schema is the schema of the first element. That's not - // quite what we want for GeoJSON. So again, we might be better off defining our own streaming logic. - // By streaming over the features, we avoid loading them all into memory but imitate a lot of the logic - // in DefaultFeatureCollection. - FeatureCollection featureCollection = - featureJSON.readFeatureCollection(inputStream); - - // As a GeoJSON file, this is known to be in WGS84. ReferencedEnvelope is a subtype of Envelope. + FeatureCollection featureCollection = featureSource.getFeatures(); + // The schema of the FeatureCollection does seem to reflect all attributes present on all features. + // However the type of those attributes seems to be restricted to that of the first value encountered. + // Conversions may fail silently on any successive instances of that property with a different type. + SimpleFeatureType featureType = featureCollection.getSchema(); + // Note: this somewhat duplicates ShapefileReader.attributes, code should be reusable across formats + // But look into the null checking and duplicate attribute checks there. + dataSource.attributes = new ArrayList<>(); + for (AttributeDescriptor descriptor : featureType.getAttributeDescriptors()) { + dataSource.attributes.add(new SpatialAttribute(descriptor)); + } + // The schema always reports the geometry type as the very generic "Geometry" class. + // Check that all features have the same concrete Geometry type. + Class firstGeometryType = null; + FeatureIterator iterator = featureCollection.features(); + while (iterator.hasNext()) { + SimpleFeature feature = iterator.next(); + Geometry geometry = (Geometry) feature.getDefaultGeometry(); + if (geometry == null) { + dataSource.addIssue(ERROR, "Geometry is null on feature: " + feature.getID()); + continue; + } + if (firstGeometryType == null) { + firstGeometryType = geometry.getClass(); + } else if (firstGeometryType != geometry.getClass()) { + dataSource.addIssue(ERROR, "Inconsistent geometry type on feature: " + feature.getID()); + continue; + } + } + // Set SpatialDataSource fields (Conveyal metadata) from GeoTools model ReferencedEnvelope envelope = featureCollection.getBounds(); + // TODO Also check bounds for antimeridian crossing or out-of-range checkWgsEnvelopeSize(envelope); - dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); - dataSource.attributes = ShapefileReader.attributes(featureCollection.getSchema()); - dataSource.geometryType = geometryType(featureCollection); + // Cannot set from FeatureType because it's always Geometry for GeoJson. + dataSource.geometryType = ShapefileReader.GeometryType.forBindingClass(firstGeometryType); dataSource.featureCount = featureCollection.size(); - } catch (Exception e) { - throw new RuntimeException("Error parsing GeoJSON. Ensure the files you uploaded are valid.", e); + } catch (Throwable t) { + throw new RuntimeException("Error parsing GeoJSON. Please ensure the files you uploaded are valid.", t); } } + + /** + * GeoJSON used to allow CRS, but the RFC now says GeoJSON is always in WGS84 and no other CRS are allowed. + * QGIS and GeoTools both seem to support this, but it's an obsolete feature. + */ + private static void checkCrs (FeatureType featureType) throws FactoryException { + CoordinateReferenceSystem crs = featureType.getCoordinateReferenceSystem(); + if (crs != null && !DefaultGeographicCRS.WGS84.equals(crs) && !CRS.decode("CRS:84").equals(crs)) { + throw new RuntimeException("GeoJSON should specify no coordinate reference system, and contain unprojected " + + "WGS84 coordinates. CRS is: " + crs.toString()); + } + + } } diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java new file mode 100644 index 000000000..2730efe9f --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -0,0 +1,62 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.models.Bounds; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.util.ShapefileReader; +import org.geotools.data.DataStore; +import org.geotools.data.DataStoreFinder; +import org.geotools.data.geojson.GeoJSONDataStore; +import org.geotools.data.simple.SimpleFeatureSource; +import org.geotools.feature.FeatureCollection; +import org.geotools.feature.FeatureIterator; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.locationtech.jts.geom.Geometry; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.feature.simple.SimpleFeatureType; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.util.ShapefileReader.geometryType; + +/** + * Logic to create SpatialDataSource metadata from an uploaded GeoPackage file and perform validation. + * We are using the (unsupported) GeoTools module for loading GeoPackages into OpenGIS Features. + * + * Note that a GeoPackage can be a vector or a raster (coverage). We should handle both cases. + */ +public class GeoPackageDataSourceIngester extends DataSourceIngester { + + private final SpatialDataSource dataSource; + + @Override + public DataSource dataSource () { + return dataSource; + } + + public GeoPackageDataSourceIngester () { + // Note we're using the no-arg constructor creating a totally empty object. + // Its ID and other general fields will be set later by the enclosing DataSourceUploadAction. + this.dataSource = new SpatialDataSource(); + dataSource.fileFormat = FileStorageFormat.SHP; + } + + @Override + public void ingest (File file, ProgressListener progressListener) { + try { + Map params = new HashMap(); + params.put("dbtype", "geopkg"); + params.put("database", file.getAbsolutePath()); + DataStore datastore = DataStoreFinder.getDataStore(params); + // TODO Remaining logic should be similar to Shapefile and GeoJson + } catch (Exception e) { + throw new RuntimeException("Error parsing GeoPackage. Ensure the file you uploaded is valid.", e); + } + } + +} diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index d2eaac06c..d158a01cd 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -7,12 +7,14 @@ import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader; import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; import org.opengis.referencing.FactoryException; import org.opengis.referencing.operation.TransformException; import java.io.File; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.google.common.base.Preconditions.checkState; /** * Logic to create SpatialDataSource metadata from a Shapefile. @@ -38,8 +40,12 @@ public void ingest (File file, ProgressListener progressListener) { progressListener.beginTask("Validating files", 1); try { ShapefileReader reader = new ShapefileReader(file); + // Iterate over all features to ensure file is readable, geometries are valid, and can be reprojected. Envelope envelope = reader.wgs84Bounds(); checkWgsEnvelopeSize(envelope); + reader.wgs84Stream().forEach(f -> { + checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); + }); dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); dataSource.attributes = reader.attributes(); dataSource.geometryType = reader.geometryType(); diff --git a/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java index 4e7cef853..4cecbc103 100644 --- a/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.datasource; import org.locationtech.jts.geom.Geometry; +import org.opengis.feature.type.AttributeDescriptor; import org.opengis.feature.type.AttributeType; /** @@ -20,20 +21,33 @@ public class SpatialAttribute { /** The data type of the attribute - for our purposes primarily distinguishing between numbers and text. */ public Type type; - private enum Type { - NUMBER, // internally, we generally parse as doubles + /** On how many features does this attribute occur? */ + public int occurrances = 0; + + public enum Type { + NUMBER, // internally, we generally work with doubles so all numeric GIS types can be up-converted TEXT, GEOM, - ERROR + ERROR; + public static Type forBindingClass (Class binding) { + if (Number.class.isAssignableFrom(binding)) return Type.NUMBER; + else if (String.class.isAssignableFrom(binding)) return Type.TEXT; + else if (Geometry.class.isAssignableFrom(binding)) return Type.GEOM; + else return Type.ERROR; + } } + /** + * Given an OpenGIS AttributeType, create a new Conveyal attribute metadata object reflecting it. + */ public SpatialAttribute(String name, AttributeType type) { this.name = name; this.label = name; - if (Number.class.isAssignableFrom(type.getBinding())) this.type = Type.NUMBER; - else if (String.class.isAssignableFrom(type.getBinding())) this.type = Type.TEXT; - else if (Geometry.class.isAssignableFrom(type.getBinding())) this.type = Type.GEOM; - else this.type = Type.ERROR; + this.type = Type.forBindingClass(type.getBinding()); + } + + public SpatialAttribute(AttributeDescriptor descriptor) { + this(descriptor.getLocalName(), descriptor.getType()); } /** No-arg constructor required for Mongo POJO deserialization. */ diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java index e4a99e90d..529aa44db 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSource.java +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -5,6 +5,7 @@ import com.conveyal.r5.analyst.progress.WorkProduct; import org.bson.codecs.pojo.annotations.BsonDiscriminator; +import java.util.ArrayList; import java.util.List; import static com.conveyal.r5.analyst.progress.WorkProductType.DATA_SOURCE; @@ -42,8 +43,11 @@ public abstract class DataSource extends BaseModel { // This type uses (north, south, east, west), ideally we'd use (minLon, minLat, maxLon, maxLat). public Bounds wgsBounds; - /** Problems encountered while loading. TODO should this be a separate json file in storage? */ - public List issues; + /** + * Problems encountered while loading. + * TODO should this be a separate json file in storage? Should it be a Set to deduplicate? + */ + public List issues = new ArrayList<>(); public DataSource (UserPermissions user, String name) { super(user, name); diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index 0bd132107..d8ddc16b5 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -2,10 +2,14 @@ import org.bson.codecs.pojo.annotations.BsonIgnore; +/** + * An enumeration of all the file types we handle as uploads, derived internal data, or work products. + * Really this should be a union of several enumerated types (upload/internal/product) but Java does not allow this. + */ public enum FileStorageFormat { FREEFORM("pointset", "application/octet-stream"), GRID("grid", "application/octet-stream"), - POINTSET("pointset", "application/octet-stream"), + POINTSET("pointset", "application/octet-stream"), // Why is this "pointset" extension duplicated? PNG("png", "image/png"), TIFF("tiff", "image/tiff"), CSV("csv", "text/csv"), @@ -17,7 +21,10 @@ public enum FileStorageFormat { // In our internal storage, we may want to force less ambiguous .gtfs.zip .osm.pbf and .geo.json. GTFS("zip", "application/zip"), OSMPBF("pbf", "application/octet-stream"), - GEOJSON("json", "application/json"); + // Also can be application/geo+json, see https://www.iana.org/assignments/media-types/application/geo+json + GEOJSON("json", "application/json"), + // See requirement 3 http://www.geopackage.org/spec130/#_file_extension_name + GEOPACKAGE("gpkg", "application/geopackage+sqlite3"); // These should not be serialized into Mongo. Default Enum codec uses String name() and valueOf(String). // TODO clarify whether the extension is used for backend storage, or for detecting type up uploaded files. diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 9525c8a5e..362d8a3ce 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -713,7 +713,7 @@ public static List fromShapefile (File shapefile, int zoom, ProgressListen } else if (geom instanceof Polygon || geom instanceof MultiPolygon) { grid.rasterize(geom, numericVal); } else { - throw new IllegalArgumentException("Unsupported geometry type"); + throw new IllegalArgumentException("Unsupported geometry type: " + geom); } } diff --git a/src/main/java/com/conveyal/r5/labeling/SpeedLabeler.java b/src/main/java/com/conveyal/r5/labeling/SpeedLabeler.java index 6360da4d2..7bbb599d1 100644 --- a/src/main/java/com/conveyal/r5/labeling/SpeedLabeler.java +++ b/src/main/java/com/conveyal/r5/labeling/SpeedLabeler.java @@ -13,8 +13,9 @@ import static systems.uom.common.USCustomary.KNOT; import static systems.uom.common.USCustomary.MILE_PER_HOUR; -import static tec.uom.se.unit.Units.KILOMETRE_PER_HOUR; -import static tec.uom.se.unit.Units.METRE_PER_SECOND; +import static tech.units.indriya.unit.Units.KILOMETRE_PER_HOUR; +import static tech.units.indriya.unit.Units.METRE_PER_SECOND; +import static tech.units.indriya.unit.Units.KILOMETRE_PER_HOUR; /** * Gets information about max speeds based on highway tags from build-config diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 48c13b484..6769b5884 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -118,6 +118,9 @@ public double getAreaSqKm () throws IOException, TransformException, FactoryExce public Stream wgs84Stream () throws IOException, TransformException { return stream().map(f -> { org.locationtech.jts.geom.Geometry g = (org.locationtech.jts.geom.Geometry) f.getDefaultGeometry(); + if (g == null) { + throw new RuntimeException("Null geometry on feature: " + f.getID()); + } try { // TODO does this leak beyond this function? f.setDefaultGeometry(JTS.transform(g, transform)); @@ -169,6 +172,12 @@ public static List attributes (SimpleFeatureType schema) { /** These are very broad. For example, line includes linestring and multilinestring. */ public enum GeometryType { POLYGON, POINT, LINE; + public static GeometryType forBindingClass (Class binding) { + if (Polygonal.class.isAssignableFrom(binding)) return POLYGON; + if (Puntal.class.isAssignableFrom(binding)) return POINT; + if (Lineal.class.isAssignableFrom(binding)) return LINE; + throw new IllegalArgumentException("Could not determine geometry type of features in DataSource."); + } } public GeometryType geometryType () { @@ -178,10 +187,7 @@ public GeometryType geometryType () { /** Static utility method for reuse in other classes importing GeoTools FeatureCollections. */ public static GeometryType geometryType (FeatureCollection featureCollection) { Class geometryClass = featureCollection.getSchema().getGeometryDescriptor().getType().getBinding(); - if (Polygonal.class.isAssignableFrom(geometryClass)) return POLYGON; - if (Puntal.class.isAssignableFrom(geometryClass)) return POINT; - if (Lineal.class.isAssignableFrom(geometryClass)) return LINE; - throw new IllegalArgumentException("Could not determine geometry type of features in DataSource."); + return GeometryType.forBindingClass(geometryClass); } } diff --git a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java new file mode 100644 index 000000000..78fcab849 --- /dev/null +++ b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java @@ -0,0 +1,40 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.r5.analyst.progress.NoopProgressListener; +import org.junit.jupiter.api.Test; + +import java.io.File; + +import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Created by abyrd on 2021-08-27 + * TODO Instead of loading from files, build GeoJSON programmatically, serialize it to temp files, then load it. + */ +class GeoJsonDataSourceIngesterTest { + + String[] inputs = new String[] { +// "/Users/abyrd/geodata/test-ingest/hkzones-too-small.geojson", +// "/Users/abyrd/geodata/test-ingest/hkzones-empty.geojson", + "/Users/abyrd/geodata/test-ingest/hkzones-type-mismatch.geojson", + "/Users/abyrd/geodata/test-ingest/hkzones-extra-attribute.geojson", + "/Users/abyrd/geodata/test-ingest/hkzones-mixed-numeric.geojson", + "/Users/abyrd/geodata/test-ingest/hkzones-mixed-geometries.geojson", + "/Users/abyrd/geodata/test-ingest/hkzones-mercator.geojson", + "/Users/abyrd/geodata/test-ingest/hkzones-wgs84.geojson", + }; + + @Test + void testGeoJsonProcessing () { + for (String input : inputs) { + TestingProgressListener progressListener = new TestingProgressListener(); + DataSourceIngester ingester = DataSourceIngester.forFormat(GEOJSON); + File geoJsonInputFile = new File(input); + ingester.ingest(geoJsonInputFile, progressListener); + ingester.toString(); + // TODO progressListener.assertUsedCorrectly(); + } + } + +} \ No newline at end of file diff --git a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java new file mode 100644 index 000000000..5de16b59e --- /dev/null +++ b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java @@ -0,0 +1,30 @@ +package com.conveyal.analysis.datasource; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.File; + +import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static com.conveyal.file.FileStorageFormat.SHP; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * + */ +class ShapefileDataSourceIngesterTest { + + @Test + void testNullPoints () { + TestingProgressListener progressListener = new TestingProgressListener(); + DataSourceIngester ingester = DataSourceIngester.forFormat(SHP); + File inputFile = new File("/Users/abyrd/geodata/test-ingest/nl-null-points.shp"); + Throwable thrown = assertThrows( + RuntimeException.class, + () -> ingester.ingest(inputFile, progressListener), + "Expected exception on shapefile with null geometries." + ); + // TODO progressListener.assertUsedCorrectly(); + } + +} \ No newline at end of file diff --git a/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java b/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java new file mode 100644 index 000000000..f4a2c8b6c --- /dev/null +++ b/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java @@ -0,0 +1,37 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.analyst.progress.WorkProduct; +import org.junit.jupiter.api.Assertions; + +/** + * A mock ProgressListener for use in tests, which makes sure all the interface methods are called and shows progress. + */ +public class TestingProgressListener implements ProgressListener { + + private String description; + private WorkProduct workProduct; + private int count = 0; + + @Override + public void beginTask (String description, int totalElements) { + this.description = description; + } + + @Override + public void increment (int n) { + count += n; + } + + @Override + public void setWorkProduct (WorkProduct workProduct) { + this.workProduct = workProduct; + } + + public void assertUsedCorrectly () { + Assertions.assertNotNull(description); + Assertions.assertNotNull(workProduct); + Assertions.assertTrue(count > 0); + } + +} From 707cc6908e155885c319080e033f4d0800906bd7 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Sat, 28 Aug 2021 23:42:01 +0800 Subject: [PATCH 083/187] wire up geojson and geopkg ingesters --- .../analysis/datasource/DataSourceIngester.java | 2 +- .../datasource/GeoJsonDataSourceIngester.java | 5 ++++- .../GeoPackageDataSourceIngester.java | 17 ++++++++++++++++- .../analysis/datasource/SpatialLayers.java | 9 ++++++++- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index a74a38e6a..9ef67d945 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -67,7 +67,7 @@ public static DataSourceIngester forFormat (FileStorageFormat format) { } else if (format == TIFF) { // really this enum value should be GEOTIFF rather than just TIFF. return new GeoTiffDataSourceIngester(); } else { - throw new IllegalArgumentException("Ingestion logic not yet defined for format: " + format); + return new GeoPackageDataSourceIngester(); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 4f3ea231d..a81dbf305 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -155,15 +155,18 @@ public void ingest (File file, ProgressListener progressListener) { continue; } } + checkCrs(featureType); // Set SpatialDataSource fields (Conveyal metadata) from GeoTools model ReferencedEnvelope envelope = featureCollection.getBounds(); - // TODO Also check bounds for antimeridian crossing or out-of-range + // TODO Range-check lats and lons, projection of bad inputs can give negative areas (even check every feature) + // TODO Also check bounds for antimeridian crossing checkWgsEnvelopeSize(envelope); dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); // Cannot set from FeatureType because it's always Geometry for GeoJson. dataSource.geometryType = ShapefileReader.GeometryType.forBindingClass(firstGeometryType); dataSource.featureCount = featureCollection.size(); } catch (Throwable t) { + // Unexpected errors cause immediate failure; predictable issues will be recorded on the DataSource object. throw new RuntimeException("Error parsing GeoJSON. Please ensure the files you uploaded are valid.", t); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java index 2730efe9f..a45a20875 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -8,11 +8,13 @@ import com.conveyal.r5.util.ShapefileReader; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; +import org.geotools.data.FeatureSource; import org.geotools.data.geojson.GeoJSONDataStore; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; import org.geotools.geometry.jts.ReferencedEnvelope; +import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; @@ -22,7 +24,9 @@ import java.util.Map; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.util.ShapefileReader.attributes; import static com.conveyal.r5.util.ShapefileReader.geometryType; +import static com.google.common.base.Preconditions.checkState; /** * Logic to create SpatialDataSource metadata from an uploaded GeoPackage file and perform validation. @@ -43,7 +47,7 @@ public GeoPackageDataSourceIngester () { // Note we're using the no-arg constructor creating a totally empty object. // Its ID and other general fields will be set later by the enclosing DataSourceUploadAction. this.dataSource = new SpatialDataSource(); - dataSource.fileFormat = FileStorageFormat.SHP; + dataSource.fileFormat = FileStorageFormat.GEOPACKAGE; } @Override @@ -54,6 +58,17 @@ public void ingest (File file, ProgressListener progressListener) { params.put("database", file.getAbsolutePath()); DataStore datastore = DataStoreFinder.getDataStore(params); // TODO Remaining logic should be similar to Shapefile and GeoJson + FeatureSource featureSource = datastore.getFeatureSource(datastore.getTypeNames()[0]); + FeatureCollection featureCollection = featureSource.getFeatures(); + Envelope envelope = featureCollection.getBounds(); + checkWgsEnvelopeSize(envelope); // Note this may be projected TODO reprojection logic +// reader.wgs84Stream().forEach(f -> { +// checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); +// }); + dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); + dataSource.attributes = attributes(featureCollection.getSchema()); + dataSource.geometryType = geometryType(featureCollection); + dataSource.featureCount = featureCollection.size(); } catch (Exception e) { throw new RuntimeException("Error parsing GeoPackage. Ensure the file you uploaded is valid.", e); } diff --git a/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java index e9d9d8966..b0b7e27c6 100644 --- a/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java @@ -61,6 +61,7 @@ public static FileStorageFormat detectUploadFormatAndValidate (List fi } // Even if we've already detected a shapefile, run the other tests to check for a bad mixture of file types. + // TODO factor the size == 1 check out of all cases if (fileExtensions.contains("GRID")) { if (fileExtensions.size() == 1) { uploadFormat = FileStorageFormat.GRID; @@ -75,10 +76,16 @@ public static FileStorageFormat detectUploadFormatAndValidate (List fi String message = "When uploading CSV you may only upload one file at a time."; throw AnalysisServerException.fileUpload(message); } + } else if (fileExtensions.contains("GEOJSON") || fileExtensions.contains("JSON")) { + uploadFormat = FileStorageFormat.GEOJSON; + } else if (fileExtensions.contains("GPKG")) { + uploadFormat = FileStorageFormat.GEOPACKAGE; + } else if (fileExtensions.contains("TIFF") || fileExtensions.contains("TIF")) { + uploadFormat = FileStorageFormat.TIFF; } if (uploadFormat == null) { - throw AnalysisServerException.fileUpload("Could not detect format of opportunity dataset upload."); + throw AnalysisServerException.fileUpload("Could not detect format of uploaded spatial data."); } return uploadFormat; } From 1e3dbe1150a2b0dfc96d40009d4a4d38b9da1a82 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Sat, 28 Aug 2021 23:53:24 +0800 Subject: [PATCH 084/187] check that there is one typeName in geopackage --- .../datasource/GeoPackageDataSourceIngester.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java index a45a20875..17f28ce4f 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -58,7 +58,14 @@ public void ingest (File file, ProgressListener progressListener) { params.put("database", file.getAbsolutePath()); DataStore datastore = DataStoreFinder.getDataStore(params); // TODO Remaining logic should be similar to Shapefile and GeoJson - FeatureSource featureSource = datastore.getFeatureSource(datastore.getTypeNames()[0]); + // Some GeoTools DataStores have multiple tables ("type names") available. GeoPackage seems to allow this. + // Shapefile has only one per DataStore, so the ShapefileDataStore provides a convenience method that does + // this automatically. + String[] typeNames = datastore.getTypeNames(); + if (typeNames.length != 1) { + throw new RuntimeException("GeoPackage must contain only one table, this file has " + typeNames.length); + } + FeatureSource featureSource = datastore.getFeatureSource(typeNames[0]); FeatureCollection featureCollection = featureSource.getFeatures(); Envelope envelope = featureCollection.getBounds(); checkWgsEnvelopeSize(envelope); // Note this may be projected TODO reprojection logic From ca08e5f92716a11d237e02db6f132adafa32455d Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 30 Aug 2021 16:31:35 +0800 Subject: [PATCH 085/187] add unit test files --- .../analysis/datasource/hk-zones-epsg2326.gpkg | Bin 0 -> 98304 bytes .../analysis/datasource/hk-zones-wgs84.gpkg | Bin 0 -> 98304 bytes .../analysis/datasource/hkzones-empty.geojson | 0 .../datasource/hkzones-extra-attribute.geojson | 11 +++++++++++ .../datasource/hkzones-mercator.geojson | 11 +++++++++++ .../datasource/hkzones-mixed-geometries.geojson | 11 +++++++++++ .../datasource/hkzones-mixed-numeric.geojson | 11 +++++++++++ .../datasource/hkzones-no-collection.geojson | 6 ++++++ .../datasource/hkzones-too-small.geojson | 1 + .../datasource/hkzones-type-mismatch.geojson | 11 +++++++++++ .../analysis/datasource/hkzones-wgs84.geojson | 11 +++++++++++ .../analysis/datasource/nl-null-points.cpg | 1 + .../analysis/datasource/nl-null-points.dbf | Bin 0 -> 286 bytes .../analysis/datasource/nl-null-points.prj | 1 + .../analysis/datasource/nl-null-points.qpj | 1 + .../analysis/datasource/nl-null-points.shp | Bin 0 -> 612 bytes .../analysis/datasource/nl-null-points.shx | Bin 0 -> 260 bytes 17 files changed, 76 insertions(+) create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hk-zones-epsg2326.gpkg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hk-zones-wgs84.gpkg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-empty.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-extra-attribute.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-mercator.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-geometries.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-numeric.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-no-collection.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-too-small.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-type-mismatch.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/hkzones-wgs84.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/nl-null-points.cpg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/nl-null-points.dbf create mode 100644 src/test/resources/com/conveyal/analysis/datasource/nl-null-points.prj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/nl-null-points.qpj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/nl-null-points.shp create mode 100644 src/test/resources/com/conveyal/analysis/datasource/nl-null-points.shx diff --git a/src/test/resources/com/conveyal/analysis/datasource/hk-zones-epsg2326.gpkg b/src/test/resources/com/conveyal/analysis/datasource/hk-zones-epsg2326.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..4e2300de17d1c860cb45299cc08d92230121fa3d GIT binary patch literal 98304 zcmeI5eQYDgb-=meNt8}s=h%02s`zfqSRpsJltk+Ld>5N1t$2GUkrF>F%Zj^MlS^uQ zC7&dT<%glo^;Kf(zlRA?#!E+H^2AZ%+4&uO|LCWx`+yjS{8IP={e(}D9_Uf zc|4vW{QD^WwT|=n%_-{yzfvAg@CW2PzIr2WOSiur@{CQ)cjJ_<;kQZpqr>kEzdb+= ze5P-!|7-m>eJ}T2=-%;NIW^byCiNwIL3rOFu)lc5H#%+rP*b6A%jveF3pBwSCgp>gMy}BNRg2)yigKR(4|8#h+dME zlAwzk$|!8#};vurT2Bhr`sX z^F}N<{mgkjXgOZ&>N1O*xhP~*!plNgL>czk3`!*NZ+3Z^v1!UBGAxPwdR|kM9G1H! zO2sW54*-tB#^_@PWb3@c3Ra|D9A3M;CfyRzVlqoQR9DH&+-se=kW8dAct}ZPP~|4C z>t(c>;#Oj*4RnLuurpw6Ax1N{Vg+2RT;^@G$mqt!6V4iFlwz0gaFSSH(+6d zhi%!NWV195R?-;hrDpd&yV?w%?+F7#k*1j4#-($1FUcmjwJe_K8&TXOk7Jz3%gu(j z33n9QPH)Qb$M+?!ugCxVY+GNd8>QwEe6*!coaTD(zAAI{akkJ`J=g8?kB?K;&4yzt zMHO4i9PkVU>`YsL(>(wHKVB^bh2#|2#~i=p7?MtHOw z9?fS)uU#5lxingMF31FBWhY3}jC~|$kbMMa%C>G)PrvV!d;YpZ=PWz4+?bi+9u^MK*|j3oV?O8Z*6g7{hs+QAtUMxtmWK`jB$C}^a?w=t` z`{Ch1{P)!9cRWLp!M{8G&d{guDI7om2mk>f00e*l5C8%|;8&5rE16H6>y3HNc70s* zTvOyCxo1eN?u<`Zw3yR3ovmu8F$MB7`%qdu9!u6 z(gaVM2r%hYe8VEiEp7%lywX->mbJ1DyZeo5aIyLiri$-QtHs+!2Jh-yyI6*YjhjHO~LY=%v32Jj6eN#D+kyi(v5e8)_Z157kJHW`UVCaq(t zsYDJh(p9|wR~E71B>~GvB2#0L*|}-!z=UiqXgLAjd@Be!(l!%m$}0#ZteJ=x@seB& zFmsbLGxJkp6Vok(WJT4tOsUBH#LWEI-1CfykL(L#dXZMJCQQz5SkcMo3^v#YJ-#cR zv%MF}S-dTc7lL9Q;a#7iDpa_W!sQo=4NBD^Iks-m0|5ngTL zc-Z`fb@-n2oTV<@Y_yUSRD9Ej8!V3pbBfKmT5>k=LSsYSxW6|l^XmoOVrj<9&PE1Y^ zzjp(kF#+G-h%j|1;eo_(@^Q=C@eV595-^OWAxTDgF(;Mrcw$DPX51#+5>bt&R@o9& zMaq-)KQ;VC5B`Az2mk>f00e*l5C8%|00;m9AOHk_01&u;1U9Mu@yEsouXyp%wRJOm zOOZt_ELOB4+3=}-l16i7L?@z=@Wfm=I+vN4y%d?eG%+(a8<~7=;LF~DwZy=8hX-!| zUf;RXxL?%pn;!fF2M_=PKmZ5;0U!VbfB+Bx0zd!=00AIy@(DcR9q)7P62SWZf z00e*l5C8%|;A9dwOL@mB*X@5V-2XqB<^-*Q01yBIKmZ5;0U!VbfB+Bx0zd!=973SY z{r}fJ!>=Dw3X*{U5C8%|00;m9AOHk_01yBIKmZ5;fd`ksDCHYJJJQwL>$>Mp9s#f00e*l5C8%|00;nqBM^Z5|3`oZMSuVh00KY&2mk>f00e*l z5C8%|00^8g0)sRC9BCks&kuM84Y1ew$k@ET$zg@`-y6BWyNU^aD zi!v!L9%oZ1=*(|bM-W|Qmy#(KWmgwT;p7skHE6Vik_mS`L4->nHnwmLrIPEof;g8z zX?B@i$e>h=OS7Ta)nqEeprENIQe>nHFO))=|gGCDLpvb4-k8o%a#r^g z$|B0J&t^~}iGQ=p%Z#n7Tq47g5xJh%G+G^&yCq7+EgiYCjak7Qom=M}RKOdX+aldU9+L4>4Wp9eg5fb>J_s}r(JkWbdBRA zt-q}`iG;m3%@(!ynbA~@m-24olqH$pb-^X6d+twSGrg z@k+jbS=7}%KBttn%d%#e-$dW80n>E_ddJY48Ck43=Bn@LT($E&e^>ro8Z>6_LI91(jIHpom zv9-+c%22?b(BpPUd$!FkZ3do#DCpa&h+El~8*?`l$J^qvw_A%l zdy?TwP1GZyca0i1*!hy^jM;Tq~gX{dM z9{)#s%B_SQL3ZCnEQ})-)+H zsJU^P-rw+>+X7D>wgj_Z4Igzy+oBHVnilW2*ro2r6)!@5k{8O?4dVr?ti7 z{n`~Y+18Y}JGQk#Ha~r&BR{QiIhLO`)lp}ApSK3@CCiP6eIx65e{lY=n6GVkm^%|k zb<|ds)M>i*uWP#?_sdnAlx?}$)p`fJrgmcs``Wuqw*Jo89??;o)P2B4)F$1Ifzr4T z>w3$dPE*yVjB6fMS4EL`J?Gh6byoxxuRuBw@i$!?1v)4Z2+0aw-YE6`BinWwW422K z=qf@!e=|5R3-9l6(uV-y86TgpMPzQs!rZnMODGumTh%3B+lN2 zl170*$arZ@K0nL$Ru}H5W;+FInmxm%GTGQN${3r%Z7m)^*)*4kBcs-k zlsA@=yT+=K{FUTALn#vaI*weP~B0NLCNpw;B^Mm(8dP#}Tt=LC!$fKYULn#L&$WWOzdR@g-@yMp&{Y4ZR7 z-l0*?@NW!#3%|et1b_e#00KY&2mk>f00e*l5C8)Ip9xg`-X72Xx%X1>Rgd>vpT~3g zeLwwnaFdRG<)=Tn;(39N{r$+N-v77%7>a%A<-{Mp_GTzXc`+^u|90bFOZW%RefjC3 ze+k97>;L_ur};)9tCD|Mdgimg_@_|pR`}`P{4Ig5SLQ6AP*i;jEeqwc5Uz&0ux2x( zb+%XBRG`QpERMx+h*iXjJ{`2>~7^+wLmG8X!)1Ug|Q0z}W_qkWOb-G@;v%VXm zEQ|R)lu>r%uIj@SfjClE_07xEb3;3Htna@b|D(r#hmMWJ{{1gf00e*l5IE@s&QjiS%Jo45uetvJv}gF!C%uE9 zKM(){KmZ5;0U!VbfB+Bx0zd!=0D%)nph)$MpAEnGxNE^qKH_hDErI#}J;wJ%ZRvXD z()jq!&dwOlt_ZoCLQx#UUl%a0Rm5B*Ix+s<`oGxlM!6jof8kVEkxP5z`~Ruo?|bkM z96$gF00AHX1b_e#00KY&2mk>f00e-*Nh0w6ZoGNW*U{dA@&EtBKk*E|GyFDwfddEt z0U!VbfB+Bx0zd!=00AHX1b_e#_*Ed_^?E&h-Y)V7e@jE3@%jJ$um}IZ0R(^m5C8%| z00;m9AOHk_fStf=tJEpaC!fGc*L`NPr;EG(p)hxY zd7Wb)`4P9|pR1*Nx$CcgfV**~i;MTX#>GFFtfj*X5C8%|;FnH-tp7b{e(B9ZgFpZX z00AHX1b_e#00KY&2mk>f00e-*FOdLQ{||fq!!!KVp$`vUA6OlDs9!n#-P5CeFZq7f zv)+BqdzpG0Xa8y))t7_5ku-kFXm{=LqU|1*%IC$BsEe-urm!C2nBwAbHigstus^Yit53rTkd!d5+{!^@;5AGP=sfxr9}CDV1Dl->6Yj zibYZ$T|vp^#WCZFl20NwvADnRKKwY<@!s2modDRb;AiBDQ!Z8Sia@Or3xV~Z>oy4L zDx=4*pai>a>W^ZHMHDhl&BtAGX#-9&h0++&Mh0g7nY4WExTkr)wSJV zQFl=s==Ud3nv9DFn1c)NH2}WCb`CIdF91gV4*}j?j6CQrek)+ifww=>iG|E3EFaH6 zW*wG&EIe%{fRHTixF*5Q1}Llc2XuU?GLQTG=`>Y+>WX3iSkvqoE|tl~ zmQm);77w6onoGoyQEN!bGpH=dyZEoLOa4mop7DNxZM;r-{RCnIv^IyD39gG?sbYweDA8sP-$V4Hd242PwF*lxJ&g7KR zc3HM1)D>x@85b|)N`}dc1xc24Ns*1xjSMZPRLmM1X$#3jI+Kdwix5;4c|9khY=T?M zvUVjHWK)1a&5hIa{`iP*M8b=Twc3DMd#PRf^951Rw^fnPkw>ueuA$VGV6m~*7k5{k zHO#hOYM3jE+O@dBUn#0sLs`c|j)u}~1~sXTuP>UDa_B9*5ZKyL(7*Z5WYJ3rk Smj4)IgCjFUf{MW<>Hh^=G)21r literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/hk-zones-wgs84.gpkg b/src/test/resources/com/conveyal/analysis/datasource/hk-zones-wgs84.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..cb8e7abac1a79df3d42aad9bf611011046f44497 GIT binary patch literal 98304 zcmeI5eQX=&dBE>POO&iHJBiJ7isRRG6!PYy^(|Tcm_7D6hZ_+ zc<}Gj_}6?ri$Cl!5AZul5We@d`oNUD@Exn&==}-55(fer^+4W|} zQs>*9%kHmq9By58pV@PwDlpSv#^Xuq{zm4#f6mvhDC9H$C-p2JR) zg*EeSRmiAIoFFQ^tU6_8;!Kcb5E~DNLQEX_ocZ;1_|P;n7mG6}98EBB7R92q<$MTB zASO6-4#i`aaDh-biW1B`GsB{IFq~lg!Rc6>rIF9D3rQkUQ`hr6^3~Yj^PyJ+DW6h# z1*IiXQRS2%swYvomU{J2Iwgvdimvl0&npU2mr^1c@=@s0IR;zmk205z+vHE9A?g%j zqO%+4d)<8(F~%c}Fy54;0+;7s;q%-|F@w!;LfRH$jfM)sRzFn1V{)$^8Px&re7$79v7y3yDKzRj5~w*8Ys!s09!a^>iYn>HoYPI*kTa!c^Vz*_&)67w+px)*&nbLOKdz;9 zZW#`mgcXN9B&^VAMOHW=Q|sM=AaZLp-qBLmYg1Ee^<;5st*qS_IklkNH)lBftn0?r z(?+MNhmHEydRHl?R3Vk;WIoF&>k9Ih1*uSHo9D_{(;mj;YrU-bqw5yoc6%l!$X5?n z8+?wJ3cM<>b7?8RQVnIHepB+;H8P*c)!YcU4C3<2*x0>0qndGcZ7wu2ch4 z%E{PUMu(>xU`=SN4M=OY#V=cQJXt=auE;z-%ZgYXU1piNU~=9Vx~O1VnP4C3cTOfzZmE* z1o|^<|GAU>k(2$|7k#u(lvaHdMO%Av8d-aAx?t(Jah9dSc8m+PjX~C$gEn&Pq<5`! zu9JJ(JQ3fP9yYyDZV2XK>!hpsqDJ7GMYR=dySBF4aFP1hyGd1p!!Zu%vF}K0sSOo} zgfV*5nT$2)QLTL^E73kU!KAOHk_01yBIKmZ5;0U!Vb9u)#t3HKS|K>OiB z5^qc6g&?0nc-JQ7CC*bRG<&*rXqP+3}UobI+wm8W~_M6#`W5j`$TivBYmqH12 zVpKCP!>dA?SxAKTeGp<|p_#-LuaW7c@kU89q91ZYlP5;KbYcNt4~&Ipt0Lng6DNj7 zCg|bG6UT?gCWePc#>PkJV3Iu-i-+08D_%{rm-bE!jtqLQU8|EbJ~BM6APY#a_O-_!E)U#o7Xn453 z4Wot)c}dI(>Pm(mz?Vz)4|oR^Zwcs5Q;;B{44)PXsXTty&a3E`372?Orl}N{cv%uM z+WMdD{X+u(zzYZf0U!VbfB+Bx0zd!=00AHX1b_e#_;nHx$WH1I)qTc|UoA_^ft!-Z zD}mLVa$>ank28urb$D=iC@^>;Fnod?oH#i+d2(pt_{8Mci&Izc{QX}{Ufwq~K1!ed z#>amFMjy?)v59L>sS51_|d)(_TqYxy+0@L54?Z?5C8%|00;m9AOHk_01yBI zKmZ5;fyaWteizkY-#LKw|6}1;s00uI0zd!=00AHX1b_e#00KY&2mk>Kf!g)I@&ErH z6TKf>#NZf00e*l5C8%|00;m9An?c%c!G3Mr2X>03$Fh^vhD=KfdCKy z0zd!=00AHX1b_e#00KY&2s9wj==%RpiQbf z00e*l5C8%|pcw)O_fQ8M_x{s10^tAun}G*1KmZ5;0U!VbfB+Bx0zd!=00AHX1Rf;< z_5c5$*!L*a0c-^VKmZ5;0U!VbfB+Bx0zd!=00AHX1n!l9cKsjL|MyA_f00e*l5C8%|00=x<1hnV>!TSHv@)=kQ1b_e#00KY&2mk>f00e*l5C8%|;9d#9 z_5XV%2J%1v2mk>f00e*l5C8%|00;m9AOHj&Jp$c5pAfw*9}zvDcm1sA`#rziMRvW} zvDEo?=d$}N9fw=rXa_3}WNqP>6{mpEJLn4j-Ck=3;RMB^PG3!m&A2KB0al z6pPmC@gZCSF~OO0C?30n6@r6o~O<&+?*C+$Y0Q=%xT=sJ(`yrLj=DJ7yIAB8TRW3c7^D0At!&FIr; zh&qLs=9PiS6F78agUi!_fp2XLpIw(D?u{p6_+{UBuCOq!GrO zl2qXG{40EZr?FY1p@Og+nNk*2Wp)X4=W%HUp_`D;}$cp&jBo+ui0#1V};q zPjcovkGsw0ADo_NkTVzgbtRmbD)5M9F0&{a!@tS-dD>FdaFk`V9=Vcd>#Z8)F7ZNc zNkui;`lw*^&h_&e6)XyCxP7j1ig1%hv$3RBp^8aH=6e0inOHQz;;uc)qT({Ass*$V z4@ZLWMRb8#v@)PAZIq%d#j?0qp}<*aX`@a!6m@E#P@I{=-FkF}Nf@~Cu%Pmys#s7{ z)aJ8$-JY>A^0uMU=@(AnYx;35t#eCnY7*Ak*ga~kGp)!9CuC}!QxHUMtp=`^x?Y=_ zTB|3EQ)^}ID06B-xo<2TZqjw*>S=>~)j>vmYn`JMQ>u{4b26XhlywF9%Ysy>v(0m5 ztm$@&$=9}3%^zL22)ElaF+sk1xY}N|`!DmVyw0Vi{7OMoboV>qWkvK%>~WiP&&-QENn%R#&J^vUnP|HSJ_tEfs6@Szm#U4>CXbt_CNJ6@ZxL>)WjdoN&Fwyx!X0g%YZF_>QpG9dh~S3n#>C!m z`@O3&#t`QjI!Xsy-JXE~vUH^ym{Ly0-ZJ_t-2iJsTWvsEvn_tvqT|W(DRo8W@mW^H z>fkWT%mtJ4#>hhj+uA(Ifh}=a&s%PH)+G1lGc9h9&qv-~t=7#}FY~}vTPHKi8faTo z2(M&PoYs`=GPIDvoxYG2c-hD>qLizw)C*+g>pkM4_hZ)%@;KS-z>TZuVdS_ zwd{tA)W_aUsu~=QaX^oKM_Nm5s5m5y(WA~>szHxx?GtH9&ZR`*CLWH=$f)$P)-2p` zBsqhJs=C8>hR+J3psR3%qciTP?)J}!4kAYM{AKrb_n)_2Y;AS@S<5w&Bx2o5T?5@z z=No%J+xt?-JKL}Rb*QC*r`;ZkA~(LIL$yD6(0;Vv2GVQK^ddjeK)q+Vj z7LMZb5ymY|BR68{2Xr zOPoH{RGgN%>?%%eIx1`*@CMucWV!gHyYCX-Wt(i&=F8g##`=Dzfm%;Vg=WnzaCtA` zL4~T#*t%Or_4k3xW>+_Y&plzV^)$zJhkMo{ayaL%S-(#PQZ71&aixOVgNcCoCv~fmFfGT;A+Y=6xC1Z}= zSgj_#b?(7tE!0$PX`i-Kqe^{;-JS~<$WlO`s5U;gEa6ScZL$wEYn!gj>(}4vvNNz! zrGwA9J!j9hm(FZ$T|>Ld2E7oDv&rB*V)bobZ$oh}N+!b55Yh$wLPlRHuIYz~&tf^hw#=h!iFv%k7%;@9xTrh?Zi#s*gP@JSmMeKA$dtXWsr>{Zbf;d0lsv9B zV&mXZGPFRrKEO|llV_jG{PNYSQ(s&^@K+nZb9L%B)o+t;%#Ba|bKxh$KMapgkuHpj z#J{ch*A#wk;U^{W-1yY_|GE0!c}pQv$@udpfB!$Rk~cqj{k3(RZz^+^k4mz-gyvI) zLMl-5w_wZ0`zw~cJoMG~e{|>S)OW^8mw$|vypulqVqkQ9>R*-b{?WTa>e|xv^?SHs1K1Kih z^FRK*xW!ZrqA5)L&6Qi1aHjAl()^EcVY85_f00e*l5C8%|00;m9AOHk_z+*t*3DQN8_Wu!c8SDQyiQYFKg9<_w zfB+Bx0zd!=00AHX1b_e#00KY&2t0BGGGrTdAaLV|eZj7M!(IO-jqzO`{eDqPs#-ib zFtED1dK_mLQ|aYYjz5lX7cig{`Sj56;K0$!S5r8>+^URkI2Dk@{JM7kf3o*|0{_4Z z2mk>f00e*l5C8%|00;m9AOHk_01$Y@2>4p@=0V5ydk6aW|9k(7=>4MiQ~U-mAOHk_ z01yBIKmZ5;0U!VbfB+Bx0zlw#AmDPjhz?ha_6Ogkp+o=w|IPq`f8YfKfB+Bx0zd!= z00AHX1b~2*z*iT@J;Wb8hm*AXn`tj{`tHTm>AR`r;9D<6f^U6YG?Iyy>AT~Hrti+B zgWvn65d7YM{H&brn!bDX-RZlt2ZL{oemD45^oephya53q00e&R1hn-(f$!)DFCYK} zfB+Bx0zd!=00AHX1b_e#00KbZ@g<zJM*F4OpuExWm5EDm@pZZ!Z z?M1;k77NZY^9;)%{E%#~U7;6UI>$tjl+PUZYWJV;qG*go(d7I*nr1@bs9AC@9*b;o zI$ceiK|%(dL9zMS0@t#6VbarF(d3RroYTx|wwl!WU;^+7Bqjn3MuX4G#S!eS} z;f?BCF%um*gQCnO!)6qW&LY2lXz&OpbQ-ZZO0Zm5VG2cc-I7mHI|YT|`YSfjRSPH` z+a}YmW4m*xNo*s#!!~lCvEeY%xoJ4Hkh;(CY*7}DCYU&ju#4i%!hCRs(Vo8?+cI2V z2+k*&MA^|=tkT%Ph#+3Jx|Y_gj-=M?iH6`}JwaQd;m3`E=Zx^b(WD=Tcjw1pryu7~ zxnVbK{FNZYZTA7R1Dh*xhq1Zus7~zmNRz~dc1LXJ|BBeXHGzjZh?l+ksC8?3TbCFQ zJ>IEDjMAn8^6kk^7^&1bWulo#pL!%}@l|0Dp35UrNwT+dUbYmN}+=Iu+`0X~v5ZRXX^r+jI78d+E&9)-|-NY|sng zIGYU4BX%3NdQmbFj)st~&o5+XR1m~9{5Q3x{T0M@{qs6?=1`mG;xUrY?>na4$m83H zaj6wm(vLa)#$=USjVauaBjKsi>N9T7v181$7jD{~J z8LJUAvKT-k`^hP41NFK4(s+h@w%j|G-^z1vF3YFX6`AMK(u$~Z_O8)>WKuDw_xGU4 z>@&BR=^GRJ^0ZiIF6Crwm8jxgL_rCLMK)9MbijTbKttMc0nhXE(kd@o+(cnnBh6>8 zk*_=pLcdWTkL36<$p9j>9L+ RIMJKFBh#-n8yc6S{ui9~j;;Uz literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-empty.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-empty.geojson new file mode 100644 index 000000000..e69de29bb diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-extra-attribute.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-extra-attribute.geojson new file mode 100644 index 000000000..5d3ede896 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-extra-attribute.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hkzones-extra-attribute", +"description": "Some features have attributes that others do not.", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "Name": "Kowloon", "Count": 5, "ExtraAttribute": "This attribute exists only in Kowloon." }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.159122921974657, 22.32164790382194 ], [ 114.168719395665576, 22.296045686522024 ], [ 114.190255668018438, 22.302549920319244 ], [ 114.190255668018438, 22.322473703634543 ], [ 114.159122921974657, 22.32164790382194 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "Kennedy Town", "Count": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.105703481652796, 22.233478265431106 ], [ 114.122266697678228, 22.203817018165608 ], [ 114.128240316572658, 22.205199673266289 ], [ 114.137472273045859, 22.216511794259716 ], [ 114.119687180428357, 22.241143923358219 ], [ 114.105703481652796, 22.233478265431106 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "North Lamma", "Count": 45, "OtherExtraAttribute": 1234 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.124474544327114, 22.282936206858849 ], [ 114.12956699448219, 22.275726416719557 ], [ 114.132978936086076, 22.27709300679836 ], [ 114.132418766569032, 22.28811948640017 ], [ 114.124474544327114, 22.282936206858849 ] ] ] } } +] +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-mercator.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-mercator.geojson new file mode 100644 index 000000000..047325a8f --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-mercator.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hkzones-mercator", +"description": "This file is projected into spherical web Mercator. GeoJSON CRS entries are obsolete, it is suppposed to always be in WGS84.", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::3857" } }, +"features": [ +{ "type": "Feature", "properties": { "Name": "Kowloon", "Count": 5 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 12708135.433080945163965, 2550186.97031799983233 ], [ 12709203.707645628601313, 2547106.364596586674452 ], [ 12711601.114517534151673, 2547888.937824204098433 ], [ 12711601.114517534151673, 2550286.344696110114455 ], [ 12708135.433080945163965, 2550186.97031799983233 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "Kennedy Town", "Count": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 12702188.80818585306406, 2539580.266773413866758 ], [ 12704032.616959704086185, 2536013.554719078820199 ], [ 12704697.597173223271966, 2536179.799772459082305 ], [ 12705725.293866846710443, 2537539.98657284071669 ], [ 12703745.466412955895066, 2540502.17116033937782 ], [ 12702188.80818585306406, 2539580.266773413866758 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "North Lamma", "Count": 45 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 12704278.393324406817555, 2545529.175306653603911 ], [ 12704845.282282559201121, 2544661.835200681351125 ], [ 12705225.097884520888329, 2544826.232998545281589 ], [ 12705162.740099124610424, 2546152.753160620573908 ], [ 12704278.393324406817555, 2545529.175306653603911 ] ] ] } } +] +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-geometries.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-geometries.geojson new file mode 100644 index 000000000..5cef126bd --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-geometries.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hkzones-mixed-geometries", +"description": "Each feature in this file is of a different geometry type. We only accept feature collections where all features have the same type.", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "Name": "Kowloon", "Count": 5 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.159122921974657, 22.32164790382194 ], [ 114.168719395665576, 22.296045686522024 ], [ 114.190255668018438, 22.302549920319244 ], [ 114.190255668018438, 22.322473703634543 ], [ 114.159122921974657, 22.32164790382194 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "Kennedy Town", "Count": 2 }, "geometry": { "type": "LineString", "coordinates": [ [ 114.105703481652796, 22.233478265431106 ], [ 114.122266697678228, 22.203817018165608 ], [ 114.128240316572658, 22.205199673266289 ], [ 114.137472273045859, 22.216511794259716 ], [ 114.119687180428357, 22.241143923358219 ], [ 114.105703481652796, 22.233478265431106 ] ] } }, +{ "type": "Feature", "properties": { "Name": "North Lamma", "Count": 45 }, "geometry": { "type": "Point", "coordinates": [ 114.124474544327114, 22.282936206858849 ] } } +] +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-numeric.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-numeric.geojson new file mode 100644 index 000000000..729139897 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-mixed-numeric.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hkzones-mixed-numeric", +"description": "Each feature's Count field has a different apparent numeric type.", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "Name": "Kowloon", "Count": -5 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.159122921974657, 22.32164790382194 ], [ 114.168719395665576, 22.296045686522024 ], [ 114.190255668018438, 22.302549920319244 ], [ 114.190255668018438, 22.322473703634543 ], [ 114.159122921974657, 22.32164790382194 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "Kennedy Town", "Count": 2.345 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.105703481652796, 22.233478265431106 ], [ 114.122266697678228, 22.203817018165608 ], [ 114.128240316572658, 22.205199673266289 ], [ 114.137472273045859, 22.216511794259716 ], [ 114.119687180428357, 22.241143923358219 ], [ 114.105703481652796, 22.233478265431106 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "North Lamma", "Count": 6.02e23 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.124474544327114, 22.282936206858849 ], [ 114.12956699448219, 22.275726416719557 ], [ 114.132978936086076, 22.27709300679836 ], [ 114.132418766569032, 22.28811948640017 ], [ 114.124474544327114, 22.282936206858849 ] ] ] } } +] +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-no-collection.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-no-collection.geojson new file mode 100644 index 000000000..6a0af731d --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-no-collection.geojson @@ -0,0 +1,6 @@ +{ +"type": "Feature", +"description": "This file has no FeatureCollection, just a single unwrapped Feature.", +"properties": { "Name": "Kowloon", "Count": 5 }, +"geometry": { "type": "Polygon", "coordinates": [ [ [ 114.159122921974657, 22.32164790382194 ], [ 114.168719395665576, 22.296045686522024 ], [ 114.190255668018438, 22.302549920319244 ], [ 114.190255668018438, 22.322473703634543 ], [ 114.159122921974657, 22.32164790382194 ] ] ] } +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-too-small.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-too-small.geojson new file mode 100644 index 000000000..31ca215bc --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-too-small.geojson @@ -0,0 +1 @@ +{"type":"feature"} \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-type-mismatch.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-type-mismatch.geojson new file mode 100644 index 000000000..a7afee6e4 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-type-mismatch.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hkzones-type-mismatch", +"description": "Each feature has a value of a different type in the TypeMismatch property.", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "Name": "Kowloon", "Count": 5, "TypeMismatch": 1234 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.159122921974657, 22.32164790382194 ], [ 114.168719395665576, 22.296045686522024 ], [ 114.190255668018438, 22.302549920319244 ], [ 114.190255668018438, 22.322473703634543 ], [ 114.159122921974657, 22.32164790382194 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "Kennedy Town", "Count": 2, "TypeMismatch": "This is text." }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.105703481652796, 22.233478265431106 ], [ 114.122266697678228, 22.203817018165608 ], [ 114.128240316572658, 22.205199673266289 ], [ 114.137472273045859, 22.216511794259716 ], [ 114.119687180428357, 22.241143923358219 ], [ 114.105703481652796, 22.233478265431106 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "North Lamma", "Count": 45, "TypeMismatch": {"description": "This is an object."} }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.124474544327114, 22.282936206858849 ], [ 114.12956699448219, 22.275726416719557 ], [ 114.132978936086076, 22.27709300679836 ], [ 114.132418766569032, 22.28811948640017 ], [ 114.124474544327114, 22.282936206858849 ] ] ] } } +] +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-wgs84.geojson b/src/test/resources/com/conveyal/analysis/datasource/hkzones-wgs84.geojson new file mode 100644 index 000000000..3935903bb --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/hkzones-wgs84.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hkzones-wgs84", +"description": "This is the original file with no errors, in WGS84 coordinates.", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "Name": "Kowloon", "Count": 5 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.159122921974657, 22.32164790382194 ], [ 114.168719395665576, 22.296045686522024 ], [ 114.190255668018438, 22.302549920319244 ], [ 114.190255668018438, 22.322473703634543 ], [ 114.159122921974657, 22.32164790382194 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "Kennedy Town", "Count": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.105703481652796, 22.233478265431106 ], [ 114.122266697678228, 22.203817018165608 ], [ 114.128240316572658, 22.205199673266289 ], [ 114.137472273045859, 22.216511794259716 ], [ 114.119687180428357, 22.241143923358219 ], [ 114.105703481652796, 22.233478265431106 ] ] ] } }, +{ "type": "Feature", "properties": { "Name": "North Lamma", "Count": 45 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 114.124474544327114, 22.282936206858849 ], [ 114.12956699448219, 22.275726416719557 ], [ 114.132978936086076, 22.27709300679836 ], [ 114.132418766569032, 22.28811948640017 ], [ 114.124474544327114, 22.282936206858849 ] ] ] } } +] +} diff --git a/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.cpg b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.cpg new file mode 100644 index 000000000..3ad133c04 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.dbf b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.dbf new file mode 100644 index 0000000000000000000000000000000000000000..8586574897d046e9512c365611b6f8779d266933 GIT binary patch literal 286 zcmZXOF%E+;3;-jspjO?ODh5Bnj={A@x^+QpeM^5|ml!1~9<0;ll%)Ig{z5JUztkxs)c^nh literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.prj b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.prj new file mode 100644 index 000000000..4a3b7cad7 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.prj @@ -0,0 +1 @@ +PROJCS["Amersfoort_RD_New",GEOGCS["GCS_Amersfoort",DATUM["D_Amersfoort",SPHEROID["Bessel_1841",6377397.155,299.1528128]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Double_Stereographic"],PARAMETER["latitude_of_origin",52.15616055555555],PARAMETER["central_meridian",5.38763888888889],PARAMETER["scale_factor",0.9999079],PARAMETER["false_easting",155000],PARAMETER["false_northing",463000],UNIT["Meter",1]] \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.qpj b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.qpj new file mode 100644 index 000000000..ac9c05d13 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.qpj @@ -0,0 +1 @@ +PROJCS["Amersfoort / RD New",GEOGCS["Amersfoort",DATUM["Amersfoort",SPHEROID["Bessel 1841",6377397.155,299.1528128,AUTHORITY["EPSG","7004"]],TOWGS84[565.2369,50.0087,465.658,-0.406857,0.350733,-1.87035,4.0812],AUTHORITY["EPSG","6289"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4289"]],PROJECTION["Oblique_Stereographic"],PARAMETER["latitude_of_origin",52.15616055555555],PARAMETER["central_meridian",5.38763888888889],PARAMETER["scale_factor",0.9999079],PARAMETER["false_easting",155000],PARAMETER["false_northing",463000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH],AUTHORITY["EPSG","28992"]] diff --git a/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.shp b/src/test/resources/com/conveyal/analysis/datasource/nl-null-points.shp new file mode 100644 index 0000000000000000000000000000000000000000..5d0fc23980fb2d9483d6ff8200258e42070d0ea8 GIT binary patch literal 612 zcmZQzQ0HR63K)%EFf%YP0_8FSv@U%T`Rf2uCJ`m&2%;xg@i>Z^eP)+Lk1e0rSSQ}!v?yp0f!p$9P4ua)?E`aE>e-!>9GW4$l zlASCtIhgwx*B1O}x(C+B3X==>S};wz@~^{&i|+F5hdjaq9<7KIEtBlW|w#&3+98w@BkyA z9LOFADD4NOW1w^vl&*l%ZBTj&kY)nnSx|ZblwJm;LFR3N(ubh*1t@(FO22{9zaTUt M2b30r(kdVt0JaJh761SM literal 0 HcmV?d00001 From 851c2a5cbae3c48f1112e2acebc93caf1c7ca69d Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 30 Aug 2021 18:27:30 +0800 Subject: [PATCH 086/187] update datasource ingester tests --- .../datasource/DataSourceException.java | 13 ++ .../datasource/GeoJsonDataSourceIngester.java | 11 +- .../ShapefileDataSourceIngester.java | 11 +- .../conveyal/analysis/models/DataSource.java | 3 + .../java/com/conveyal/r5/analyst/Grid.java | 8 +- .../com/conveyal/r5/util/ShapefileReader.java | 5 +- .../GeoJsonDataSourceIngesterTest.java | 122 ++++++++++++++---- .../ShapefileDataSourceIngesterTest.java | 82 ++++++++++-- .../analysis/datasource/continents.cpg | 1 + .../analysis/datasource/continents.dbf | Bin 0 -> 923 bytes .../analysis/datasource/continents.geojson | 10 ++ .../analysis/datasource/continents.gpkg | Bin 0 -> 98304 bytes .../analysis/datasource/continents.prj | 1 + .../analysis/datasource/continents.shp | Bin 0 -> 1148 bytes .../analysis/datasource/continents.shx | Bin 0 -> 124 bytes .../analysis/datasource/duplicate-fields.cpg | 1 + .../analysis/datasource/duplicate-fields.dbf | Bin 0 -> 162 bytes .../analysis/datasource/duplicate-fields.prj | 1 + .../analysis/datasource/duplicate-fields.shp | Bin 0 -> 268 bytes .../analysis/datasource/duplicate-fields.shx | Bin 0 -> 108 bytes .../{hkzones-empty.geojson => empty.geojson} | 0 .../datasource/new-zealand-antimeridian.cpg | 1 + .../datasource/new-zealand-antimeridian.dbf | Bin 0 -> 648 bytes .../new-zealand-antimeridian.geojson | 9 ++ .../datasource/new-zealand-antimeridian.gpkg | Bin 0 -> 98304 bytes .../datasource/new-zealand-antimeridian.prj | 1 + .../datasource/new-zealand-antimeridian.shp | Bin 0 -> 596 bytes .../datasource/new-zealand-antimeridian.shx | Bin 0 -> 116 bytes ...es-too-small.geojson => too-small.geojson} | 0 29 files changed, 231 insertions(+), 49 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/datasource/DataSourceException.java create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.cpg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.dbf create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.gpkg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.prj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.shp create mode 100644 src/test/resources/com/conveyal/analysis/datasource/continents.shx create mode 100644 src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.cpg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.dbf create mode 100644 src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.prj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.shp create mode 100644 src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.shx rename src/test/resources/com/conveyal/analysis/datasource/{hkzones-empty.geojson => empty.geojson} (100%) create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.cpg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.dbf create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.geojson create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.gpkg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.prj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.shp create mode 100644 src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.shx rename src/test/resources/com/conveyal/analysis/datasource/{hkzones-too-small.geojson => too-small.geojson} (100%) diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceException.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceException.java new file mode 100644 index 000000000..31c9bbf4e --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceException.java @@ -0,0 +1,13 @@ +package com.conveyal.analysis.datasource; + +public class DataSourceException extends RuntimeException { + + public DataSourceException (String message) { + super(message); + } + + public DataSourceException (String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index a81dbf305..d0406ca6a 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -119,7 +119,7 @@ public void ingest (File file, ProgressListener progressListener) { throw new IllegalArgumentException("File does not exist: " + file.getPath()); } if (file.length() < MIN_GEOJSON_FILE_LENGTH) { - throw new IllegalArgumentException("File is too short to be GeoJSON, length is: " + file.length()); + throw new DataSourceException("File is too short to be GeoJSON, length is: " + file.length()); } try { // Note that most of this logic is identical to Shapefile and GeoPackage, extract common code. @@ -165,9 +165,10 @@ public void ingest (File file, ProgressListener progressListener) { // Cannot set from FeatureType because it's always Geometry for GeoJson. dataSource.geometryType = ShapefileReader.GeometryType.forBindingClass(firstGeometryType); dataSource.featureCount = featureCollection.size(); - } catch (Throwable t) { + } catch (FactoryException | IOException e) { // Unexpected errors cause immediate failure; predictable issues will be recorded on the DataSource object. - throw new RuntimeException("Error parsing GeoJSON. Please ensure the files you uploaded are valid.", t); + // Catch only checked exceptions to preserve the top-level exception type when possible. + throw new DataSourceException("Error parsing GeoJSON. Please ensure the files you uploaded are valid."); } } @@ -178,8 +179,8 @@ public void ingest (File file, ProgressListener progressListener) { private static void checkCrs (FeatureType featureType) throws FactoryException { CoordinateReferenceSystem crs = featureType.getCoordinateReferenceSystem(); if (crs != null && !DefaultGeographicCRS.WGS84.equals(crs) && !CRS.decode("CRS:84").equals(crs)) { - throw new RuntimeException("GeoJSON should specify no coordinate reference system, and contain unprojected " + - "WGS84 coordinates. CRS is: " + crs.toString()); + throw new DataSourceException("GeoJSON should specify no coordinate reference system, and contain " + + "unprojected WGS84 coordinates. CRS is: " + crs.toString()); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index d158a01cd..85654c4ae 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -12,6 +12,7 @@ import org.opengis.referencing.operation.TransformException; import java.io.File; +import java.io.IOException; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; import static com.google.common.base.Preconditions.checkState; @@ -46,15 +47,17 @@ public void ingest (File file, ProgressListener progressListener) { reader.wgs84Stream().forEach(f -> { checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); }); + reader.close(); dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); dataSource.attributes = reader.attributes(); dataSource.geometryType = reader.geometryType(); dataSource.featureCount = reader.featureCount(); } catch (FactoryException | TransformException e) { - throw new RuntimeException("Shapefile transform error. Try uploading an unprojected (EPSG:4326) file.", e); - } catch (Exception e) { - // Must catch because ShapefileReader throws a checked IOException. - throw new RuntimeException("Error parsing shapefile. Ensure the files you uploaded are valid.", e); + throw new DataSourceException("Shapefile transform error. " + + "Try uploading an unprojected WGS84 (EPSG:4326) file.", e); + } catch (IOException e) { + // ShapefileReader throws a checked IOException. + throw new DataSourceException("Error parsing shapefile. Ensure the files you uploaded are valid.", e); } } diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java index 529aa44db..68bcbe5c1 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSource.java +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -38,6 +38,9 @@ public abstract class DataSource extends BaseModel { */ public String originalFileName; + /** The size of the uploaded file, including any sidecar files. */ + public int fileSizeBytes; + public FileStorageFormat fileFormat; // This type uses (north, south, east, west), ideally we'd use (minLon, minLat, maxLon, maxLat). diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 362d8a3ce..70c260be6 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -1,5 +1,6 @@ package com.conveyal.r5.analyst; +import com.conveyal.analysis.datasource.DataSourceException; import com.conveyal.r5.common.GeometryUtils; import com.conveyal.r5.util.InputStreamProvider; import com.conveyal.r5.util.ProgressListener; @@ -799,15 +800,16 @@ public static double roughWgsEnvelopeArea (Envelope wgsEnvelope) { */ public static void checkWgsEnvelopeSize (Envelope envelope) { if (roughWgsEnvelopeArea(envelope) > MAX_BOUNDING_BOX_AREA_SQ_KM) { - throw new IllegalArgumentException("Shapefile extent (" + roughWgsEnvelopeArea(envelope) + " sq. km.) " + - "exceeds limit (" + MAX_BOUNDING_BOX_AREA_SQ_KM + "sq. km.)."); + throw new DataSourceException(String.format("Geographic extent of spatial layer (%.0f km2) exceeds limit of %.0f km2.", + roughWgsEnvelopeArea(envelope), MAX_BOUNDING_BOX_AREA_SQ_KM)); + } } public static void checkPixelCount (WebMercatorExtents extents, int layers) { int pixels = extents.width * extents.height * layers; if (pixels > MAX_PIXELS) { - throw new IllegalArgumentException("Number of zoom level " + extents.zoom + " pixels (" + pixels + ")" + + throw new DataSourceException("Number of zoom level " + extents.zoom + " pixels (" + pixels + ")" + "exceeds limit (" + MAX_PIXELS +"). Reduce the zoom level or the file's extents or number of " + "numeric attributes."); } diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 6769b5884..c674a3e15 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -1,6 +1,7 @@ package com.conveyal.r5.util; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.datasource.DataSourceException; import com.conveyal.analysis.datasource.SpatialAttribute; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; @@ -119,7 +120,7 @@ public Stream wgs84Stream () throws IOException, TransformExcepti return stream().map(f -> { org.locationtech.jts.geom.Geometry g = (org.locationtech.jts.geom.Geometry) f.getDefaultGeometry(); if (g == null) { - throw new RuntimeException("Null geometry on feature: " + f.getID()); + throw new DataSourceException("Null (missing) geometry on feature: " + f.getID()); } try { // TODO does this leak beyond this function? @@ -164,7 +165,7 @@ public static List attributes (SimpleFeatureType schema) { } }); if (attributes.size() != uniqueAttributes.size()) { - throw new AnalysisServerException("Shapefile has duplicate attributes."); + throw new DataSourceException("Spatial layer has attributes with duplicate names."); } return attributes; } diff --git a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java index 78fcab849..6dd398c1e 100644 --- a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java @@ -1,11 +1,12 @@ package com.conveyal.analysis.datasource; -import com.conveyal.r5.analyst.progress.NoopProgressListener; +import com.conveyal.analysis.models.SpatialDataSource; import org.junit.jupiter.api.Test; import java.io.File; import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -14,27 +15,98 @@ */ class GeoJsonDataSourceIngesterTest { - String[] inputs = new String[] { -// "/Users/abyrd/geodata/test-ingest/hkzones-too-small.geojson", -// "/Users/abyrd/geodata/test-ingest/hkzones-empty.geojson", - "/Users/abyrd/geodata/test-ingest/hkzones-type-mismatch.geojson", - "/Users/abyrd/geodata/test-ingest/hkzones-extra-attribute.geojson", - "/Users/abyrd/geodata/test-ingest/hkzones-mixed-numeric.geojson", - "/Users/abyrd/geodata/test-ingest/hkzones-mixed-geometries.geojson", - "/Users/abyrd/geodata/test-ingest/hkzones-mercator.geojson", - "/Users/abyrd/geodata/test-ingest/hkzones-wgs84.geojson", - }; - - @Test - void testGeoJsonProcessing () { - for (String input : inputs) { - TestingProgressListener progressListener = new TestingProgressListener(); - DataSourceIngester ingester = DataSourceIngester.forFormat(GEOJSON); - File geoJsonInputFile = new File(input); - ingester.ingest(geoJsonInputFile, progressListener); - ingester.toString(); - // TODO progressListener.assertUsedCorrectly(); - } - } - -} \ No newline at end of file + @Test + void basicValidGeoJson () { + SpatialDataSource spatialDataSource = ingest("hkzones-wgs84"); + } + + @Test + void typeMismatch () { + SpatialDataSource spatialDataSource = ingest("hkzones-type-mismatch"); + } + + @Test + void extraAttribute () { + SpatialDataSource spatialDataSource = ingest("hkzones-extra-attribute"); + } + + @Test + void mixedNumeric () { + SpatialDataSource spatialDataSource = ingest("hkzones-mixed-numeric"); + } + + @Test + void mixedGeometries () { + SpatialDataSource spatialDataSource = ingest("hkzones-mixed-geometries"); + } + + @Test + void mercatorBadProjection () { + SpatialDataSource spatialDataSource = ingest("hkzones-mercator"); + } + + // TODO span antimeridian, giant input geometry + + @Test + void fileEmpty () { + assertThrows( + DataSourceException.class, + () -> ingest("empty"), + "Expected exception on empty input file." + ); + } + + @Test + void fileTooSmall () { + assertThrows( + DataSourceException.class, + () -> ingest("too-small"), + "Expected exception on input file too short to be GeoJSON." + ); + } + + /** + * Test on a GeoJSON file containing huge shapes: the continents of Africa, South America, and Australia. + */ + @Test + void continentalScale () { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest("continents"), + "Expected exception on continental-scale GeoJSON." + ); + assertTrue(throwable.getMessage().contains("exceeds")); + } + + /** + * Test on WGS84 GeoJSON containing shapes on both sides of the 180 degree antimeridian. + * This case was encountered in the wild: the North Island and the Chatham islands, both part of New Zealand. + */ + @Test + void newZealandAntimeridian () { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest("new-zealand-antimeridian"), + "Expected exception on shapefile crossing antimeridian." + ); + // TODO generate message specifically about 180 degree meridian, not excessive bbox size + assertTrue(throwable.getMessage().contains("exceeds")); + } + + private SpatialDataSource ingest (String inputFile) { + TestingProgressListener progressListener = new TestingProgressListener(); + DataSourceIngester ingester = DataSourceIngester.forFormat(GEOJSON); + File geoJsonInputFile = getResourceAsFile(inputFile + ".geojson"); + ingester.ingest(geoJsonInputFile, progressListener); + // TODO progressListener.assertUsedCorrectly(); + return ((SpatialDataSource) ingester.dataSource()); + } + + // Method is non-static since resource resolution is relative to the package of the current class. + // In a static context, you can also do XYZTest.class.getResource(). + private File getResourceAsFile (String resource) { + // This just removes the protocol and query parameter part of the URL, which for File URLs is a file path. + return new File(getClass().getResource(resource).getFile()); + } + +} diff --git a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java index 5de16b59e..406f0d3bc 100644 --- a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java @@ -1,30 +1,92 @@ package com.conveyal.analysis.datasource; -import org.junit.jupiter.api.Assertions; +import com.conveyal.analysis.models.SpatialDataSource; import org.junit.jupiter.api.Test; import java.io.File; -import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** - * + * Test that we can correctly read Shapefiles with many different characteristics, and can detect problematic inputs + * including many that we've encountered in practice. */ class ShapefileDataSourceIngesterTest { + /** + * Test on a shapefile that has mostly geometries of type "point" but some geometries of type "null", which is to + * say that some of the records in the shapefile have missing geometries. The GeoTools shapefile reader will not + * tolerate geometries of mixed types, but will tolerate inputs like this silently. We want to detect and refuse. + */ @Test - void testNullPoints () { - TestingProgressListener progressListener = new TestingProgressListener(); - DataSourceIngester ingester = DataSourceIngester.forFormat(SHP); - File inputFile = new File("/Users/abyrd/geodata/test-ingest/nl-null-points.shp"); - Throwable thrown = assertThrows( - RuntimeException.class, - () -> ingester.ingest(inputFile, progressListener), + void nullPointGeometries () { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest("nl-null-points"), "Expected exception on shapefile with null geometries." ); + assertTrue(throwable.getMessage().contains("missing")); + } + + /** + * Test on a shapefile that has two attributes with the same name (one text and one integer). This is actually + * possible and has been encountered in the wild, probably due to names being truncated to a fixed length (the + * DBF file used in shapefiles allows field names of at most ten characters). Apparently this has been "fixed" in + * Geotools which now silently renames one of the columns with a numeric suffix. It would be preferable to just + * refuse this kind of input, since the fields are likely to have different names when opened in different software. + */ + @Test + void duplicateAttributeNames () { + SpatialDataSource spatialDataSource = ingest("duplicate-fields"); + // id, the_geom, DDDDDDDDDD, and DDDDDDDDDD. The final one will be renamed on the fly to DDDDDDDDDD1. + assertTrue(spatialDataSource.attributes.size() == 4); + assertTrue(spatialDataSource.attributes.get(3).name.endsWith("1")); + } + + /** + * Test on a shapefile containing huge shapes, + * in this case the continents of Africa, South America, and Australia. + */ + @Test + void continentalScale () { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest("continents"), + "Expected exception on continental-scale shapefile." + ); + assertTrue(throwable.getMessage().contains("exceeds")); + } + + /** + * Test on a shapefile containing shapes on both sides of the 180 degree antimeridian. + * This case was encountered in the wild: the North Island and the Chatham islands, both part of New Zealand. + */ + @Test + void newZealandAntimeridian () { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest("new-zealand-antimeridian"), + "Expected exception on shapefile crossing antimeridian." + ); + // TODO generate message specifically about 180 degree meridian, not excessive bbox size + assertTrue(throwable.getMessage().contains("exceeds")); + } + + private SpatialDataSource ingest (String shpBaseName) { + TestingProgressListener progressListener = new TestingProgressListener(); + DataSourceIngester ingester = DataSourceIngester.forFormat(SHP); + File geoJsonInputFile = getResourceAsFile(shpBaseName + ".shp"); + ingester.ingest(geoJsonInputFile, progressListener); // TODO progressListener.assertUsedCorrectly(); + return ((SpatialDataSource) ingester.dataSource()); + } + + /** Method is non-static since resource resolution is relative to the package of the current class. */ + private File getResourceAsFile (String resource) { + // This just removes the protocol and query parameter part of the URL, which for File URLs leaves a file path. + return new File(getClass().getResource(resource).getFile()); } } \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/continents.cpg b/src/test/resources/com/conveyal/analysis/datasource/continents.cpg new file mode 100644 index 000000000..3ad133c04 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/continents.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/continents.dbf b/src/test/resources/com/conveyal/analysis/datasource/continents.dbf new file mode 100644 index 0000000000000000000000000000000000000000..bf28e82c83b0cd793f9352ea1ebff1cafc1c8db2 GIT binary patch literal 923 zcmZRsjO5M~5Y7$7Y(1t#VPVu-+aP&zL$HxN1_L8RDYN3kv0dL%J{ohZ^+o<cgGLjnct}9)#6rSgAe;yh1pY`6 z%ki)JvfvNp>H&T?5QO!4>9}vGzfY5{{kV|Wu;Z5%IHj!NT`7He!(STStu@r%Sv^+s zSj{2RwbiRDCQRGQHxL;ckKIaHb_bfE62sT$8T_mK~Xp~7slZT?Tz{VIF8!g37 zs&5l)L>Qx4j%QMWUZ&eexdIdl_}qPcln+_;`Nit6qMcN)*GD0@$4~hJ$m=OAXGK^7 zQLgS?$miXQ3-q}?$WIMW-2votx&4&QwbSbhkjSd&g%TVRLI-0EvKHuIwW7mpA{G)D z9)%MeFQh^&Cu~8PR?5vo;Sk3q1ay!=F^1=nFc#vF!)ix+cTreVn}^!FAy0lAa@e;b z%F{EwyTP<#7-MWKh4Dxt5vOC!VJ1e8CnH!5J)|>2ET|!iFgVmB@lTARoXj}_Rdk#W zP>l-3aJJ^-)JShbtr?ix>J^w3bAUNdkIbDnUgf50~FHdBAZ+Z&4uC&3Q+q3$m7Mo!GQr%qp8~yproFk&CBa0rE)!6ZA0yVoSmvqu=z;|)p;%BrMfP8HAW_ImsQJfwI6D0zq$ zgg6@Vxd&ao5!6qOXc>^2E@me+#YStUJe+vCDZhdn4@UyS z3A_fi-M*u)((G?G5G)sACi!b(cqXMo<3d6{ruAl`9r~2|nadZL%pDzuV~S4te41wp z#&IdFW>$BQB(ym4T|x_ujmJzh~)l{1Icg)$sn0%p?C41B!B}^uB zXQ$!1)wvEIWfE~lNKMh$G`U3OPx zr8(F#D=zJMGyP7RWSUu7W-?o?hGP@Cw&k@;JunuwNzKwa+AIyitHls4b)|e6l8xY5 zgB@j=l#-!%FBxI@aEeU|EM5)EGq&1vtCfs}c!3V%Wo;Bska*1)a1Z9DNTdqd@alL} zVB?I<+BDMK6mM>d1e$hjX&T(pG$pAS|QUB)WeD{``PfR$@fej)`O<6%BFh5j-8Ko{=48wOP31BvKJPRTZ7SBg`nv zv2qnUcg*-5ZvH|kv~Osr!+(u+UnJ`BpZYO;052c_1b_e#00KY&2mk>f00e*l5IBDX zt|Cm^iKSJm<3YU9j6*vng77|0G!;sYv0>!*`8{3_HLm7mKWK&Ic#-AsH7$fUIhhp0 zg&Bn7M}dj&e_wr>wh~JXs}JSQ5)P&CwKx`EVauH4{cPpg5BL&@Dm2SUW|n+@w0b?U zwCpIGUbVN+k2Y_V)Qd0z8>WW*eG5-X(92}M{rO+q#a$GH(E&l5Wd3a zb@${%Hnn$dcC>erZC#r;v~_g0wY7I_YA0R6z%H-P9T?efk!rS(md@7pR?F2_7s=Vw z-nL1~k**|A_-uVChT$gI@R)^cEh4_TtJNWi<6WG zE`oPZ@s@yWVji&^iZEd|9*W_2NlRY7OgP4%40}E~#-tK#M2i0n4R;gx2VOt`2mk>f z00e*l5C8%|00;m9AOHk_!1*8$GSoD#YOLFC!ml#v3k-Pc2Y&}K^W_;nm(teS=4fu+ z+}z$8Xx+SJW9JsfrVZ_z99N2g{>YctK0=E9w?De&rk_+f!&_f@*>U2ebLY`Tul4`J zweZve+%7}II|TlL7Z3mfKmZ5;0U!VbfB+Bx0zd!=00AIy(GaLJHdg0$4j}%&XbcMt z0Rlh(2mk>f00e*l5C8%|00;m9AfO?TAO9=g|36JMoYsiIM<4(MfB+Bx0zd!=00AHX z1b_e#00KbZf+kRJFg6-;m;a4${r`eCCnygDfB+Bx0zd!=00AHX1b_e#00Kau1c74L z|6d>)UML|B$v^-I00AHX1b_e#00KY&2mk>f00e-*hnj%JU}{{oqO7VacfDVI{O6E& zV5HCMQSblv2GQ`whq@`C91s8kKmZ5;0U!VbfB+Bx0zd!=00AH{7Xr)58&_%W`=@RM z!1w>>0uN+>01yBIKmZ5;0U!VbfB+Bx0zd!=TxbLezyD7xywKVJ`T_wU00e*l5C8%| z00;m9AOHk_01yBIXG1``{txm0*$@L+AOHk_01yBIKmZ5;0U!VbfB+Bx0v8?u>G^*U z|6h1^1C4Sv076JA{E%k8IZuAN?AfJ9bBPn6)05IPuRkhMT3s}&t)6EWNx9^r>A@IorYa>AB; zCBh+&O9<#7gJKNNBVjDWA&1qD_U@vv<~9$tcSD}i+mOS)6;Ym^>7Ioq8SA1sVP!d< zNeOyvXNOg(Ba7AT@l(FQIbu}mJdYT6H<(roWAAJ%h4Dxt5vOC!VJ0?T-z=yhi!kGv ziBXi3IY*%L_e+fm#k{gL-zzoJ+fZx9C%4rrFe~N&bDkcVJ1_m=^2eNFXrB0X)K!}O z%?5(yBFrRzO^g*7Iy5dMD+ zn_LOah2jhfQ2PSNxV4 zh31)paa>BPnbn$-gtj&L7S*OiP(6Y<7M;f6RQestrsb6Y*MMYSjWy*B1|QjVgjXn;|L14 zJtI;l%J#Z=VUotuxUWeklUm7Zy-%w)iI+_h16h&0E;o)#hqYc3^tgwEc)gVCgn9C~ zi%RlbzTq{(bLyFTZwk}r?o00KO7qp7v-(ojC>f8S+lu3+wjHeiUmT@jOIrLDyuHb+&^VnUL zmF8f}thlu2&Gb8Ml4)jTnaOOm8jelm+LqTY^}txzCN)dzXtOj3M=~K=>Pq=CBpbnl zK0C@XDJ4VkUNXY);S`$`SR7Qy6Rz5HtCfs}c!3V%=pc&cHyq6b+=ICp4yl4R9Au0N zY@D%Kn?{}HwK+~=*O@mvSMz6GzRxUANwcAPUNKPW{2u{W|12@jn4A{DPp|CTg zr0yWi9P=4nvx93_d8K*KI;(|MD`Yx?dRWnAoO@Bl@ztW`IF7B7V%ce^bMq z%>%OU>(a^!T}3CM%pOJ7QYA)I?U+a=Qqd5{9>LR*>KWNlR-1)8P9hb-Q&rLFJHm{z z94l9$b4R`3+3TvFC8~*DqW+1xgQlaEw(=K@A1~WTJWur2-BRnVbK{+Wz zo^_S#^O@ZMC2XIsqZX?qOS52eH?u`>zFaj=*`hl^#kY4eYUlQMcU_{eHP4OhJRLPp z-4EG_ny1@2pd^U$6L19rUx7%kX*8J!2MuXue=sGa7%AFP*JW}<&IqX~IvGmgh1!|& z&H1ahGl^SlTmlC^iDFASQaPg}U{9No$?SF;(#jgVbhVQ7?77S4s-aZXtolizoISl_ zwaMJyZ%8-GE85cMP9^Xr<{7e=&Q&+LGK)=qE6Ps6YELh_++^Odqbj|9cI`^)l~HJq z+ZPDB22eoW7PgcWx1gZk?de0ZfQ^mFk>sQtG)jM2Zc6?YISmaZ7wlz`S>a;aSDF|oe@7d~Bn45rX16NJ&_b9&g}Y37 zTT~>;~S%ttwQMf2taBW$c%g^6MQR)BxjWtgZ_3_$2;5T>y z0U!VbfB+Bx0zd!=00AKIVI?r#Zmb}V?pfV8L>N!uhj<0v`sk0p>$}SN%n8TKuWSuF z zdYAaPyLH(o4;*lkbk)&gkM9sGy>-s_?g%+oacyPoo!#Qo*MDc~o#?^rNz{{mWBpy# zC+^!WPJZ^}%Fr(lI=4Rj=tGX@I>Z~oAE`O{JzPGcj2Ar<`}1c*&L^fUKlrEP4)Iz0 zBR~DopAI-5bN$ETD{k8$K5+MQdr!W1zC zQX?N(`{hV~Ks>eX%HP}?b&1tKxoOLzxA%*0{QBmfFMQGF%u-v7sXg?)->uy1_KVDK z>pfwr(McZu%0^Y;too5acFr`~$!-d!SB ze&7e<7dDG`l;0S>^|!ml7Y`lyeoco<{Pv4Cwlfc^zO?$&ulvMTm%aDwZ4c}c z)%KiN`{qL{1DnKuzxB0GIPV%39dA$2Km2=#xS06Y-M20Fi}wVamwjfG6jgYyQSU!s z{pMm(&&Qt_7sik)&ZO9IXl9|If;hhHQkDF_J@kwE)~~dRH+=df|D&hf&TEO4ccmY8 ziLPJtzqaV;&K&urnEdae5B`47M=VY?lR35jQ`h;Hi)+kZs%+csa(<;M`HA&`R`HSE zcmDWAcc)XWeEq3t!-6)u(|+wl`RyGR@jIvPz4IrAI%oUPBhDA;rOvN?=A|X;Zd&PF z=-%}FE67n@G=1)0tj;G!PI^DOwN2dJ<$m?aKecC-bROn*cQ3fY zdHleE&Z(z%h&P((M4AiV$imCd`H zOHr*qe*6-r+GxY8O&^bKTj#vtW#js5U#FZ8Klz)V9XrwF{L%g0-}i)eI&b+4`Of00e-* z1w|lYsBB!-d`)96SXck$UH(rQ<$rbL`$dfjLUKz>%f!UQ1}u^cg%5?I%m#eBfEGT< zgdJ_IEo*ZBtsf00e*l5C8%|00;m9 zAaH>ZuvXyBgX%Ny9mxOx-|#lk@Rx>n@f*B=01yBIKmZ5;0U!VbfB+Bx0zd!=0D+5v zfYE3qs*PpRAADnrYWe&BHO&P6ffo<}0zd!=00AHX1b_e#00LS9Hw+odi4$val5`_I z>18;1YOQ=jzm5&5IYqd2dMwwK}XR*=|CdLR%?q569r>OeeeSMS*) z6v9u>w&W_apuM{&4@$%$8!YPmCM?L~4Ioc&U;yo;`rID1WUtRVILqnes(ch;BWOGF z4)koWjIt5SRz!JvrZ=uLt?)Kh9W|b@wc|*i1fZXu#D?NzYUE&g!mvTmwPMpE0=P zt0Y!(kmdI2%}Yw!4~8ZSGn4`&r3w^eD0ZE0U#oNn!yN6}Hh;Ue>FwH!G#$O@V32i4 z+>g&=-^epB9z;Jdex!io7LI(3=fRY zRA;d%53DcxEPJI!Nx+^qBa_+fHl*!m&{C2vt)rB?Y_1wg!n5j!*Bv2ydc|s!xxe3# zZa#x%sl)^uDXp5Ey>za+$(?7*n&furvUP`x^Qh`Fkxj0F=0b4>1*m-iGMtEw z$2m^teUZAM$61wX6;UoG9h6+#n9ozI49u0oWOFp-(qdlPbv*H z9GGJT>eyK#Do_^wnI*K$I1x?B=a!YEw&FOH0skCuoS_00hdS2lrjm2Uv3_1ShAyp?y2Xg>)0u9?GuMnWT$s@v UUZVPe+@*DyHmN(1xTO9607$AA<^TWy literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/continents.prj b/src/test/resources/com/conveyal/analysis/datasource/continents.prj new file mode 100644 index 000000000..f45cbadf0 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/continents.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/continents.shp b/src/test/resources/com/conveyal/analysis/datasource/continents.shp new file mode 100644 index 0000000000000000000000000000000000000000..b01eb2280ba85df7503b1d33bb532466b71079cf GIT binary patch literal 1148 zcmZvbYfO`86vtmmrvd`X)nVtfKr5kLX`8|T2d8!r-jV>NotMeRTRQ0GfRNOP{Q!(a zE1=v9QyUobHXy`xgE0s?PY1%7OmsHtHV4S1U_b__yQi#8Y?Be51s&E{(AnzWULCr_mgM09^VV=rvLxZ#dbw| ztsgt?dquQIU@^fklp4iYUNb# z0G5}uzkW#yVUEij?{kZTi9dwKn@kb>bs=v@iw2;DozLTK1nAcnWw?6A!^lO+P>OOd z)cnI4x9SB@KH(Z(JeL5*zNxz>>O;ZNi_3KAN4a}g(0XL`QwBE){C94a=jnVvBbqc* zy%LZY=MaOvTd`teV}NQ}0$JRg;q8C&5RT?cCGK&cqo^qDwRn_9lhb|Ah2YTI&>kH6 z$@gsNJ!ncTiWT4%GV;NsnFIQu(xAYH!6;1Ygt1J2xN>IPk5$Qlu;M~`gNTU(mX>q3 zT|B|TF5IH_b{B*}`=)|Z;fe_COk8<(iHw45%b}{1>Dw?k@bU(MR19nR*|5@VW?XNk z7S`KSwQ4awepG#L%h+zP&%yV_96AP{F!qyj%%1T|Q6=nxOr3(^a#`k#EhPUXye>(R zW$LbLE%Sp#&HP&>l_ZMe4#;+N&&4}xH!qexKB@LY*3qe}oeL6dy2jUnCl?R7hAN{E zyos;g(_B14kHljIRSPRENtjQkhVjDd*hsHX)y_z9>_ckELnOxQW7WI@RT?(S?uPh1 zO~4!1h0?eJ8Jg+KK0|?0%&w|nc3nwx{@1n{l2*Pa9i#m{;!6a{Xyv}_RyUW9(*f2| zeQOdr^wfvUcQ6!@_;c;-DKN;8zg(Dk^J6~x5Het1uebpD& z^08sFZ4Oam=d4&Lx^*Kx1M-$EWl?g8b9Y|pk$sW49D;Vv-ZrxzN372(O>^gT6pk8n z=O1{y12F@M2SYZCFje`1WF$8ea=a~uc_tTQ-mQF;QbWPRXFl)mGP1F?$-LiWrJyOS zpZE0l-8dT7d1vG~1>NFbdbaCd!_=MqxqqKkLa&xN{FRQ6<*LITra>h*dO3F1#8Cc} z1+m7i4!&9B?Dgw#gIoRiH}KoC0qu%9%c*5UswY?*ZTa~EnlMU&J!C|{X^H|)=BDE( Zdg9Tz@AKbgS2IAuR#ai3Y literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.cpg b/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.cpg new file mode 100644 index 000000000..3ad133c04 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.dbf b/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.dbf new file mode 100644 index 0000000000000000000000000000000000000000..1a09bbbfe691ff6def4df6ac215371ffc70777b2 GIT binary patch literal 162 zcmZRs+4)i$0XO{vUZ@o==@2kmlHe65blDx Y(`7%8?4*Mlj>qMeUNb01_ygU10D6{9xc~qF literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.shx b/src/test/resources/com/conveyal/analysis/datasource/duplicate-fields.shx new file mode 100644 index 0000000000000000000000000000000000000000..8e5fa1a3e391a331467066958d18e9a5dd42fea2 GIT binary patch literal 108 zcmZQzQ0HR64$NLKGcd3M<(@Y4_j6`zI)2*Ap*?qxhNH|{)u=NSnvT|0v)e8l({RMD L9Yxd#$O`}fU^Wje literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-empty.geojson b/src/test/resources/com/conveyal/analysis/datasource/empty.geojson similarity index 100% rename from src/test/resources/com/conveyal/analysis/datasource/hkzones-empty.geojson rename to src/test/resources/com/conveyal/analysis/datasource/empty.geojson diff --git a/src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.cpg b/src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.cpg new file mode 100644 index 000000000..3ad133c04 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.dbf b/src/test/resources/com/conveyal/analysis/datasource/new-zealand-antimeridian.dbf new file mode 100644 index 0000000000000000000000000000000000000000..cff192237d17cfc78c91dad109effa2a90b5b50e GIT binary patch literal 648 zcmZRsjO5M~5Y7$7Y(1t#VPVu-+aP&zL$Hx1T0D96J+d?(h7a8Cx4Yl|_SjMAw5Jcdc`gL^S-l#3z{&s*kc=B5vIHhgi6+Qj7fma7!>Gkw} zq#i(#nUv_G|1A62kYdIqj2aL zqS?sSnE~(6S&Z>OGmM+ET;vME!$N^uEoHG9PDq!9*ib_iVR|8=^Otj|CUcEIO&zZW zs7(booa6O4ZPLflcm?M1eiLR#3ox(JB5UndY`O~bI@@DSF;{DB0pIp)uh%y^>Ur0U zF&CV6=DhB+439b$qmeTg1&vBLi7yH$MK7fg8^_=2Sd6r3%CIR~pZV3ix>?dJcSR8M zD;jnHj>3lbF(;&ZUbBKFaUD;ubxsvG1r&*=^$t~4GBY>aGiT#$GKF0Vn?j|FoTe4g zLV}s663ggbdfCo^v`w)PX)Bh)#fnAFMoaf@Oq6xj5JCxh4!aX}mQI?uu{+WPNmFg8 zL!py>oxbF#hY+Q#u&zE-z$+=ouWGV!%{hmO3p-Q#x9{BT^&L9odCb(wna`<0gCEz^ zdUiAiUBZr|-X!eMNL5ieG21x0MN#6`8%9S@-DphZ*WF}sYNKrZSmd>$`kD=fW1Nk; zargAqsW!r9e;Y?vE%BPj7dS=8aq5POf)&9~w5QW|{!q&!jLA2SvOXTYmk6)dH$Cln z_x{?1&kJ%<(3A}>BNtYSl4=aUGqJ02teqmw2b(l&cJmC?O0@T#w6QFANt!qb*ejrD74>gsV;LX1cDtD%1NoTj(zD>+t%<#ysWo zHN%wi3LY)f;Ta0p6FM6W(w=P_mmM5WPT;jwMZm4BO10G`LeEj@n7MR?sLoDbdTd8r z_I4}t&YtAmzP-)s3j{omt=0Ng*Dvd$y|GVLmfg^H=n%eJRQT%i!DEJYFF2J!6WQuPu?t5)9(I(DiBJG}>M0O8O7Hy6jXW1OKV_j&R42nJ-^p)dk zyld6D&g|;+%?EZIVQUmB6T!Odm}zglsA>3CQ9Xj~(qpTwc8{_4Zii~d;aCR@+jpe3 zHJXY;!dyLEZpNB;RJ%_kWhKu`;wD~>EX$~RS$h_qII@z(OO@;LofUGTBpOvX%&}s3 zxchtb(0*W`5C7cNvqAK?_x*YQ3Vs7GAOHk_01yBIKmZ5;0U!VbUV8$U0_B(P>53A! zw!No=VDF1Yre@F>VudwypTHM*DU0U$5{fEfmR?9kAG~KF5kE7VJn!F8)K5m~c(f+w zCvgCop2wwxHLg-G7Zv2JpotkSf@AO^7oM4#@sr5~{P-l!M9%ve9QP}d5rFUo6rP!w z@{@-rr;kh?IYLg%OkcP_>f5d~t~9C$g0v=PR{Z4n1rkTLsq_AR z#LUE@@XXBAWtzCbfLtuXo8$`e(uu6MkG_W6spXw2`Ct8b3*5;%t2%FdI&1 zzg1Ut$wIf{n0dqLzqmmf^{1 zY?en3P8^z^n3z0t*v-zS$0v1+E_QZ!a^kR_<7Uz>;zwr2!@Bsj*t%g6v5KYoNo(-U zWLe4^7VIZ0rqVEHrc8|smgJaa^mqpqZwVMftRhiDSs^19@eDR+q-s1SToF)(y;@ok z6j{vb@xN!_83KR73kU!KAOHk_01yBIKmZ5;0U!VbfB+D9Ll8*cG1zy)>t#GWgT}jd zZ+n}x$yffGM#XO@#wWs~<3~m($5Z1oNA<6XgHwm5@1stA=V^rdkWM{FF<<%MC-B=m zaodCU?8a|A1HUBj7rcM~5C8%|00;m9AOHk_01yBIKmZ5;ft!s$XZv6`-YaflB;u{{8f00e*l z5C8%|00;m9AaFAf@Os(@J^1l|8$AEN8HN-300e*l5C8%|00;m9AOHk_01yBIK%fbM zy668d69X?dk%MF)00e*l5C8%|00;m9AOHk_01yBIK;T9oFy!$L-Z9kH)rFt;TgQJc z#AD0RIBR|W-%p8wpWX;v0Udz=5C8%|00;m9AOHk_01yBIKmZ5;fmR3%>>9j7|J*-o zBLM#XzZGgA0|bBo5C8%|00;m9AOHk_01yBIK;Q-?VEg-jV$TiQ2T&Ub00AHX1b_e# z00KY&2mk>f00e*l5NL^j{`?=}|CUgJ5D)+YKmZ5;0U!VbfB+Bx0zd!=0D&8pfPVfT z#Q!(!@PXn$00;m9AOHk_01yBIKmZ5;0U!VbS|R|?|69TYLO=ir00AHX1b_e#00KY& z2mk>f00eGm0)72IBL>=jO!WV%_j~=%_5V(D?QKlT=YKLy}#r09c#OewEfug zBr)GYH~W9q_OYS#XxF8%ril5xpyW#z^PDCY1g^;Aq;>1pnwZsAxYbgY*951`Y=Wjz zG)g6yXp~N%fHU8%jsQAE&&3loN-sq8!tps&Y0zj1#o5Ms0tlBtG&Oq~CE|;?f+)kH zBpst?Qz$_(NjgZKiYHPe3YdB#Swb5BKtVu(2Au){^sp!waBozUktJ1Acu~@h)+>?W zB}vxM0|F`ts*1D~UP9qO2rZtbvF1URUOZT*^l=mp9YZu5*^2D(8d&$XgjJMOLD8Jp zt_-VLM-?l>Ch0`#8Znx6UPp{)2E0ROv2`A3hH+Dti(El?SSVbrZ8p?UMOd*+IfrU8 z*9i3L?b4=#n^lh2Tcu6Uf<}b=Up>~Pjcovulua&Po0X<$eD|RMkSoY z7X_4}mr{t0vLv)jb3R-$c`1UG?Zwy}v)47;pPqUx+G zHYV9d8qYPoF};_n-TLfmllZntXJ9B&H|17w>5Sb<(kyc}jqjIgldw)6Pf=Z7svmfp za7#1OX-ygK+HHyH?(|)l-qDt-gHo{wZo<_jb~9a9UX{6qI9up0@9XgT#>PD5^EJbi z^9mj<(_a}1*b_P%4bq-%86p3lgs9F=UwUjuT=sS= z^Uj{+-M+oe>k9-tkFC}ER@X1iXePzX*H$=W2?lYHk<-MY5DW}2j}8|{ zhqI~S(?^Hrj}GVV3y=XxUJHamq}`K~$nL?(qRnyREStl2tP72kLD8p!zH(fRcda_t znO&W}`M{1NY>h%?B3PFlGwrPxH4Wb?x(mm#U3x6L)$TFY-tAD$I2`MMVf&7>wnkHN zNSLdK%e_<+k81acq^#t5N!-NCk!2ZGFKf@j6Gv9Ec&Tz-zOzD3ltiNnhdDa!F4Wic zHKLoipXh(S?*Z?n&fu=^w!gD&iufjRf8RHHKhXF4y?cA!)6>3N>Grh{|JOte!kf2w zeS?FRXTgUK;IF-P`20AJwbrxh;OWT$m?RM}qVeY4k+=&s%K8yCm%=+}HMd~i z0O2P9EDB1(+CkfGjdkCLC^lk*+Q}5hsC2QUZ5V5FO*9m$JiV~$P2d=JAx6#8`Ylx) zxhUjE=crg3?@gIi>9=1bu76&XmYkK7NYCKc8#BxT%gqWjWSBe0<^BU+-^r7n@`!QY z?E!Isn&Go}8+(~9o#V_3{6R^^(U@%E{Tqat>adqA*L5Rgt7SGs?+rU*WL4O06U-J#)$fa|8Mz_Wx9xf5 z1y0KdNRRfnQhUw25O^pG4@DcEEt_lk^?N9)|NnnmAAbJ-Df|OoKmZ5;0U!VbfB+Bx z0zd!=00AIyGY}}xv~>_$=SHFnMEgAt62##T{pM$p3k-GX*%v?X!LKmX$|vrA-&@jY z>Q_IHJpZ;&rzuZ6?zIPhd-1m=-1)=j)_?ioG&T6Ct)I zpJS*T!~WSvaV3T0UtWIV?`oA=jZ921|2gvUH1)4bN2R}fCsS?AS^w+`j~_}F5u+A( zDXVUs^K=l8?Hjgqk4;~^>*?>Dq8@$Zso78cV2`((#4mn(lkC z?v}8X`Gqf?{X_Mk2=)96#lycHU83IpzTF>VZi`TFSeCF9xFaG;E>fav!!H-^gGE!CI zi_|SsPk)`JZvXYm|MS1)Me6f^^`jq-Jx5cYeBtd+J^I;2>c0KI{`$w?F-QH;-GP5< zdttHKi7%i1e&|n9OVr{gI}Wgq%u(;`ynCW=Wr_O7A07DP;s2wl2Y&GQ=SIess6Fp{ z>XmN{)6`Jkd&#fmmMHnjo^OBpTMCbVbI(WhsrudZLb<-PA-rkwp} zWkp-5kk#XV&%iGT`~@!{00e*l5C8%|00;m9AOHk_01yBIK;Y&g;Ptc*dhnmFwVCn% z6U4w1H&+LtH$VUg00AHX1b_e#00KY&2mk>f00eF*0v9};gLjNRbN~nI*1xf00e-*jY$A?;LU^X9rg~4|NkF&g&24h z|9}?|00KY&2mk>f00e*l5C8%|00;m9AaD~AXm4vLy4&#wJaij>|KBr8;4gRq0U!Vb zfB+Bx0zd!=00AIiC-C@!XBYAQyKz!mCE0Uk-}jz4GkxOtnP>i{|IAm3@k)BznKS?W z)-!D3g)^V|^h;+xOP#Y<1Mh(V5C8&i1Oj^ePwaUkXaYI_0U!VbfB+Bx0zd!=00AHX z1b_e#00OTU0X_a7XnU0y`0L*9_Wo(_n|uDCr)T$O_kQn3JKwYG+wEU!d$s?IecyP! zT8Dapz;z=~J{>U5c-~@~pYfa(3IaZNwxu(kOI6Nsf3O?`~xsvSo{BPPkFV=)v@=x0e^<`C=FDPUy5suE;P z1@oNg?U7-F$-DY4$7J$Km`uJ_Oz`>M29(y1_imj`oqgO8r-E#FdQJ zgmbj!Q0?4w$K#nN;VnI2nT zP)O{e=S6AB;lo3u?z`vt^~Madz;d$!4H>3?F83es`c9tolt-EzG|!cIMUb?{^X4lV zzPf|o{Y~ZkLGx&P*HiD!Hgthdxea-J$Buc*L(LS@#ax=vllNU#8C_~eRq*o6hsvG% zy}lDCJmvk(lu+a~G3%m*k$&rC)iA{TLF1Pz5nWG74wXH3dVS$=S9z?7B8@6k+CIl5 zQfVrNQq-v!?QV4GB*R9LQA1G7lBg(3>-Z2b`oU%@7j`7Psl0Z(*Ecfa*&a5!;K+7d4%U%#WTK!^11IrC0j21r z6q%6=t3}C{P*tRshOkpa`WjDWg`6mfnkY*~=~{-GkxOQcwY1qdn@lArJY`W{;Ixc@ z(kyc}P1}_qkxc;-)i)jrmBo;EXnwG3>ye7vs{G>2dsHqb@Y<>(a2a`3(m3q8T)fxi zidas}d82FI%U13f4UTeN!7WMnK3Y>zl1`zz=J9GLdxf|RpkU~MIUYNZwdptD-1gy3=JD#h^7V2t` zbj)AEeQU){7stGPGnBtRj&Ky{-Sq96s;i?kZ%?o)wJH0OjwI7Z3;y*fX^&2f^7`bnm};f_1jynLa*-PLjN z-ndP5Yr`E6ISQFXEv*&gAJD?44y3+6dHW_i2#%~~Js7}v`+&r`z1QK~fk#k(uvjt7nf zPAo2Taoo8#*R0w-!V%s7z+hnl;s8)Mn$7tC$tBhkBTjyOb=j%4BGeHSj_g1T0x-2~ zFSb|woEhqPDZ(mmb%UoPOf4)vjf^}WY0nRJ%&|EYx$p?WY>Adtso~o_9n(Dh7R?66 Yzwq*=RS9*Tja?Dtu(km4j>e#Yv|J3PcJsscWF8;=PAkq=L Qb`((~Aa4N>UjyO*04(DY1ONa4 literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/hkzones-too-small.geojson b/src/test/resources/com/conveyal/analysis/datasource/too-small.geojson similarity index 100% rename from src/test/resources/com/conveyal/analysis/datasource/hkzones-too-small.geojson rename to src/test/resources/com/conveyal/analysis/datasource/too-small.geojson From d5664b55eda6442b6010b57ecadd14d2924cf6a2 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 31 Aug 2021 11:27:32 +0800 Subject: [PATCH 087/187] Add immutable header to each GTFSController response --- .../analysis/controllers/GTFSController.java | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index cef74e227..21d17f12d 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -1,32 +1,57 @@ package com.conveyal.analysis.controllers; +import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.models.Bundle; +import com.conveyal.analysis.persistence.Persistence; import com.conveyal.gtfs.GTFSCache; import com.conveyal.gtfs.GTFSFeed; import com.conveyal.gtfs.model.Pattern; import com.conveyal.gtfs.model.Route; import com.conveyal.gtfs.model.Stop; -import com.conveyal.gtfs.model.StopTime; import com.conveyal.gtfs.model.Trip; +import com.mongodb.QueryBuilder; import org.mapdb.Fun; +import org.mongojack.DBCursor; import spark.Request; import spark.Response; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; import static com.conveyal.analysis.util.JsonUtil.toJson; +/** + * Controller for retrieving data from the GTFS cache. + * + * Each endpoint starts with it's `feedGroupId` and `feedId` for retrieving the feed from the cache. No database + * interaction is done. This assumes that if the user is logged in and retrieving the feed by the appropriate ID then + * they have access to it, without checking the access group. This setup will allow for putting this endpoint behind a + * CDN in the future. Everything retrieved is immutable. Once it's retrieved and stored in the CDN, it doesn't need to + * be pulled from the cache again. + */ public class GTFSController implements HttpController { private final GTFSCache gtfsCache; public GTFSController (GTFSCache gtfsCache) { this.gtfsCache = gtfsCache; } + /** + * Use the same Cache-Control header for each endpoint here. 2,592,000 seconds is one month. + * https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control + */ + private final String cacheControlImmutable = "public, max-age=2592000 immutable"; + + /** + * Extracted into a common method to allow turning off during development. + */ + private void addImmutableResponseHeader (Response res) { + res.header("Cache-Control", cacheControlImmutable); + } + private static class BaseIdentifier { public final String _id; public final String name; @@ -66,11 +91,13 @@ static String getRouteName (Route route) { } private RouteAPIResponse getRoute(Request req, Response res) { + addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); return new RouteAPIResponse(feed.routes.get(req.params("routeId"))); } private List getRoutes(Request req, Response res) { + addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); return feed.routes .values() @@ -102,6 +129,7 @@ static GeoJSONLineString serialize (com.vividsolutions.jts.geom.LineString geome } private List getPatternsForRoute (Request req, Response res) { + addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); final String routeId = req.params("routeId"); return feed.patterns @@ -124,10 +152,40 @@ static class StopAPIResponse extends BaseIdentifier { } private List getStops (Request req, Response res) { + addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); return feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); } + static class AllStopsAPIResponse { + public final String feedId; + public final List stops; + + AllStopsAPIResponse(GTFSFeed feed) { + this.feedId = feed.feedId; + this.stops = feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); + } + } + + private List getAllStops (Request req, Response res) { + addImmutableResponseHeader(res); + String feedGroupId = req.params("feedGroupId"); + DBCursor cursor = Persistence.bundles.find(QueryBuilder.start("feedGroupId").is(feedGroupId).get()); + if (!cursor.hasNext()) { + throw AnalysisServerException.notFound("Bundle could not be found for the given feed group ID."); + } + + List allStopsByFeed = new ArrayList<>(); + Bundle bundle = cursor.next(); + for (Bundle.FeedSummary feedSummary : bundle.feeds) { + String bundleScopedFeedId = Bundle.bundleScopeFeedId(feedSummary.feedId, feedGroupId); + GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); + allStopsByFeed.add(new AllStopsAPIResponse(feed)); + feed.close(); + } + return allStopsByFeed; + } + static class TripAPIResponse extends BaseIdentifier { public final String headsign; public final Integer startTime; @@ -139,8 +197,8 @@ static class TripAPIResponse extends BaseIdentifier { headsign = trip.trip_headsign; directionId = trip.direction_id; - Map.Entry st = feed.stop_times.ceilingEntry(new Fun.Tuple2(trip.trip_id, null)); - Map.Entry endStopTime = feed.stop_times.floorEntry(new Fun.Tuple2(trip.trip_id, Fun.HI)); + var st = feed.stop_times.ceilingEntry(new Fun.Tuple2(trip.trip_id, null)); + var endStopTime = feed.stop_times.floorEntry(new Fun.Tuple2(trip.trip_id, Fun.HI)); startTime = st != null ? st.getValue().departure_time : null; @@ -153,6 +211,7 @@ static class TripAPIResponse extends BaseIdentifier { } private List getTripsForRoute (Request req, Response res) { + addImmutableResponseHeader(res); final GTFSFeed feed = getFeedFromRequest(req); final String routeId = req.params("routeId"); return feed.trips @@ -165,6 +224,7 @@ private List getTripsForRoute (Request req, Response res) { @Override public void registerEndpoints (spark.Service sparkService) { + sparkService.get("/api/gtfs/:feedGroupId/stops", this::getAllStops, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes", this::getRoutes, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId", this::getRoute, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId/patterns", this::getPatternsForRoute, toJson); From f11aa1f596957273f735545002093f04598bd5b0 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 31 Aug 2021 11:28:20 +0800 Subject: [PATCH 088/187] Remove `ApiMain` initialization All of the gtfs/api code can now be removed. To be done in a subsequent PR or commit. --- src/main/java/com/conveyal/analysis/BackendMain.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/BackendMain.java b/src/main/java/com/conveyal/analysis/BackendMain.java index b3729682d..c20239711 100644 --- a/src/main/java/com/conveyal/analysis/BackendMain.java +++ b/src/main/java/com/conveyal/analysis/BackendMain.java @@ -46,7 +46,6 @@ private static void startServerInternal (BackendComponents components, TaskActio // TODO remove the static ApiMain abstraction layer. We do not use it anywhere but in handling GraphQL queries. // TODO we could move this to something like BackendComponents.initialize() Persistence.initializeStatically(components.config); - ApiMain.initialize(components.gtfsCache); PointSetCache.initializeStatically(components.fileStorage); // TODO handle this via components without explicit "if (offline)" From b82be6e11c851b692c557a689da587559819aacb Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 31 Aug 2021 11:28:41 +0800 Subject: [PATCH 089/187] Remove now unused Project model --- .../RegionalAnalysisController.java | 2 -- .../com/conveyal/analysis/models/Project.java | 22 ------------------- .../analysis/persistence/Persistence.java | 3 --- 3 files changed, 27 deletions(-) delete mode 100644 src/main/java/com/conveyal/analysis/models/Project.java diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index d0109c725..0c835207f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -7,12 +7,10 @@ import com.conveyal.analysis.models.AnalysisRequest; import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; -import com.conveyal.analysis.models.Project; import com.conveyal.analysis.models.RegionalAnalysis; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.results.CsvResultType; import com.conveyal.analysis.util.JsonUtil; -import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; diff --git a/src/main/java/com/conveyal/analysis/models/Project.java b/src/main/java/com/conveyal/analysis/models/Project.java deleted file mode 100644 index 184a3df30..000000000 --- a/src/main/java/com/conveyal/analysis/models/Project.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.conveyal.analysis.models; - -import com.conveyal.analysis.AnalysisServerException; - -/** - * Represents a TAUI project - */ -public class Project extends Model implements Cloneable { - public String regionId; - - public String bundleId; - - public AnalysisRequest analysisRequestSettings; - - public Project clone () { - try { - return (Project) super.clone(); - } catch (CloneNotSupportedException e) { - throw AnalysisServerException.unknown(e); - } - } -} diff --git a/src/main/java/com/conveyal/analysis/persistence/Persistence.java b/src/main/java/com/conveyal/analysis/persistence/Persistence.java index 71129c0a9..54a7655c9 100644 --- a/src/main/java/com/conveyal/analysis/persistence/Persistence.java +++ b/src/main/java/com/conveyal/analysis/persistence/Persistence.java @@ -6,7 +6,6 @@ import com.conveyal.analysis.models.Model; import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; -import com.conveyal.analysis.models.Project; import com.conveyal.analysis.models.Region; import com.conveyal.analysis.models.RegionalAnalysis; import com.conveyal.analysis.util.JsonUtil; @@ -36,7 +35,6 @@ public class Persistence { private static DB db; public static MongoMap modifications; - public static MongoMap projects; public static MongoMap bundles; public static MongoMap regions; public static MongoMap regionalAnalyses; @@ -55,7 +53,6 @@ public static void initializeStatically (AnalysisDB.Config config) { } db = mongo.getDB(config.databaseName()); modifications = getTable("modifications", Modification.class); - projects = getTable("projects", Project.class); bundles = getTable("bundles", Bundle.class); regions = getTable("regions", Region.class); regionalAnalyses = getTable("regional-analyses", RegionalAnalysis.class); From 8ea09db553fadf940218ecf72e022ff175761d4c Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 31 Aug 2021 17:37:25 +0800 Subject: [PATCH 090/187] update spatial DataSourceIngester tests parameterize tests for different formats increase code reuse between tests check range of envelopes to ensure valid wgs84 assert expected use of ProgressListeners wrap exceptions only for checked exceptions some tests are failing because DataSources contain unprojected envelopes --- .../datasource/GeoJsonDataSourceIngester.java | 21 +--- .../GeoPackageDataSourceIngester.java | 9 +- .../ShapefileDataSourceIngester.java | 5 +- .../conveyal/analysis/models/DataSource.java | 5 +- .../com/conveyal/file/FileStorageFormat.java | 3 +- .../java/com/conveyal/r5/analyst/Grid.java | 32 ++++- .../GeoJsonDataSourceIngesterTest.java | 73 ++--------- .../ShapefileDataSourceIngesterTest.java | 52 +------- .../SpatialDataSourceIngesterTest.java | 114 ++++++++++++++++++ .../datasource/TestingProgressListener.java | 20 ++- .../datasource/valid-polygon-projected.cpg | 1 + .../datasource/valid-polygon-projected.dbf | Bin 0 -> 985 bytes ...eojson => valid-polygon-projected.geojson} | 0 ...2326.gpkg => valid-polygon-projected.gpkg} | Bin 98304 -> 98304 bytes .../datasource/valid-polygon-projected.prj | 1 + .../datasource/valid-polygon-projected.shp | Bin 0 -> 524 bytes .../datasource/valid-polygon-projected.shx | Bin 0 -> 124 bytes .../datasource/valid-polygon-wgs84.cpg | 1 + .../datasource/valid-polygon-wgs84.dbf | Bin 0 -> 985 bytes ...84.geojson => valid-polygon-wgs84.geojson} | 0 ...es-wgs84.gpkg => valid-polygon-wgs84.gpkg} | Bin 98304 -> 98304 bytes .../datasource/valid-polygon-wgs84.prj | 1 + .../datasource/valid-polygon-wgs84.shp | Bin 0 -> 524 bytes .../datasource/valid-polygon-wgs84.shx | Bin 0 -> 124 bytes 24 files changed, 199 insertions(+), 139 deletions(-) create mode 100644 src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.cpg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.dbf rename src/test/resources/com/conveyal/analysis/datasource/{hkzones-mercator.geojson => valid-polygon-projected.geojson} (100%) rename src/test/resources/com/conveyal/analysis/datasource/{hk-zones-epsg2326.gpkg => valid-polygon-projected.gpkg} (99%) create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.prj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.shp create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.shx create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.cpg create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.dbf rename src/test/resources/com/conveyal/analysis/datasource/{hkzones-wgs84.geojson => valid-polygon-wgs84.geojson} (100%) rename src/test/resources/com/conveyal/analysis/datasource/{hk-zones-wgs84.gpkg => valid-polygon-wgs84.gpkg} (99%) create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.prj create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.shp create mode 100644 src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.shx diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index d0406ca6a..e47319ab8 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -2,16 +2,11 @@ import com.conveyal.analysis.models.Bounds; import com.conveyal.analysis.models.DataSource; -import com.conveyal.analysis.models.DataSourceValidationIssue; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.file.FileStorageFormat; -import com.conveyal.geojson.GeoJsonModule; -import com.conveyal.r5.analyst.progress.ProgressInputStream; import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader; -import org.geotools.data.FeatureReader; import org.geotools.data.geojson.GeoJSONDataStore; -import org.geotools.data.geojson.GeoJSONFeatureSource; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; @@ -20,28 +15,19 @@ import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.index.strtree.STRtree; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.AttributeDescriptor; -import org.opengis.feature.type.AttributeType; import org.opengis.feature.type.FeatureType; import org.opengis.referencing.FactoryException; import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.operation.TransformException; -import java.io.BufferedInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; import static com.conveyal.analysis.models.DataSourceValidationIssue.Level.ERROR; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; -import static com.conveyal.r5.util.ShapefileReader.geometryType; /** * Logic to create SpatialDataSource metadata from an uploaded GeoJSON file and perform validation. @@ -86,7 +72,7 @@ * Allow Attributes to be of "AMBIGUOUS" or null type, or just drop them if they're ambiguous. * Flag them as hasMissingValues, or the quantity of missing values. * - * Be careful, QGIS will happily export GeoJSON with a CRS property which is no longer considered valid: + * Be careful, QGIS will happily export GeoJSON with a CRS property which is no longer allowed by the GeoJSON spec: * "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::3857" } } * If a CRS is present, make sure it matches one of the names for WGS84. Throw a warning if the field is present at all. * @@ -113,6 +99,8 @@ public GeoJsonDataSourceIngester () { @Override public void ingest (File file, ProgressListener progressListener) { + progressListener.beginTask("Processing and validating uploaded GeoJSON", 2); + progressListener.setWorkProduct(dataSource.toWorkProduct()); // Check that file exists and is not empty. // Geotools GeoJson reader fails with stack overflow on empty/missing file. TODO: File GeoTools issue. if (!file.exists()) { @@ -155,8 +143,10 @@ public void ingest (File file, ProgressListener progressListener) { continue; } } + progressListener.increment(); checkCrs(featureType); // Set SpatialDataSource fields (Conveyal metadata) from GeoTools model + // TODO project into WGS84, perhaps using Query. ReferencedEnvelope envelope = featureCollection.getBounds(); // TODO Range-check lats and lons, projection of bad inputs can give negative areas (even check every feature) // TODO Also check bounds for antimeridian crossing @@ -165,6 +155,7 @@ public void ingest (File file, ProgressListener progressListener) { // Cannot set from FeatureType because it's always Geometry for GeoJson. dataSource.geometryType = ShapefileReader.GeometryType.forBindingClass(firstGeometryType); dataSource.featureCount = featureCollection.size(); + progressListener.increment(); } catch (FactoryException | IOException e) { // Unexpected errors cause immediate failure; predictable issues will be recorded on the DataSource object. // Catch only checked exceptions to preserve the top-level exception type when possible. diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java index 17f28ce4f..ff0890630 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -20,6 +20,7 @@ import org.opengis.feature.simple.SimpleFeatureType; import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -52,6 +53,8 @@ public GeoPackageDataSourceIngester () { @Override public void ingest (File file, ProgressListener progressListener) { + progressListener.beginTask("Validating uploaded GeoPackage", 2); + progressListener.setWorkProduct(dataSource.toWorkProduct()); try { Map params = new HashMap(); params.put("dbtype", "geopkg"); @@ -69,6 +72,7 @@ public void ingest (File file, ProgressListener progressListener) { FeatureCollection featureCollection = featureSource.getFeatures(); Envelope envelope = featureCollection.getBounds(); checkWgsEnvelopeSize(envelope); // Note this may be projected TODO reprojection logic + progressListener.increment(); // reader.wgs84Stream().forEach(f -> { // checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); // }); @@ -76,8 +80,9 @@ public void ingest (File file, ProgressListener progressListener) { dataSource.attributes = attributes(featureCollection.getSchema()); dataSource.geometryType = geometryType(featureCollection); dataSource.featureCount = featureCollection.size(); - } catch (Exception e) { - throw new RuntimeException("Error parsing GeoPackage. Ensure the file you uploaded is valid.", e); + progressListener.increment(); + } catch (IOException e) { + throw new RuntimeException("Error reading GeoPackage due to IOException.", e); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index 85654c4ae..157983a1c 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -38,7 +38,8 @@ public ShapefileDataSourceIngester () { @Override public void ingest (File file, ProgressListener progressListener) { - progressListener.beginTask("Validating files", 1); + progressListener.beginTask("Validating uploaded shapefile", 2); + progressListener.setWorkProduct(dataSource.toWorkProduct()); try { ShapefileReader reader = new ShapefileReader(file); // Iterate over all features to ensure file is readable, geometries are valid, and can be reprojected. @@ -48,10 +49,12 @@ public void ingest (File file, ProgressListener progressListener) { checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); }); reader.close(); + progressListener.increment(); dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); dataSource.attributes = reader.attributes(); dataSource.geometryType = reader.geometryType(); dataSource.featureCount = reader.featureCount(); + progressListener.increment(); } catch (FactoryException | TransformException e) { throw new DataSourceException("Shapefile transform error. " + "Try uploading an unprojected WGS84 (EPSG:4326) file.", e); diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java index 68bcbe5c1..8586a361c 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSource.java +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -43,7 +43,10 @@ public abstract class DataSource extends BaseModel { public FileStorageFormat fileFormat; - // This type uses (north, south, east, west), ideally we'd use (minLon, minLat, maxLon, maxLat). + /** + * The geographic bounds of this data set in WGS84 coordinates (independent of the original CRS of uploaded file). + * This type uses (north, south, east, west), ideally for consistency we'd use (minLon, minLat, maxLon, maxLat). + */ public Bounds wgsBounds; /** diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index d8ddc16b5..ef07e0723 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -22,7 +22,8 @@ public enum FileStorageFormat { GTFS("zip", "application/zip"), OSMPBF("pbf", "application/octet-stream"), // Also can be application/geo+json, see https://www.iana.org/assignments/media-types/application/geo+json - GEOJSON("json", "application/json"), + // The extension used to be defined as .json TODO ensure that changing it to .geojson hasn't broken anything. + GEOJSON("geojson", "application/json"), // See requirement 3 http://www.geopackage.org/spec130/#_file_extension_name GEOPACKAGE("gpkg", "application/geopackage+sqlite3"); diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 70c260be6..2b131ff2d 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -799,10 +799,12 @@ public static double roughWgsEnvelopeArea (Envelope wgsEnvelope) { * Throw an exception if the provided envelope is too big for a reasonable destination grid. */ public static void checkWgsEnvelopeSize (Envelope envelope) { + checkWgsEnvelopeRange(envelope); if (roughWgsEnvelopeArea(envelope) > MAX_BOUNDING_BOX_AREA_SQ_KM) { - throw new DataSourceException(String.format("Geographic extent of spatial layer (%.0f km2) exceeds limit of %.0f km2.", - roughWgsEnvelopeArea(envelope), MAX_BOUNDING_BOX_AREA_SQ_KM)); - + throw new DataSourceException(String.format( + "Geographic extent of spatial layer (%.0f km2) exceeds limit of %.0f km2.", + roughWgsEnvelopeArea(envelope), MAX_BOUNDING_BOX_AREA_SQ_KM + )); } } @@ -815,4 +817,28 @@ public static void checkPixelCount (WebMercatorExtents extents, int layers) { } } + /** + * We have to range-check the envelope before checking its size. Large unprojected y values interpreted as latitudes + * can yield negaive cosines, producing negative estimated areas, producing false negatives on size checks. + */ + private static void checkWgsEnvelopeRange (Envelope envelope) { + checkLon(envelope.getMinX()); + checkLon(envelope.getMaxX()); + checkLat(envelope.getMinY()); + checkLat(envelope.getMaxY()); + } + + private static void checkLon (double longitude) { + if (!Double.isFinite(longitude) || Math.abs(longitude) > 180) { + throw new DataSourceException("Longitude is not a finite number with absolute value below 180."); + } + } + + private static void checkLat (double latitude) { + // Longyearbyen on the Svalbard archipelago is the world's northernmost permanent settlement (78 degrees N). + if (!Double.isFinite(latitude) || Math.abs(latitude) > 80) { + throw new DataSourceException("Longitude is not a finite number with absolute value below 80."); + } + } + } diff --git a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java index 6dd398c1e..8b3bb0af7 100644 --- a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java @@ -5,53 +5,42 @@ import java.io.File; +import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.ingest; import static com.conveyal.file.FileStorageFormat.GEOJSON; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Created by abyrd on 2021-08-27 - * TODO Instead of loading from files, build GeoJSON programmatically, serialize it to temp files, then load it. + * Beyond the standard cases in SpatialDataSourceIngesterTest, special cases for GeoJSON ingestion. + * TODO Maybe instead of loading from files, build GeoJSON programmatically, serialize it to temp files, then load it. */ class GeoJsonDataSourceIngesterTest { - @Test - void basicValidGeoJson () { - SpatialDataSource spatialDataSource = ingest("hkzones-wgs84"); - } - @Test void typeMismatch () { - SpatialDataSource spatialDataSource = ingest("hkzones-type-mismatch"); + SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-type-mismatch"); } @Test void extraAttribute () { - SpatialDataSource spatialDataSource = ingest("hkzones-extra-attribute"); + SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-extra-attribute"); } @Test void mixedNumeric () { - SpatialDataSource spatialDataSource = ingest("hkzones-mixed-numeric"); + SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-mixed-numeric"); } @Test void mixedGeometries () { - SpatialDataSource spatialDataSource = ingest("hkzones-mixed-geometries"); + SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-mixed-geometries"); } - @Test - void mercatorBadProjection () { - SpatialDataSource spatialDataSource = ingest("hkzones-mercator"); - } - - // TODO span antimeridian, giant input geometry - @Test void fileEmpty () { assertThrows( DataSourceException.class, - () -> ingest("empty"), + () -> ingest(GEOJSON, "empty"), "Expected exception on empty input file." ); } @@ -60,53 +49,9 @@ void fileEmpty () { void fileTooSmall () { assertThrows( DataSourceException.class, - () -> ingest("too-small"), + () -> ingest(GEOJSON, "too-small"), "Expected exception on input file too short to be GeoJSON." ); } - /** - * Test on a GeoJSON file containing huge shapes: the continents of Africa, South America, and Australia. - */ - @Test - void continentalScale () { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest("continents"), - "Expected exception on continental-scale GeoJSON." - ); - assertTrue(throwable.getMessage().contains("exceeds")); - } - - /** - * Test on WGS84 GeoJSON containing shapes on both sides of the 180 degree antimeridian. - * This case was encountered in the wild: the North Island and the Chatham islands, both part of New Zealand. - */ - @Test - void newZealandAntimeridian () { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest("new-zealand-antimeridian"), - "Expected exception on shapefile crossing antimeridian." - ); - // TODO generate message specifically about 180 degree meridian, not excessive bbox size - assertTrue(throwable.getMessage().contains("exceeds")); - } - - private SpatialDataSource ingest (String inputFile) { - TestingProgressListener progressListener = new TestingProgressListener(); - DataSourceIngester ingester = DataSourceIngester.forFormat(GEOJSON); - File geoJsonInputFile = getResourceAsFile(inputFile + ".geojson"); - ingester.ingest(geoJsonInputFile, progressListener); - // TODO progressListener.assertUsedCorrectly(); - return ((SpatialDataSource) ingester.dataSource()); - } - - // Method is non-static since resource resolution is relative to the package of the current class. - // In a static context, you can also do XYZTest.class.getResource(). - private File getResourceAsFile (String resource) { - // This just removes the protocol and query parameter part of the URL, which for File URLs is a file path. - return new File(getClass().getResource(resource).getFile()); - } - } diff --git a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java index 406f0d3bc..6ccc8b948 100644 --- a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java @@ -5,13 +5,15 @@ import java.io.File; +import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.ingest; import static com.conveyal.file.FileStorageFormat.SHP; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** + * Beyond the standard cases in SpatialDataSourceIngesterTest, special cases for ESRI Shapefile ingestion. * Test that we can correctly read Shapefiles with many different characteristics, and can detect problematic inputs - * including many that we've encountered in practice. + * including ones we've encountered in practice. */ class ShapefileDataSourceIngesterTest { @@ -24,7 +26,7 @@ class ShapefileDataSourceIngesterTest { void nullPointGeometries () { Throwable throwable = assertThrows( DataSourceException.class, - () -> ingest("nl-null-points"), + () -> ingest(SHP, "nl-null-points"), "Expected exception on shapefile with null geometries." ); assertTrue(throwable.getMessage().contains("missing")); @@ -39,54 +41,10 @@ void nullPointGeometries () { */ @Test void duplicateAttributeNames () { - SpatialDataSource spatialDataSource = ingest("duplicate-fields"); + SpatialDataSource spatialDataSource = ingest(SHP, "duplicate-fields"); // id, the_geom, DDDDDDDDDD, and DDDDDDDDDD. The final one will be renamed on the fly to DDDDDDDDDD1. assertTrue(spatialDataSource.attributes.size() == 4); assertTrue(spatialDataSource.attributes.get(3).name.endsWith("1")); } - /** - * Test on a shapefile containing huge shapes, - * in this case the continents of Africa, South America, and Australia. - */ - @Test - void continentalScale () { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest("continents"), - "Expected exception on continental-scale shapefile." - ); - assertTrue(throwable.getMessage().contains("exceeds")); - } - - /** - * Test on a shapefile containing shapes on both sides of the 180 degree antimeridian. - * This case was encountered in the wild: the North Island and the Chatham islands, both part of New Zealand. - */ - @Test - void newZealandAntimeridian () { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest("new-zealand-antimeridian"), - "Expected exception on shapefile crossing antimeridian." - ); - // TODO generate message specifically about 180 degree meridian, not excessive bbox size - assertTrue(throwable.getMessage().contains("exceeds")); - } - - private SpatialDataSource ingest (String shpBaseName) { - TestingProgressListener progressListener = new TestingProgressListener(); - DataSourceIngester ingester = DataSourceIngester.forFormat(SHP); - File geoJsonInputFile = getResourceAsFile(shpBaseName + ".shp"); - ingester.ingest(geoJsonInputFile, progressListener); - // TODO progressListener.assertUsedCorrectly(); - return ((SpatialDataSource) ingester.dataSource()); - } - - /** Method is non-static since resource resolution is relative to the package of the current class. */ - private File getResourceAsFile (String resource) { - // This just removes the protocol and query parameter part of the URL, which for File URLs leaves a file path. - return new File(getClass().getResource(resource).getFile()); - } - } \ No newline at end of file diff --git a/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java new file mode 100644 index 000000000..57fa1742b --- /dev/null +++ b/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java @@ -0,0 +1,114 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.file.FileStorageFormat; +import com.conveyal.r5.util.ShapefileReader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.locationtech.jts.geom.Envelope; + +import java.io.File; +import java.lang.invoke.MethodHandles; +import java.util.EnumSet; +import java.util.List; +import java.util.Optional; + +import static com.conveyal.analysis.datasource.SpatialAttribute.Type.NUMBER; +import static com.conveyal.analysis.datasource.SpatialAttribute.Type.TEXT; +import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static com.conveyal.file.FileStorageFormat.GEOPACKAGE; +import static com.conveyal.file.FileStorageFormat.SHP; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test ingestion of all different GeoTools based geographic (spatial) data source files, including GeoPackage. + */ +class SpatialDataSourceIngesterTest { + + // TODO parameter provider method for typed FileStorageFormat enum values instead of String arrays. + // Or a separate local enum that gets mapped to the FileStorageFormat enum. + + /** Envelope around Hong Kong Island, Kowloon, and Lamma. */ + public static final Envelope HK_ENVELOPE = new Envelope(114.09, 114.40, 22.18, 22.20); + + /** + * Test on small basic data sets with no errors, but projected into some relatively obscure local coordinate system. + */ + @ParameterizedTest + @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) + void basicValid (FileStorageFormat format) { + // JUnit can't yet do cartesian products of parameters - iterate within this method. + // According to Github issues, soon we should have List Arguments.cartesianProduct(Set... sets) + // TODO for (String geomType : List.of("point", "polygon", "linestring")) { + // For now all test files are polygons with three features and two additional attributes (name and count). + for (String fileSuffix : List.of("wgs84", "projected")) { + SpatialDataSource spatialDataSource = ingest(format, "valid-polygon-" + fileSuffix); + assertTrue(spatialDataSource.issues.isEmpty()); + assertTrue(spatialDataSource.geometryType == POLYGON); + assertTrue(spatialDataSource.featureCount == 3); + assertTrue(hasAttribute(spatialDataSource.attributes, "Name", TEXT)); + assertTrue(hasAttribute(spatialDataSource.attributes, "Count", NUMBER)); + assertFalse(hasAttribute(spatialDataSource.attributes, "Count", TEXT)); + // FIXME projected DataSources are returning projected bounds, not WGS84. + assertTrue(HK_ENVELOPE.contains(spatialDataSource.wgsBounds.envelope())); + } + } + + /** Test on files containing huge shapes: the continents of Africa, South America, and Australia. */ + @ParameterizedTest + @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) + void continentalScale (FileStorageFormat format) { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest(format, "continents"), + "Expected exception on continental-scale geographic features." + ); + assertTrue(throwable.getMessage().contains("exceeds")); + } + + /** + * Test on projected (non-WGS84) GeoPackage containing shapes on both sides of the 180 degree antimeridian. + * This case was encountered in the wild: the North Island and the Chatham islands, both part of New Zealand. + */ + @ParameterizedTest + @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) + void newZealandAntimeridian (FileStorageFormat format) { + Throwable throwable = assertThrows( + DataSourceException.class, + () -> ingest(format, "new-zealand-antimeridian"), + "Expected exception on geographic feature collection crossing antimeridian." + ); + // TODO generate message specifically about 180 degree meridian, not excessive bbox size + assertTrue(throwable.getMessage().contains("exceeds")); + } + + public static SpatialDataSource ingest (FileStorageFormat format, String inputFile) { + TestingProgressListener progressListener = new TestingProgressListener(); + DataSourceIngester ingester = DataSourceIngester.forFormat(format); + ingester.initializeDataSource("TestName", "Test Description", "test_region_id", + new UserPermissions("test@email.com", false, "test_group")); + File geoJsonInputFile = getResourceAsFile(String.join(".", inputFile, format.extension)); + ingester.ingest(geoJsonInputFile, progressListener); + progressListener.assertUsedCorrectly(); + return ((SpatialDataSource) ingester.dataSource()); + } + + /** Method is static, so resolution is always relative to the package of the class where it's defined. */ + private static File getResourceAsFile (String resource) { + // This just removes the protocol and query parameter part of the URL, which for File URLs leaves a file path. + return new File(SpatialDataSourceIngesterTest.class.getResource(resource).getFile()); + } + + protected static boolean hasAttribute (List attributes, String name, SpatialAttribute.Type type) { + Optional optional = attributes.stream().filter(a -> a.name.equals(name)).findFirst(); + return optional.isPresent() && optional.get().type == type; + // && attribute.occurrances > 0 + } + +} diff --git a/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java b/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java index f4a2c8b6c..ac7cffd21 100644 --- a/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java +++ b/src/test/java/com/conveyal/analysis/datasource/TestingProgressListener.java @@ -4,6 +4,9 @@ import com.conveyal.r5.analyst.progress.WorkProduct; import org.junit.jupiter.api.Assertions; +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertTrue; + /** * A mock ProgressListener for use in tests, which makes sure all the interface methods are called and shows progress. */ @@ -11,16 +14,21 @@ public class TestingProgressListener implements ProgressListener { private String description; private WorkProduct workProduct; - private int count = 0; + private int taskCount = 0; + private int totalElements = 0; + private int elementsCompleted = 0; @Override public void beginTask (String description, int totalElements) { this.description = description; + this.totalElements = totalElements; + taskCount += 1; } @Override public void increment (int n) { - count += n; + elementsCompleted += n; + assertTrue(elementsCompleted <= totalElements); } @Override @@ -29,9 +37,11 @@ public void setWorkProduct (WorkProduct workProduct) { } public void assertUsedCorrectly () { - Assertions.assertNotNull(description); - Assertions.assertNotNull(workProduct); - Assertions.assertTrue(count > 0); + assertNotNull(description); + assertNotNull(workProduct); + assertTrue(taskCount > 0); + assertTrue(elementsCompleted > 0); + assertEquals(totalElements, elementsCompleted); } } diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.cpg b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.cpg new file mode 100644 index 000000000..3ad133c04 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.dbf b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.dbf new file mode 100644 index 0000000000000000000000000000000000000000..23fa132f6f53063390d0a8c48d1f318f011ca126 GIT binary patch literal 985 zcmZRsdH`Ui2YUbj diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.prj b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.prj new file mode 100644 index 000000000..e5e7cad96 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.prj @@ -0,0 +1 @@ +PROJCS["Hong_Kong_1980_Grid",GEOGCS["GCS_Hong_Kong_1980",DATUM["D_Hong_Kong_1980",SPHEROID["International_1924",6378388.0,297.0]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",836694.05],PARAMETER["False_Northing",819069.8],PARAMETER["Central_Meridian",114.178555555556],PARAMETER["Scale_Factor",1.0],PARAMETER["Latitude_Of_Origin",22.3121333333333],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.shp b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.shp new file mode 100644 index 0000000000000000000000000000000000000000..f711da82f3d1ad749d1343b753e666a581aa08d3 GIT binary patch literal 524 zcmZQzQ0HR63K-d5Ff%Z)0_EZb&#qwg(RAe1F_oCJNW+oykK$_C37U>hbLFl&aB4bY z)s8C22;@3|%zABOYQa~o=_u{^<#6&ROtU}=LFzyNrZ&Q54@Vb=rehn^A~$4vVeZH| zDjUM`L&NdzX@#KUbqKT3^#hG$0%8wreo|kU{$^IHrsLk9;@_Sf(r^U1gAIs50Oo%t zzDc!ido&z3&YRcf8IDj3)4S@B>7;ALX*zz?6McMkv4-Q-bCn5Ek%;g^ zHxFn9Fr*>jW25%@pn9~X;}z*?S{EN^IIjBq(Uu`e({V|i-^$L{5O=}C2d4H$)G`OY k=NgVxx(@Zdi3qhYy?5Ng=KO!C;ppl0_r60s!fbT?05Qmz$N&HU literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.shx b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-projected.shx new file mode 100644 index 0000000000000000000000000000000000000000..2d47ba9712396b601b5b245fe03d039aead405e2 GIT binary patch literal 124 zcmZQzQ0HR64(whqGcd3M<>Cd;u3+`ibmY}Bm6)?g!;$lk;%eCmnvPC$<*qt#YC2-q Vjv{IVdH`Xo2aNy# diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.prj b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.prj new file mode 100644 index 000000000..f45cbadf0 --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.shp b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.shp new file mode 100644 index 0000000000000000000000000000000000000000..7a2b11b62dac8d04125caf31452465ad913e20e8 GIT binary patch literal 524 zcmZQzQ0HR63K-d5Ff%Z)0_Cy{J8v9ri*eYItQ@6lV&pZU3StN(=3oekU9{6sb$y~-4YJ82i;y6f17UX%VJ+M zhs>q#T#btwl#AAxq}UeK>+6e$|x`Eqt<2) zx;fcjRmE!-WM|knk9e5_Io=py!QXvWPTZ_9u|J+<^in$hBPF6 zid;lez5~^2l`LbN=4R&bJ4tj`6VR+{xwnlDc|zO;3m=$Tm4Mrgc^+mCvp)6rR3fQ` Y=`~Zcn)n`Qme-${%U(cvnAzz10e14AUjP6A literal 0 HcmV?d00001 diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.shx b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.shx new file mode 100644 index 0000000000000000000000000000000000000000..8778a8c1e3ce26c271ffe883007296f47faa622c GIT binary patch literal 124 zcmZQzQ0HR64(whqGcd3M<+2PrZyaumaoCZp9Hnbw=AgnTqo&&v@~ literal 0 HcmV?d00001 From 049e92f82ef49011f12bea5597b0b3f01226866f Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 31 Aug 2021 17:49:16 +0800 Subject: [PATCH 091/187] assert ERROR issues when geometries are mismatched --- .../datasource/GeoJsonDataSourceIngesterTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java index 8b3bb0af7..83e5fcc6f 100644 --- a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java @@ -1,12 +1,15 @@ package com.conveyal.analysis.datasource; +import com.conveyal.analysis.models.DataSourceValidationIssue; import com.conveyal.analysis.models.SpatialDataSource; import org.junit.jupiter.api.Test; import java.io.File; import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.ingest; +import static com.conveyal.analysis.models.DataSourceValidationIssue.Level.ERROR; import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -19,21 +22,32 @@ class GeoJsonDataSourceIngesterTest { @Test void typeMismatch () { SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-type-mismatch"); + // Currently we can't detect problems with inconsistent schema across features. + // GeoTools seems to report the same schema on every feature. + assertTrue(spatialDataSource.issues.isEmpty()); } @Test void extraAttribute () { SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-extra-attribute"); + // Currently we can't detect problems with inconsistent schema across features. + // GeoTools seems to report the same schema on every feature. + assertTrue(spatialDataSource.issues.isEmpty()); } @Test void mixedNumeric () { SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-mixed-numeric"); + // Currently we can't detect problems with inconsistent schema across features. + // GeoTools seems to report the same schema on every feature. + assertTrue(spatialDataSource.issues.isEmpty()); } @Test void mixedGeometries () { SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-mixed-geometries"); + // Inconsistent geometry between features is detected. + assertTrue(spatialDataSource.issues.stream().anyMatch(i -> i.level == ERROR)); } @Test From 77e77fe73e021622f35ea3f251359d91065865b3 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 31 Aug 2021 21:47:25 +0800 Subject: [PATCH 092/187] more work on spatial DataSourceIngester tests enlarge HK envelope to the north factor out test method expecting DataSourceException use Query to reproject features into WGS84 all tests should now pass --- .../datasource/GeoJsonDataSourceIngester.java | 21 +++--- .../GeoPackageDataSourceIngester.java | 29 +++++--- .../GeoJsonDataSourceIngesterTest.java | 26 +++----- .../ShapefileDataSourceIngesterTest.java | 15 ++--- .../SpatialDataSourceIngesterTest.java | 62 +++++++++--------- .../datasource/valid-polygon-wgs84.gpkg | Bin 98304 -> 98304 bytes 6 files changed, 73 insertions(+), 80 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index e47319ab8..44b541acb 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -6,6 +6,7 @@ import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader; +import org.geotools.data.Query; import org.geotools.data.geojson.GeoJSONDataStore; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; @@ -114,11 +115,13 @@ public void ingest (File file, ProgressListener progressListener) { GeoJSONDataStore dataStore = new GeoJSONDataStore(file); SimpleFeatureSource featureSource = dataStore.getFeatureSource(); // This loads the whole thing into memory. That should be harmless given our file upload size limits. - FeatureCollection featureCollection = featureSource.getFeatures(); + Query query = new Query(Query.ALL); + query.setCoordinateSystemReproject(DefaultGeographicCRS.WGS84); + FeatureCollection wgsFeatureCollection = featureSource.getFeatures(query); // The schema of the FeatureCollection does seem to reflect all attributes present on all features. // However the type of those attributes seems to be restricted to that of the first value encountered. // Conversions may fail silently on any successive instances of that property with a different type. - SimpleFeatureType featureType = featureCollection.getSchema(); + SimpleFeatureType featureType = wgsFeatureCollection.getSchema(); // Note: this somewhat duplicates ShapefileReader.attributes, code should be reusable across formats // But look into the null checking and duplicate attribute checks there. dataSource.attributes = new ArrayList<>(); @@ -128,7 +131,7 @@ public void ingest (File file, ProgressListener progressListener) { // The schema always reports the geometry type as the very generic "Geometry" class. // Check that all features have the same concrete Geometry type. Class firstGeometryType = null; - FeatureIterator iterator = featureCollection.features(); + FeatureIterator iterator = wgsFeatureCollection.features(); while (iterator.hasNext()) { SimpleFeature feature = iterator.next(); Geometry geometry = (Geometry) feature.getDefaultGeometry(); @@ -145,16 +148,13 @@ public void ingest (File file, ProgressListener progressListener) { } progressListener.increment(); checkCrs(featureType); + Envelope wgsEnvelope = wgsFeatureCollection.getBounds(); + checkWgsEnvelopeSize(wgsEnvelope); // Set SpatialDataSource fields (Conveyal metadata) from GeoTools model - // TODO project into WGS84, perhaps using Query. - ReferencedEnvelope envelope = featureCollection.getBounds(); - // TODO Range-check lats and lons, projection of bad inputs can give negative areas (even check every feature) - // TODO Also check bounds for antimeridian crossing - checkWgsEnvelopeSize(envelope); - dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); + dataSource.wgsBounds = Bounds.fromWgsEnvelope(wgsEnvelope); // Cannot set from FeatureType because it's always Geometry for GeoJson. dataSource.geometryType = ShapefileReader.GeometryType.forBindingClass(firstGeometryType); - dataSource.featureCount = featureCollection.size(); + dataSource.featureCount = wgsFeatureCollection.size(); progressListener.increment(); } catch (FactoryException | IOException e) { // Unexpected errors cause immediate failure; predictable issues will be recorded on the DataSource object. @@ -166,6 +166,7 @@ public void ingest (File file, ProgressListener progressListener) { /** * GeoJSON used to allow CRS, but the RFC now says GeoJSON is always in WGS84 and no other CRS are allowed. * QGIS and GeoTools both seem to support this, but it's an obsolete feature. + * FIXME this is never failing, even on projected input. The GeoTools reader seems to silently convert to WGS84. */ private static void checkCrs (FeatureType featureType) throws FactoryException { CoordinateReferenceSystem crs = featureType.getCoordinateReferenceSystem(); diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java index ff0890630..704c5c4a2 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -9,11 +9,13 @@ import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; import org.geotools.data.FeatureSource; +import org.geotools.data.Query; import org.geotools.data.geojson.GeoJSONDataStore; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeature; @@ -69,17 +71,24 @@ public void ingest (File file, ProgressListener progressListener) { throw new RuntimeException("GeoPackage must contain only one table, this file has " + typeNames.length); } FeatureSource featureSource = datastore.getFeatureSource(typeNames[0]); - FeatureCollection featureCollection = featureSource.getFeatures(); - Envelope envelope = featureCollection.getBounds(); - checkWgsEnvelopeSize(envelope); // Note this may be projected TODO reprojection logic + Query query = new Query(Query.ALL); + query.setCoordinateSystemReproject(DefaultGeographicCRS.WGS84); + FeatureCollection wgsFeatureCollection = featureSource.getFeatures(query); + Envelope wgsEnvelope = wgsFeatureCollection.getBounds(); + checkWgsEnvelopeSize(wgsEnvelope); progressListener.increment(); -// reader.wgs84Stream().forEach(f -> { -// checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); -// }); - dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); - dataSource.attributes = attributes(featureCollection.getSchema()); - dataSource.geometryType = geometryType(featureCollection); - dataSource.featureCount = featureCollection.size(); + FeatureIterator wgsFeatureIterator = wgsFeatureCollection.features(); + while (wgsFeatureIterator.hasNext()) { + Geometry wgsFeatureGeometry = (Geometry)(wgsFeatureIterator.next().getDefaultGeometry()); + // FIXME GeoTools seems to be returning an envelope slightly smaller than the projected shapes. + // maybe it's giving us projection(envelope(shapes)) instead of envelope(projection(shapes))? + // As a stopgap, test that they intersect. + checkState(wgsEnvelope.intersects(wgsFeatureGeometry.getEnvelopeInternal())); + } + dataSource.wgsBounds = Bounds.fromWgsEnvelope(wgsEnvelope); + dataSource.attributes = attributes(wgsFeatureCollection.getSchema()); + dataSource.geometryType = geometryType(wgsFeatureCollection); + dataSource.featureCount = wgsFeatureCollection.size(); progressListener.increment(); } catch (IOException e) { throw new RuntimeException("Error reading GeoPackage due to IOException.", e); diff --git a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java index 83e5fcc6f..35d3ccee6 100644 --- a/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngesterTest.java @@ -1,12 +1,10 @@ package com.conveyal.analysis.datasource; -import com.conveyal.analysis.models.DataSourceValidationIssue; import com.conveyal.analysis.models.SpatialDataSource; import org.junit.jupiter.api.Test; -import java.io.File; - -import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.ingest; +import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.assertIngestException; +import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.testIngest; import static com.conveyal.analysis.models.DataSourceValidationIssue.Level.ERROR; import static com.conveyal.file.FileStorageFormat.GEOJSON; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -21,7 +19,7 @@ class GeoJsonDataSourceIngesterTest { @Test void typeMismatch () { - SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-type-mismatch"); + SpatialDataSource spatialDataSource = testIngest(GEOJSON, "hkzones-type-mismatch"); // Currently we can't detect problems with inconsistent schema across features. // GeoTools seems to report the same schema on every feature. assertTrue(spatialDataSource.issues.isEmpty()); @@ -29,7 +27,7 @@ void typeMismatch () { @Test void extraAttribute () { - SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-extra-attribute"); + SpatialDataSource spatialDataSource = testIngest(GEOJSON, "hkzones-extra-attribute"); // Currently we can't detect problems with inconsistent schema across features. // GeoTools seems to report the same schema on every feature. assertTrue(spatialDataSource.issues.isEmpty()); @@ -37,7 +35,7 @@ void extraAttribute () { @Test void mixedNumeric () { - SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-mixed-numeric"); + SpatialDataSource spatialDataSource = testIngest(GEOJSON, "hkzones-mixed-numeric"); // Currently we can't detect problems with inconsistent schema across features. // GeoTools seems to report the same schema on every feature. assertTrue(spatialDataSource.issues.isEmpty()); @@ -45,27 +43,19 @@ void mixedNumeric () { @Test void mixedGeometries () { - SpatialDataSource spatialDataSource = ingest(GEOJSON, "hkzones-mixed-geometries"); + SpatialDataSource spatialDataSource = testIngest(GEOJSON, "hkzones-mixed-geometries"); // Inconsistent geometry between features is detected. assertTrue(spatialDataSource.issues.stream().anyMatch(i -> i.level == ERROR)); } @Test void fileEmpty () { - assertThrows( - DataSourceException.class, - () -> ingest(GEOJSON, "empty"), - "Expected exception on empty input file." - ); + assertIngestException(GEOJSON, "empty", DataSourceException.class, "length"); } @Test void fileTooSmall () { - assertThrows( - DataSourceException.class, - () -> ingest(GEOJSON, "too-small"), - "Expected exception on input file too short to be GeoJSON." - ); + assertIngestException(GEOJSON, "too-small", DataSourceException.class, "length"); } } diff --git a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java index 6ccc8b948..78157ead3 100644 --- a/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngesterTest.java @@ -3,9 +3,9 @@ import com.conveyal.analysis.models.SpatialDataSource; import org.junit.jupiter.api.Test; -import java.io.File; - -import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.ingest; +import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.assertIngestException; +import static com.conveyal.analysis.datasource.SpatialDataSourceIngesterTest.testIngest; +import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -24,12 +24,7 @@ class ShapefileDataSourceIngesterTest { */ @Test void nullPointGeometries () { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest(SHP, "nl-null-points"), - "Expected exception on shapefile with null geometries." - ); - assertTrue(throwable.getMessage().contains("missing")); + assertIngestException(SHP, "nl-null-points", DataSourceException.class, "missing"); } /** @@ -41,7 +36,7 @@ void nullPointGeometries () { */ @Test void duplicateAttributeNames () { - SpatialDataSource spatialDataSource = ingest(SHP, "duplicate-fields"); + SpatialDataSource spatialDataSource = testIngest(SHP, "duplicate-fields"); // id, the_geom, DDDDDDDDDD, and DDDDDDDDDD. The final one will be renamed on the fly to DDDDDDDDDD1. assertTrue(spatialDataSource.attributes.size() == 4); assertTrue(spatialDataSource.attributes.get(3).name.endsWith("1")); diff --git a/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java index 57fa1742b..e9c65e802 100644 --- a/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java @@ -3,23 +3,17 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.file.FileStorageFormat; -import com.conveyal.r5.util.ShapefileReader; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.ValueSource; import org.locationtech.jts.geom.Envelope; import java.io.File; -import java.lang.invoke.MethodHandles; -import java.util.EnumSet; import java.util.List; import java.util.Optional; import static com.conveyal.analysis.datasource.SpatialAttribute.Type.NUMBER; import static com.conveyal.analysis.datasource.SpatialAttribute.Type.TEXT; import static com.conveyal.file.FileStorageFormat.GEOJSON; -import static com.conveyal.file.FileStorageFormat.GEOPACKAGE; import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -35,7 +29,7 @@ class SpatialDataSourceIngesterTest { // Or a separate local enum that gets mapped to the FileStorageFormat enum. /** Envelope around Hong Kong Island, Kowloon, and Lamma. */ - public static final Envelope HK_ENVELOPE = new Envelope(114.09, 114.40, 22.18, 22.20); + public static final Envelope HK_ENVELOPE = new Envelope(114.09, 114.40, 22.18, 22.34); /** * Test on small basic data sets with no errors, but projected into some relatively obscure local coordinate system. @@ -48,15 +42,20 @@ void basicValid (FileStorageFormat format) { // TODO for (String geomType : List.of("point", "polygon", "linestring")) { // For now all test files are polygons with three features and two additional attributes (name and count). for (String fileSuffix : List.of("wgs84", "projected")) { - SpatialDataSource spatialDataSource = ingest(format, "valid-polygon-" + fileSuffix); - assertTrue(spatialDataSource.issues.isEmpty()); - assertTrue(spatialDataSource.geometryType == POLYGON); - assertTrue(spatialDataSource.featureCount == 3); - assertTrue(hasAttribute(spatialDataSource.attributes, "Name", TEXT)); - assertTrue(hasAttribute(spatialDataSource.attributes, "Count", NUMBER)); - assertFalse(hasAttribute(spatialDataSource.attributes, "Count", TEXT)); - // FIXME projected DataSources are returning projected bounds, not WGS84. - assertTrue(HK_ENVELOPE.contains(spatialDataSource.wgsBounds.envelope())); + if (format == GEOJSON && "projected".equals(fileSuffix)) { + // GeoTools silently ignores (illegal) non-WGS84 CRS in GeoJSON files. + assertIngestException(format, "valid-polygon-" + fileSuffix, DataSourceException.class, "value"); + } else { + SpatialDataSource spatialDataSource = testIngest(format, "valid-polygon-" + fileSuffix); + assertTrue(spatialDataSource.issues.isEmpty()); + assertTrue(spatialDataSource.geometryType == POLYGON); + assertTrue(spatialDataSource.featureCount == 3); + assertTrue(hasAttribute(spatialDataSource.attributes, "Name", TEXT)); + assertTrue(hasAttribute(spatialDataSource.attributes, "Count", NUMBER)); + assertFalse(hasAttribute(spatialDataSource.attributes, "Count", TEXT)); + // FIXME projected DataSources are returning projected bounds, not WGS84. + assertTrue(HK_ENVELOPE.contains(spatialDataSource.wgsBounds.envelope())); + } } } @@ -64,41 +63,40 @@ void basicValid (FileStorageFormat format) { @ParameterizedTest @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) void continentalScale (FileStorageFormat format) { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest(format, "continents"), - "Expected exception on continental-scale geographic features." - ); - assertTrue(throwable.getMessage().contains("exceeds")); + assertIngestException(format, "continents", DataSourceException.class, "exceeds"); } /** - * Test on projected (non-WGS84) GeoPackage containing shapes on both sides of the 180 degree antimeridian. + * Test on projected (non-WGS84) data containing shapes on both sides of the 180 degree antimeridian. * This case was encountered in the wild: the North Island and the Chatham islands, both part of New Zealand. */ @ParameterizedTest @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) void newZealandAntimeridian (FileStorageFormat format) { - Throwable throwable = assertThrows( - DataSourceException.class, - () -> ingest(format, "new-zealand-antimeridian"), - "Expected exception on geographic feature collection crossing antimeridian." - ); // TODO generate message specifically about 180 degree meridian, not excessive bbox size - assertTrue(throwable.getMessage().contains("exceeds")); + assertIngestException(format, "new-zealand-antimeridian", DataSourceException.class, "exceeds"); } - public static SpatialDataSource ingest (FileStorageFormat format, String inputFile) { + public static SpatialDataSource testIngest (FileStorageFormat format, String inputFile) { TestingProgressListener progressListener = new TestingProgressListener(); DataSourceIngester ingester = DataSourceIngester.forFormat(format); ingester.initializeDataSource("TestName", "Test Description", "test_region_id", new UserPermissions("test@email.com", false, "test_group")); - File geoJsonInputFile = getResourceAsFile(String.join(".", inputFile, format.extension)); - ingester.ingest(geoJsonInputFile, progressListener); + File resourceFile = getResourceAsFile(String.join(".", inputFile, format.extension)); + ingester.ingest(resourceFile, progressListener); progressListener.assertUsedCorrectly(); return ((SpatialDataSource) ingester.dataSource()); } + public static void assertIngestException ( + FileStorageFormat format, String inputFile, Class exceptionType, String messageWord + ) { + Throwable throwable = assertThrows(exceptionType, () -> testIngest(format, inputFile), + "Expected failure with exception type: " + exceptionType.getSimpleName()); + assertTrue(throwable.getMessage().contains(messageWord), + "Exception message is expected to contain the text: " + messageWord); + } + /** Method is static, so resolution is always relative to the package of the class where it's defined. */ private static File getResourceAsFile (String resource) { // This just removes the protocol and query parameter part of the URL, which for File URLs leaves a file path. diff --git a/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.gpkg b/src/test/resources/com/conveyal/analysis/datasource/valid-polygon-wgs84.gpkg index 7f41481e13861987c6e4e366a3125d7410c91d9f..593fa03292926d28373a070693f0794966d86d11 100644 GIT binary patch delta 22 dcmZo@U~6b#n;^|7JyFJ)QMxf Date: Tue, 31 Aug 2021 22:20:25 +0800 Subject: [PATCH 093/187] remove deprecated gt-geojson dependency This transitively eliminates the unsupported simpleJson library. Use Jackson instead, which is used by the newer gt-geojsondatastore. Our project was riddled with uses of simpleJson, which is apparently abandoned and was duplicating functionality we typically get from Jackson. --- build.gradle | 1 - .../conveyal/analysis/components/HttpApi.java | 23 ++++++++------- .../AggregationAreaController.java | 6 ++-- .../OpportunityDatasetController.java | 16 +++++----- .../RegionalAnalysisController.java | 9 +++--- .../controllers/TimetableController.java | 29 ++++++++++--------- .../com/conveyal/analysis/util/JsonUtil.java | 20 +++++++++++++ .../scenario/IndexedPolygonCollection.java | 13 ++++++--- .../r5/analyst/scenario/RoadCongestion.java | 15 ++++++---- .../com/conveyal/r5/streets/StreetLayer.java | 19 ------------ 10 files changed, 81 insertions(+), 70 deletions(-) diff --git a/build.gradle b/build.gradle index a49ba2d2b..6b0975b22 100644 --- a/build.gradle +++ b/build.gradle @@ -165,7 +165,6 @@ dependencies { implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-referencing' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-shapefile' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-coverage' - implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojson' // TODO REMOVE DEPRECATED implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojsondatastore' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geopkg' implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geotiff' diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 1cb2d9d39..2714599ff 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -9,8 +9,8 @@ import com.conveyal.analysis.controllers.HttpController; import com.conveyal.analysis.util.JsonUtil; import com.conveyal.file.FileStorage; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.fileupload.FileUploadException; -import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -19,7 +19,9 @@ import java.io.IOException; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static com.conveyal.analysis.AnalysisServerException.Type.BAD_REQUEST; import static com.conveyal.analysis.AnalysisServerException.Type.RUNTIME; @@ -137,12 +139,13 @@ private spark.Service configureSparkService () { if (e.type == AnalysisServerException.Type.UNAUTHORIZED || e.type == AnalysisServerException.Type.FORBIDDEN ){ - JSONObject body = new JSONObject(); - body.put("type", e.type.toString()); - body.put("message", e.message); + ObjectNode body = JsonUtil.objectNode() + .put("type", e.type.toString()) + .put("message", e.message); + response.status(e.httpCode); response.type("application/json"); - response.body(body.toJSONString()); + response.body(JsonUtil.toJsonString(body)); } else { respondToException(e, request, response, e.type, e.message, e.httpCode); } @@ -174,14 +177,14 @@ private void respondToException(Exception e, Request request, Response response, ErrorEvent errorEvent = new ErrorEvent(e); eventBus.send(errorEvent.forUser(request.attribute(USER_PERMISSIONS_ATTRIBUTE))); - JSONObject body = new JSONObject(); - body.put("type", type.toString()); - body.put("message", message); - body.put("stackTrace", errorEvent.stackTrace); + ObjectNode body = JsonUtil.objectNode() + .put("type", type.toString()) + .put("message", message) + .put("stackTrace", errorEvent.stackTrace); response.status(code); response.type("application/json"); - response.body(body.toJSONString()); + response.body(JsonUtil.toJsonString(body)); } // Maybe this should be done or called with a JVM shutdown hook diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 6c5981d85..7795840f1 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -14,7 +14,6 @@ import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.util.ShapefileReader; import com.google.common.base.Preconditions; -import org.json.simple.JSONObject; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.operation.union.UnaryUnionOp; @@ -197,12 +196,13 @@ private Collection getAggregationAreas (Request req, Response r ); } - private JSONObject getAggregationArea (Request req, Response res) { + private Map getAggregationArea (Request req, Response res) { AggregationArea aggregationArea = aggregationAreaCollection.findByIdIfPermitted( req.params("maskId"), UserPermissions.from(req) ); String url = fileStorage.getURL(aggregationArea.getStorageKey()); - JSONObject wrappedUrl = new JSONObject(); + // Put the URL into something that will be serialized as a JSON object with a single property. + Map wrappedUrl = new HashMap<>(); wrappedUrl.put("url", url); return wrappedUrl; } diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 72c3d0b84..62665b4b8 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -9,6 +9,7 @@ import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.FileItemInputStreamProvider; +import com.conveyal.analysis.util.JsonUtil; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; @@ -20,6 +21,7 @@ import com.conveyal.r5.util.ExceptionUtils; import com.conveyal.r5.util.InputStreamProvider; import com.conveyal.r5.util.ProgressListener; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.io.Files; import com.mongodb.QueryBuilder; import org.apache.commons.fileupload.FileItem; @@ -29,7 +31,6 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.io.FilenameUtils; import org.bson.types.ObjectId; -import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -87,11 +88,8 @@ public OpportunityDatasetController ( /** Store upload status objects FIXME trivial Javadoc */ private final List uploadStatuses = new ArrayList<>(); - private JSONObject getJSONURL (FileStorageKey key) { - JSONObject json = new JSONObject(); - String url = fileStorage.getURL(key); - json.put("url", url); - return json; + private ObjectNode getJsonUrl (FileStorageKey key) { + return JsonUtil.objectNode().put("url", fileStorage.getURL(key)); } private void addStatusAndRemoveOldStatuses(OpportunityDatasetUploadStatus status) { @@ -112,7 +110,7 @@ private Collection getRegionDatasets(Request req, Response r private Object getOpportunityDataset(Request req, Response res) { OpportunityDataset dataset = Persistence.opportunityDatasets.findByIdFromRequestIfPermitted(req); if (dataset.format == FileStorageFormat.GRID) { - return getJSONURL(dataset.getStorageKey()); + return getJsonUrl(dataset.getStorageKey()); } else { // Currently the UI can only visualize grids, not other kinds of datasets (freeform points). // We do generate a rasterized grid for each of the freeform pointsets we create, so ideally we'd redirect @@ -583,7 +581,7 @@ private Object downloadOpportunityDataset (Request req, Response res) throws IOE String regionId = req.params("_id"); String gridKey = req.params("format"); FileStorageKey storageKey = new FileStorageKey(GRIDS, String.format("%s/%s.grid", regionId, gridKey)); - return getJSONURL(storageKey); + return getJsonUrl(storageKey); } if (FileStorageFormat.GRID.equals(downloadFormat)) return getOpportunityDataset(req, res); @@ -614,7 +612,7 @@ private Object downloadOpportunityDataset (Request req, Response res) throws IOE fileStorage.moveIntoStorage(formatKey, localFile); } - return getJSONURL(formatKey); + return getJsonUrl(formatKey); } /** diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index ec9bc2383..485820ef2 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -21,10 +21,10 @@ import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.cluster.RegionalTask; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.primitives.Ints; import com.mongodb.QueryBuilder; import gnu.trove.list.array.TIntArrayList; -import org.json.simple.JSONObject; import org.mongojack.DBProjection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -317,10 +317,9 @@ private Object getRegionalResults (Request req, Response res) throws IOException fileStorage.moveIntoStorage(singleCutoffFileStorageKey, localFile); } - - JSONObject json = new JSONObject(); - json.put("url", fileStorage.getURL(singleCutoffFileStorageKey)); - return json.toJSONString(); + return JsonUtil.toJsonString( + JsonUtil.objectNode().put("url", fileStorage.getURL(singleCutoffFileStorageKey)) + ); } } diff --git a/src/main/java/com/conveyal/analysis/controllers/TimetableController.java b/src/main/java/com/conveyal/analysis/controllers/TimetableController.java index 4bccee924..ee63dab44 100644 --- a/src/main/java/com/conveyal/analysis/controllers/TimetableController.java +++ b/src/main/java/com/conveyal/analysis/controllers/TimetableController.java @@ -6,9 +6,9 @@ import com.conveyal.analysis.models.Region; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.JsonUtil; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.mongodb.QueryBuilder; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -17,6 +17,9 @@ import java.util.Collection; import java.util.List; +import static com.conveyal.analysis.util.JsonUtil.arrayNode; +import static com.conveyal.analysis.util.JsonUtil.objectNode; + /** * Created by evan siroky on 5/3/18. */ @@ -29,34 +32,33 @@ public TimetableController () { // Eventually persistence will be a component (AnalysisDatabase) instead of static. } - // Unlike many other methods, rather than serializing a Java type to JSON, - // this builds up the JSON using a map-like API. It looks like we're using org.json.simple here - // instead of Jackson which we're using elsewhere. We should use one or the other. + // Unlike many other methods, rather than serializing a POJO to JSON, + // this builds up the JSON using the Jackson tree API. private String getTimetables (Request req, Response res) { - JSONArray json = new JSONArray(); Collection regions = Persistence.regions.findAllForRequest(req); + ArrayNode json = JsonUtil.objectMapper.createArrayNode(); for (Region region : regions) { - JSONObject r = new JSONObject(); + ObjectNode r = objectNode(); r.put("_id", region._id); r.put("name", region.name); - JSONArray regionProjects = new JSONArray(); + ArrayNode regionProjects = arrayNode(); List projects = Persistence.projects.find(QueryBuilder.start("regionId").is(region._id).get()).toArray(); for (Project project : projects) { - JSONObject p = new JSONObject(); + ObjectNode p = objectNode(); p.put("_id", project._id); p.put("name", project.name); - JSONArray projectModifications = new JSONArray(); + ArrayNode projectModifications = arrayNode(); List modifications = Persistence.modifications.find( QueryBuilder.start("projectId").is(project._id).and("type").is("add-trip-pattern").get() ).toArray(); for (Modification modification : modifications) { AddTripPattern tripPattern = (AddTripPattern) modification; - JSONObject m = new JSONObject(); + ObjectNode m = objectNode(); m.put("_id", modification._id); m.put("name", modification.name); m.put("segments", JsonUtil.objectMapper.valueToTree(tripPattern.segments)); - JSONArray modificationTimetables = new JSONArray(); + ArrayNode modificationTimetables = arrayNode(); for (AddTripPattern.Timetable timetable : tripPattern.timetables) { modificationTimetables.add(JsonUtil.objectMapper.valueToTree(timetable)); } @@ -75,8 +77,7 @@ private String getTimetables (Request req, Response res) { json.add(r); } } - - return json.toString(); + return JsonUtil.toJsonString(json); } @Override diff --git a/src/main/java/com/conveyal/analysis/util/JsonUtil.java b/src/main/java/com/conveyal/analysis/util/JsonUtil.java index 88c435842..04f11ef8d 100644 --- a/src/main/java/com/conveyal/analysis/util/JsonUtil.java +++ b/src/main/java/com/conveyal/analysis/util/JsonUtil.java @@ -3,8 +3,12 @@ import com.conveyal.analysis.models.JsonViews; import com.conveyal.geojson.GeoJsonModule; import com.conveyal.r5.model.json_serialization.JavaLocalDateSerializer; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.mongojack.internal.MongoJackModule; import spark.ResponseTransformer; @@ -33,4 +37,20 @@ public static ObjectMapper getObjectMapper(Class view, boolean configureMongoJac return objectMapper; } + public static String toJsonString (JsonNode node) { + try { + return objectMapper.writeValueAsString(node); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to write JSON.", e); + } + } + + public static ObjectNode objectNode () { + return objectMapper.createObjectNode(); + } + + public static ArrayNode arrayNode () { + return objectMapper.createArrayNode(); + } + } diff --git a/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java b/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java index 339ef1149..2e488a366 100644 --- a/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java +++ b/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java @@ -1,9 +1,11 @@ package com.conveyal.r5.analyst.scenario; import com.conveyal.analysis.components.WorkerComponents; +import com.conveyal.file.FileStorageKey; +import org.geotools.data.geojson.GeoJSONDataStore; +import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; -import org.geotools.geojson.feature.FeatureJSON; import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; @@ -16,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; @@ -104,9 +107,11 @@ public IndexedPolygonCollection ( } public void loadFromS3GeoJson() throws Exception { - InputStream polygonInputStream = WorkerComponents.fileStorage.getInputStream(POLYGONS, polygonLayer); - FeatureJSON featureJSON = new FeatureJSON(); - FeatureCollection featureCollection = featureJSON.readFeatureCollection(polygonInputStream); + // FIXME this needs to be adapted to new SpatialDataSource. How will we handle .gz data? + File polygonInputFile = WorkerComponents.fileStorage.getFile(new FileStorageKey(POLYGONS, polygonLayer)); + GeoJSONDataStore dataStore = new GeoJSONDataStore(polygonInputFile); + SimpleFeatureSource featureSource = dataStore.getFeatureSource(); + FeatureCollection featureCollection = featureSource.getFeatures(); LOG.info("Validating features and creating spatial index..."); FeatureType featureType = featureCollection.getSchema(); CoordinateReferenceSystem crs = featureType.getCoordinateReferenceSystem(); diff --git a/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java b/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java index f113a0565..d35559717 100644 --- a/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java +++ b/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java @@ -2,6 +2,7 @@ import com.conveyal.analysis.components.WorkerComponents; import com.conveyal.file.FileCategory; +import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.cluster.AnalysisWorker; import com.conveyal.r5.streets.EdgeStore; import com.conveyal.r5.transit.TransportNetwork; @@ -10,9 +11,10 @@ import gnu.trove.list.array.TShortArrayList; import gnu.trove.map.TObjectIntMap; import gnu.trove.map.hash.TObjectIntHashMap; +import org.geotools.data.geojson.GeoJSONDataStore; +import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; -import org.geotools.geojson.feature.FeatureJSON; import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; @@ -25,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.InputStream; import java.util.List; import java.util.zip.GZIPInputStream; @@ -110,10 +113,12 @@ public boolean resolve (TransportNetwork network) { // and errors can all be easily recorded and bubbled back up to the UI. // Polygon should only need to be fetched once when the scenario is applied, then the resulting network is cached. // this.features = polygonLayerCache.getPolygonFeatureCollection(this.polygonLayer); - // Note: Newer JTS now has GeoJsonReader - try (InputStream inputStream = WorkerComponents.fileStorage.getInputStream(POLYGONS, polygonLayer)) { - FeatureJSON featureJSON = new FeatureJSON(); - FeatureCollection featureCollection = featureJSON.readFeatureCollection(inputStream); + // TODO integrate this with new SpatialDataSource system + try { + File polygonInputFile = WorkerComponents.fileStorage.getFile(new FileStorageKey(POLYGONS, polygonLayer)); + GeoJSONDataStore dataStore = new GeoJSONDataStore(polygonInputFile); + SimpleFeatureSource featureSource = dataStore.getFeatureSource(); + FeatureCollection featureCollection = featureSource.getFeatures(); LOG.info("Validating features and creating spatial index..."); polygonSpatialIndex = new STRtree(); FeatureType featureType = featureCollection.getSchema(); diff --git a/src/main/java/com/conveyal/r5/streets/StreetLayer.java b/src/main/java/com/conveyal/r5/streets/StreetLayer.java index 65c7c7cfc..b7bfc8fe2 100644 --- a/src/main/java/com/conveyal/r5/streets/StreetLayer.java +++ b/src/main/java/com/conveyal/r5/streets/StreetLayer.java @@ -29,7 +29,6 @@ import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.map.hash.TLongIntHashMap; import gnu.trove.set.TIntSet; -import org.geotools.geojson.geom.GeometryJSON; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -1539,24 +1538,6 @@ public Geometry addedEdgesBoundingGeometry () { } } - /** - * Given a JTS Geometry in fixed-point latitude and longitude, log it as floating-point GeoJSON. - */ - public static void logFixedPointGeometry (String label, Geometry fixedPointGeometry) { - if (fixedPointGeometry == null){ - LOG.info("{} is null.", label); - } else if (fixedPointGeometry.isEmpty()) { - LOG.info("{} is empty.", label); - } else { - String geoJson = new GeometryJSON().toString(fixedDegreeGeometryToFloating(fixedPointGeometry)); - if (geoJson == null) { - LOG.info("Could not convert non-null geometry to GeoJSON"); - } else { - LOG.info("{} {}", label, geoJson); - } - } - } - /** * Finds all the P+R stations in given envelope. This might overselect (doesn't filter the objects from the * spatial index) but it's only used in visualizations. From 223c8e3ae43f7180449f39b913f3276d57c70be7 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 31 Aug 2021 23:27:52 +0800 Subject: [PATCH 094/187] make dataSourceId a query parameter also update some javadoc and avoid class constant in LOG initialization --- .../analysis/controllers/AggregationAreaController.java | 4 ++-- .../analysis/controllers/DataSourceController.java | 7 +++++-- .../java/com/conveyal/analysis/models/AggregationArea.java | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 7795840f1..774b170b3 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -84,7 +84,7 @@ public AggregationAreaController ( private List createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); UserPermissions userPermissions = UserPermissions.from(req); - String dataSourceId = req.params("dataSourceId"); + String dataSourceId = req.queryParams("dataSourceId"); String nameProperty = req.queryParams("nameProperty"); final int zoom = parseZoom(req.queryParams("zoom")); @@ -212,7 +212,7 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/region/", () -> { sparkService.get("/:regionId/aggregationArea", this::getAggregationAreas, toJson); sparkService.get("/:regionId/aggregationArea/:maskId", this::getAggregationArea, toJson); - sparkService.post("/:regionId/aggregationArea/:sourceId", this::createAggregationAreas, toJson); + sparkService.post("/:regionId/aggregationArea", this::createAggregationAreas, toJson); }); } diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 8c4a9abaf..47277d782 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -19,6 +19,8 @@ import spark.Request; import spark.Response; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; import java.util.List; import java.util.Map; @@ -31,11 +33,11 @@ /** * Controller that handles CRUD of DataSources, which are Mongo metadata about user-uploaded files. * Unlike some Mongo documents, these are mostly created and updated by backend validation and processing methods. - * Currently this handles only one subtype: SpatialDataSource, which represents GIS-like geospatial feature data. + * Currently this handles only one subtype: SpatialDataSource, which represents GIS-like vector geospatial data. */ public class DataSourceController implements HttpController { - private static final Logger LOG = LoggerFactory.getLogger(DataSourceController.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); // Component Dependencies private final FileStorage fileStorage; @@ -96,6 +98,7 @@ private SpatialDataSource downloadLODES(Request req, Response res) { .withWorkProduct(source) .withAction((progressListener) -> { // TODO implement + throw new UnsupportedOperationException(); })); return source; diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index 8c7d9c43c..05ebe39b1 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -21,7 +21,7 @@ private AggregationArea(UserPermissions user, String name) { // FLUENT METHODS FOR CONFIGURING - /** Call this static factory to begin building a task. */ + /** Call this static factory to begin building an AggregationArea. */ public static AggregationArea create (UserPermissions user, String name) { return new AggregationArea(user, name); } From 8028f4e6ce537164c884d41300ce8ff2a3f89ca8 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Thu, 2 Sep 2021 09:26:31 +0800 Subject: [PATCH 095/187] Add comma to cache control string --- .../java/com/conveyal/analysis/controllers/GTFSController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index 21d17f12d..12a8b2ed1 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -43,7 +43,7 @@ public GTFSController (GTFSCache gtfsCache) { * Use the same Cache-Control header for each endpoint here. 2,592,000 seconds is one month. * https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control */ - private final String cacheControlImmutable = "public, max-age=2592000 immutable"; + private final String cacheControlImmutable = "public, max-age=2592000, immutable"; /** * Extracted into a common method to allow turning off during development. From 5a8bbd17bbe66201f6da97de526450f93dd3a4c1 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 2 Sep 2021 16:34:20 +0800 Subject: [PATCH 096/187] correct to AnalysisCollection perform additional type checking. also rename resource to spatialDataSource. --- .../AggregationAreaController.java | 32 ++++++++++++------- .../analysis/persistence/AnalysisDB.java | 1 + 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 774b170b3..dfb3b460c 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -40,6 +40,7 @@ import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; +import static com.google.common.base.Preconditions.checkArgument; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -60,7 +61,10 @@ public class AggregationAreaController implements HttpController { private final FileStorage fileStorage; private final TaskScheduler taskScheduler; private final AnalysisCollection aggregationAreaCollection; - private final AnalysisCollection dataSourceCollection; + + // FIXME Should we instead be using the same instance as the DataSourceController? + // Anyway the parameterized type is too specific. + private final AnalysisCollection dataSourceCollection; public AggregationAreaController ( FileStorage fileStorage, @@ -89,14 +93,20 @@ private List createAggregationAreas (Request req, Response res) final int zoom = parseZoom(req.queryParams("zoom")); // 1. Get file from storage and read its features. ============================================================= - SpatialDataSource resource = dataSourceCollection.findById(dataSourceId); - Preconditions.checkArgument(POLYGON.equals(resource.geometryType), - "Only polygons can be converted to aggregation areas."); + DataSource dataSource = dataSourceCollection.findById(dataSourceId); + checkArgument(dataSource instanceof SpatialDataSource, + "Only spatial data sets can be converted to aggregation areas."); + SpatialDataSource spatialDataSource = (SpatialDataSource) dataSource; + checkArgument(POLYGON.equals(spatialDataSource.geometryType), + "Only polygons can be converted to aggregation areas. DataSource is: " + spatialDataSource.geometryType); + checkArgument(SHP.equals(spatialDataSource.fileFormat), + "Currently, only shapefiles can be converted to aggregation areas."); + File sourceFile; List features = null; - if (SHP.equals(resource.fileFormat)) { - sourceFile = fileStorage.getFile(resource.storageKey()); + if (SHP.equals(spatialDataSource.fileFormat)) { + sourceFile = fileStorage.getFile(spatialDataSource.storageKey()); ShapefileReader reader = null; try { reader = new ShapefileReader(sourceFile); @@ -106,15 +116,15 @@ private List createAggregationAreas (Request req, Response res) } } - if (GEOJSON.equals(resource.fileFormat)) { + if (GEOJSON.equals(spatialDataSource.fileFormat)) { // TODO implement } List finalFeatures = features; - taskScheduler.enqueue(Task.create("Aggregation area creation: " + resource.name) + taskScheduler.enqueue(Task.create("Aggregation area creation: " + spatialDataSource.name) .forUser(userPermissions) .setHeavy(true) - .withWorkProduct(resource) + .withWorkProduct(spatialDataSource) .withAction(progressListener -> { progressListener.beginTask("Processing request", 1); Map areas = new HashMap<>(); @@ -133,7 +143,7 @@ private List createAggregationAreas (Request req, Response res) ); UnaryUnionOp union = new UnaryUnionOp(geometries); // Name the area using the name in the request directly - areas.put(resource.name, union.union()); + areas.put(spatialDataSource.name, union.union()); } else { // Don't union. Name each area by looking up its value for the name property in the request. finalFeatures.forEach(f -> areas.put( @@ -156,7 +166,7 @@ private List createAggregationAreas (Request req, Response res) }); AggregationArea aggregationArea = AggregationArea.create(userPermissions, name) - .withSource(resource); + .withSource(spatialDataSource); try { File gridFile = FileUtils.createScratchFile("grid"); diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 409838510..956f6ba53 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -18,6 +18,7 @@ import static org.bson.codecs.configuration.CodecRegistries.fromProviders; import static org.bson.codecs.configuration.CodecRegistries.fromRegistries; +/** TODO should we pre-create all the AnalysisCollections and fetch them by Class? */ public class AnalysisDB { private final Logger LOG = LoggerFactory.getLogger(AnalysisDB.class); From 57726eb09f5e435adade5454202dd74a75ac7e64 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 2 Sep 2021 22:36:33 +0800 Subject: [PATCH 097/187] create dataGroups when making aggregationAreas dataGroups are stored in mongo, referenced in the aggregationArea records via a dataGroupId property, and referenced in WorkProduct via an isGroup boolean property. --- .../AggregationAreaController.java | 72 ++++++++++++------- .../analysis/models/AggregationArea.java | 27 ++++--- .../conveyal/analysis/models/BaseModel.java | 1 + .../conveyal/analysis/models/DataGroup.java | 20 ++++++ .../persistence/AnalysisCollection.java | 4 ++ .../r5/analyst/progress/ProgressListener.java | 4 +- .../conveyal/r5/analyst/progress/Task.java | 14 ++-- .../r5/analyst/progress/WorkProduct.java | 16 ++++- 8 files changed, 108 insertions(+), 50 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/models/DataGroup.java diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index dfb3b460c..962baef69 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -4,16 +4,22 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.models.AggregationArea; +import com.conveyal.analysis.models.DataGroup; import com.conveyal.analysis.models.DataSource; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; +import com.conveyal.analysis.util.JsonUtil; import com.conveyal.file.FileStorage; import com.conveyal.file.FileUtils; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.progress.Task; +import com.conveyal.r5.analyst.progress.WorkProduct; +import com.conveyal.r5.analyst.progress.WorkProductType; import com.conveyal.r5.util.ShapefileReader; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; +import org.bson.conversions.Bson; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.operation.union.UnaryUnionOp; @@ -39,6 +45,7 @@ import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.conveyal.r5.analyst.progress.WorkProductType.AGGREGATION_AREA; import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; import static com.google.common.base.Preconditions.checkArgument; import static com.mongodb.client.model.Filters.and; @@ -66,6 +73,9 @@ public class AggregationAreaController implements HttpController { // Anyway the parameterized type is too specific. private final AnalysisCollection dataSourceCollection; + // TODO review July 1 notes + private final AnalysisCollection dataGroupCollection; + public AggregationAreaController ( FileStorage fileStorage, AnalysisDB database, @@ -75,23 +85,28 @@ public AggregationAreaController ( this.taskScheduler = taskScheduler; this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); this.dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); + this.dataGroupCollection = database.getAnalysisCollection("dataGroups", DataGroup.class); } /** * Create binary .grid files for aggregation (aka mask) areas, save them to S3, and persist their details. * @param req Must include a shapefile on which the aggregation area(s) will be based. + * --NOTE-- behavior has changed. union parameter is not specified. Now union = (nameProperty == null). * If HTTP query parameter union is "true", features will be merged to a single aggregation area, named * using the value of the "name" query parameter. If union is false or if the parameter is missing, each * feature will be a separate aggregation area, named using the value for the shapefile property * specified by the HTTP query parameter "nameAttribute." + * @return the Task representing the background action of creating the aggregation areas, or its ID. */ - private List createAggregationAreas (Request req, Response res) throws Exception { + private Task createAggregationAreas (Request req, Response res) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); UserPermissions userPermissions = UserPermissions.from(req); String dataSourceId = req.queryParams("dataSourceId"); - String nameProperty = req.queryParams("nameProperty"); + //String nameProperty = req.queryParams("nameProperty"); + final String nameProperty = "dist_name"; final int zoom = parseZoom(req.queryParams("zoom")); + // 1. Get file from storage and read its features. ============================================================= DataSource dataSource = dataSourceCollection.findById(dataSourceId); checkArgument(dataSource instanceof SpatialDataSource, @@ -120,13 +135,16 @@ private List createAggregationAreas (Request req, Response res) // TODO implement } - List finalFeatures = features; - taskScheduler.enqueue(Task.create("Aggregation area creation: " + spatialDataSource.name) + final List finalFeatures = features; + Task backgroundTask = Task.create("Aggregation area creation: " + spatialDataSource.name) .forUser(userPermissions) .setHeavy(true) - .withWorkProduct(spatialDataSource) .withAction(progressListener -> { - progressListener.beginTask("Processing request", 1); + String groupDescription = "Convert polygons to aggregation areas, " + + ((nameProperty == null) ? "merging all polygons." : "one area per polygon."); + DataGroup dataGroup = new DataGroup(userPermissions, spatialDataSource._id.toString(), groupDescription); + + progressListener.beginTask("Reading data source", finalFeatures.size() + 1); Map areas = new HashMap<>(); if (nameProperty != null && finalFeatures.size() > MAX_FEATURES) { @@ -156,38 +174,39 @@ private List createAggregationAreas (Request req, Response res) if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); Envelope env = geometry.getEnvelopeInternal(); Grid maskGrid = new Grid(zoom, env); - progressListener.beginTask("Creating grid for " + name, maskGrid.featureCount()); + progressListener.beginTask("Creating grid for " + name, 0); // Store the percentage each cell overlaps the mask, scaled as 0 to 100,000 List weights = maskGrid.getPixelWeights(geometry, true); weights.forEach(pixel -> { maskGrid.grid[pixel.x][pixel.y] = pixel.weight * 100_000; - progressListener.increment(); }); - AggregationArea aggregationArea = AggregationArea.create(userPermissions, name) - .withSource(spatialDataSource); + AggregationArea aggregationArea = new AggregationArea(userPermissions, name, spatialDataSource); try { File gridFile = FileUtils.createScratchFile("grid"); OutputStream os = new GZIPOutputStream(FileUtils.getOutputStream(gridFile)); maskGrid.write(os); os.close(); - - aggregationAreaCollection.insert(aggregationArea); + aggregationArea.dataGroupId = dataGroup._id.toString(); aggregationAreas.add(aggregationArea); - fileStorage.moveIntoStorage(aggregationArea.getStorageKey(), gridFile); } catch (IOException e) { throw new AnalysisServerException("Error processing/uploading aggregation area"); } progressListener.increment(); }); - }) - ); - - return aggregationAreas; - + aggregationAreaCollection.insertMany(aggregationAreas); + dataGroupCollection.insert(dataGroup); + progressListener.setWorkProduct(WorkProduct.forDataGroup( + AGGREGATION_AREA, dataGroup._id.toString(), dataSource.regionId) + ); + progressListener.increment(); + }); + + taskScheduler.enqueue(backgroundTask); + return backgroundTask; } private String readProperty (SimpleFeature feature, String propertyName) { @@ -201,20 +220,21 @@ private String readProperty (SimpleFeature feature, String propertyName) { } private Collection getAggregationAreas (Request req, Response res) { - return aggregationAreaCollection.findPermitted( - eq("regionId", req.queryParams("regionId")), UserPermissions.from(req) - ); + Bson query = eq("regionId", req.queryParams("regionId")); + String dataGroupId = req.queryParams("dataGroupId"); + if (dataGroupId != null) { + query = and(eq("dataGroupId", dataGroupId), query); + } + return aggregationAreaCollection.findPermitted(query, UserPermissions.from(req)); } - private Map getAggregationArea (Request req, Response res) { + private ObjectNode getAggregationArea (Request req, Response res) { AggregationArea aggregationArea = aggregationAreaCollection.findByIdIfPermitted( req.params("maskId"), UserPermissions.from(req) ); String url = fileStorage.getURL(aggregationArea.getStorageKey()); - // Put the URL into something that will be serialized as a JSON object with a single property. - Map wrappedUrl = new HashMap<>(); - wrappedUrl.put("url", url); - return wrappedUrl; + return JsonUtil.objectNode().put("url", url); + } @Override diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index 05ebe39b1..b1cdc9bc4 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -3,6 +3,7 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.file.FileStorageKey; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.bson.codecs.pojo.annotations.BsonIgnore; import static com.conveyal.file.FileCategory.GRIDS; @@ -12,32 +13,28 @@ * depending on how much of that pixel is overlapped by the mask. */ public class AggregationArea extends BaseModel { - public String regionId; - public String sourceId; - - private AggregationArea(UserPermissions user, String name) { - super(user, name); - } - // FLUENT METHODS FOR CONFIGURING + public String regionId; + public String dataSourceId; + public String dataGroupId; - /** Call this static factory to begin building an AggregationArea. */ - public static AggregationArea create (UserPermissions user, String name) { - return new AggregationArea(user, name); - } + /** Zero-argument constructor required for Mongo automatic POJO deserialization. */ + public AggregationArea () { } - public AggregationArea withSource (SpatialDataSource source) { - this.regionId = source.regionId; - this.sourceId = source._id.toString(); - return this; + public AggregationArea(UserPermissions user, String name, SpatialDataSource dataSource) { + super(user, name); + this.regionId = dataSource.regionId; + this.dataSourceId = dataSource._id.toString(); } @JsonIgnore + @BsonIgnore public String getS3Key () { return String.format("%s/mask/%s.grid", regionId, _id); } @JsonIgnore + @BsonIgnore public FileStorageKey getStorageKey () { // These in the GRIDS file storage category because aggregation areas are masks represented as binary grids. return new FileStorageKey(GRIDS, getS3Key()); diff --git a/src/main/java/com/conveyal/analysis/models/BaseModel.java b/src/main/java/com/conveyal/analysis/models/BaseModel.java index 10a1bf951..37562f7df 100644 --- a/src/main/java/com/conveyal/analysis/models/BaseModel.java +++ b/src/main/java/com/conveyal/analysis/models/BaseModel.java @@ -3,6 +3,7 @@ import com.conveyal.analysis.UserPermissions; import org.bson.types.ObjectId; +/** The base type for objects stored in our newer AnalysisDB using the Mongo Java driver's POJO functionality. */ public class BaseModel { // Can retrieve `createdAt` from here public ObjectId _id; diff --git a/src/main/java/com/conveyal/analysis/models/DataGroup.java b/src/main/java/com/conveyal/analysis/models/DataGroup.java new file mode 100644 index 000000000..6cf22a15c --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/DataGroup.java @@ -0,0 +1,20 @@ +package com.conveyal.analysis.models; + +import com.conveyal.analysis.UserPermissions; +import org.bson.types.ObjectId; + +/** + * When deriving other data (layers, networks, etc.) from a DataSource, we sometimes produce many outputs at once from + * the same source and configuration options. We group all those derived products together using a DataGroup. + */ +public class DataGroup extends BaseModel { + + /** The data source this group of products was derived from. */ + public String dataSourceId; + + public DataGroup (UserPermissions user, String dataSourceId, String description) { + super(user, description); + this.dataSourceId = dataSourceId; + } + +} diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index 2bd54d544..ed85cd847 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -97,6 +97,10 @@ public void insert (T model) { collection.insertOne(model); } + public void insertMany (List models) { + collection.insertMany(models); + } + public T update(T value) { return update(value, value.accessGroup); } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java b/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java index 47fd4d1f2..899ede322 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/ProgressListener.java @@ -8,7 +8,8 @@ public interface ProgressListener { /** * Call this method once at the beginning of a new task, specifying how many sub-units of work will be performed. - * This does not allow for subsequently starting sub-tasks that use the same ProgressListener while progress is + * If totalElements is zero or negative, any previously set total number of elements remains unchanged. + * This allows for subsequently starting named sub-tasks that use the same ProgressListener while progress is * still being reported. Any recursion launching sub-tasks will need to be head- or tail-recursion, launched before * you call beginTask or after the last unit of work is complete. * Rather than implementing some kind of push/pop mechanism, we may eventually have some kind of nested task system, @@ -31,6 +32,7 @@ default void increment () { * unexpected exception. Adding them to the background Task with a fluent method is also problematic as it requires * the caller to construct or otherwise hold a reference to the product to get its ID before the action is run. It's * preferable for the product to be fully encapsulated in the action, so it's reported as park of the task progress. + * On the other hand, creating the product within the TaskAction usually requires it to hold a UserPermissions. */ default void setWorkProduct (WorkProduct workProduct) { /* Default is no-op */ } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index b8401849e..d6376dc92 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -106,7 +106,7 @@ public double getPercentComplete() { List subtasks = new ArrayList<>(); - // TODO find a better way to set this than directly inside a closure + // TODO find a better way to set this than directly inside a closure - possibly using ProgressListener public WorkProduct workProduct; public void addSubtask (Task subtask) { @@ -188,11 +188,13 @@ public void run () { @Override public void beginTask(String description, int totalElements) { - // In the absence of subtasks we can call this repeatedly on the same task, which will cause the UI progress - // bar to reset to zero at each stage, while keeping the same top level title. + // In the absence of a real subtask mechanism, we can call this repeatedly on the same task with totalElements + // of zero, allow the UI progress while changing the detail message without resetting progress to zero. this.description = description; - this.totalWorkUnits = totalElements; - this.currentWorkUnit = 0; + if (totalElements > 0) { + this.totalWorkUnits = totalElements; + this.currentWorkUnit = 0; + } } @Override @@ -252,6 +254,8 @@ public Task withAction (TaskAction action) { // We can't return the WorkProduct from TaskAction, that would be disrupted by throwing exceptions. // It is also awkward to make a method to set it on ProgressListener - it's not really progress. // So we set it directly on the task before submitting it. Requires pre-setting (not necessarily storing) Model._id. + // Update: I've started setting it via progressListener, it's just more encapsulated to create inside the TaskAction. + // But this then requires the TaskAction to hold a UserPermissions instance. public Task withWorkProduct (BaseModel model) { this.workProduct = WorkProduct.forModel(model); return this; diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java index e9a9f9761..1df162014 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java @@ -3,20 +3,27 @@ import com.conveyal.analysis.models.BaseModel; /** - * A unique identifier for the final product of a single task action. Currently this serves as both an - * internal data structure and an API model class, which should be harmless as it's an immutable data class. - * The id is unique within the type, so the regionId is redundant information, but facilitates prefectches on the UI. + * A unique identifier for the final product of a single task action. Currently this serves as both an internal data + * structure and an API model class, which should be harmless as it's an immutable data class. The id is unique within + * the type, so the regionId is redundant information, but facilitates prefectches on the UI. If isGroup is true, the + * id is not that of an individual record, but the dataGroupId of several records created in a single operation. */ public class WorkProduct { public final WorkProductType type; public final String id; public final String regionId; + public final boolean isGroup; public WorkProduct (WorkProductType type, String id, String regionId) { + this(type, id, regionId, false); + } + + public WorkProduct (WorkProductType type, String id, String regionId, boolean isGroup) { this.type = type; this.id = id; this.regionId = regionId; + this.isGroup = isGroup; } // FIXME Not all Models have a regionId. Rather than pass that in as a String, refine the programming API. @@ -24,4 +31,7 @@ public static WorkProduct forModel (BaseModel model) { return new WorkProduct(WorkProductType.forModel(model), model._id.toString(), null); } + public static WorkProduct forDataGroup (WorkProductType type, String dataGroupId, String regionId) { + return new WorkProduct(type, dataGroupId, regionId, true); + } } From 9b4cabdd7c2e576ee21f0d884a37157d40e79a63 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 2 Sep 2021 23:22:13 +0800 Subject: [PATCH 098/187] responses to PR review --- .../datasource/CsvDataSourceIngester.java | 17 +---------------- .../analysis/datasource/DataSourceIngester.java | 13 +++++++++---- .../datasource/DataSourceUploadAction.java | 5 ++--- .../datasource/GeoJsonDataSourceIngester.java | 3 --- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java index 11ad2f080..b0dc8778c 100644 --- a/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java @@ -35,22 +35,7 @@ public CsvDataSourceIngester () { @Override public void ingest (File file, ProgressListener progressListener) { - progressListener.beginTask("Scanning CSV file", 1); - try { - // TODO logic based on FreeFormPointSet.fromCsv() and Grid.fromCsv() - ShapefileReader reader = new ShapefileReader(null); - Envelope envelope = reader.wgs84Bounds(); - checkWgsEnvelopeSize(envelope); - dataSource.wgsBounds = Bounds.fromWgsEnvelope(envelope); - dataSource.attributes = reader.attributes(); - dataSource.geometryType = reader.geometryType(); - dataSource.featureCount = reader.featureCount(); - } catch (FactoryException | TransformException e) { - throw new RuntimeException("Shapefile transform error. Try uploading an unprojected (EPSG:4326) file.", e); - } catch (Exception e) { - // Must catch because ShapefileReader throws a checked IOException. - throw new RuntimeException("Error parsing shapefile. Ensure the files you uploaded are valid.", e); - } + throw new UnsupportedOperationException(); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index 9ef67d945..e7e37cce0 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static com.conveyal.file.FileStorageFormat.GEOPACKAGE; import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.file.FileStorageFormat.TIFF; @@ -44,7 +45,9 @@ public abstract class DataSourceIngester { * DataSourceIngester taking care of the rest. * Our no-arg BaseModel constructors are used for deserialization so they don't create an _id or nonce ObjectId(); */ - public void initializeDataSource (String name, String description, String regionId, UserPermissions userPermissions) { + public void initializeDataSource ( + String name, String originalFileNames, String regionId, UserPermissions userPermissions + ) { DataSource dataSource = dataSource(); dataSource._id = new ObjectId(); dataSource.nonce = new ObjectId(); @@ -53,7 +56,8 @@ public void initializeDataSource (String name, String description, String region dataSource.createdBy = userPermissions.email; dataSource.updatedBy = userPermissions.email; dataSource.accessGroup = userPermissions.accessGroup; - dataSource.description = description; + dataSource.originalFileName = originalFileNames; + dataSource.description = "From uploaded files: " + originalFileNames; } /** @@ -66,9 +70,10 @@ public static DataSourceIngester forFormat (FileStorageFormat format) { return new GeoJsonDataSourceIngester(); } else if (format == TIFF) { // really this enum value should be GEOTIFF rather than just TIFF. return new GeoTiffDataSourceIngester(); - } else { + } else if (format == GEOPACKAGE) { return new GeoPackageDataSourceIngester(); } - + throw new UnsupportedOperationException("Unknown file format: " + format.name()); } + } diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 92880df9d..24a433ec4 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -131,9 +131,8 @@ public static DataSourceUploadAction forFormFields ( FileStorageFormat format = detectUploadFormatAndValidate(fileItems); DataSourceIngester ingester = DataSourceIngester.forFormat(format); - String description = "From uploaded files: " + fileItems.stream() - .map(FileItem::getName).collect(Collectors.joining(", ")); - ingester.initializeDataSource(sourceName, description, regionId, userPermissions); + String originalFileNames = fileItems.stream().map(FileItem::getName).collect(Collectors.joining(", ")); + ingester.initializeDataSource(sourceName, originalFileNames, regionId, userPermissions); DataSourceUploadAction dataSourceUploadAction = new DataSourceUploadAction(fileStorage, dataSourceCollection, fileItems, ingester); diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 44b541acb..2a64002c5 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -47,9 +47,6 @@ * capabilities. This is just support for an obsolete feature and should not be invoked. We instead range check all * incoming coordinates (via a total bounding box check) to ensure they look reasonable in WGS84. * - * Current stable Geotools documentation (version 25?) shows a GeoJSONFeatureSource and CSVFeatureSource. - * We're using 21.2 which does not have these. - * * In GeoTools FeatureSource is a read-only mechanism but it can apparently only return FeatureCollections, which load * everything into memory. FeatureReader provides iterator-style access, but seems quite low-level and not intended * for regular use. Because we limit the size of file uploads we can be fairly sure it will be harmless for the backend From f90ea981c452cb6c48d74e5322aa352986652658 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 13:33:54 +0800 Subject: [PATCH 099/187] clean up javadoc. improve error messages. --- .../datasource/GeoJsonDataSourceIngester.java | 104 +++++++++--------- .../conveyal/r5/analyst/progress/Task.java | 7 +- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 2a64002c5..79d2d9987 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -6,6 +6,7 @@ import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader; +import com.conveyal.r5.util.ShapefileReader.GeometryType; import org.geotools.data.Query; import org.geotools.data.geojson.GeoJSONDataStore; import org.geotools.data.simple.SimpleFeatureSource; @@ -26,59 +27,61 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import static com.conveyal.analysis.models.DataSourceValidationIssue.Level.ERROR; import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; /** * Logic to create SpatialDataSource metadata from an uploaded GeoJSON file and perform validation. - * We are using the (unsupported) GeoTools module for loading GeoJSON into a FeatureCollection of OpenGIS Features. - * However, GeoJSON deviates significantly from usual GIS concepts. In a GeoJSON feature collection, - * every single object can have a different geometry type and different properties. - * GeoJSON is always in WGS84, which is also how we handle things internally, so we can avoid any CRS transform logic. + * * GeoJSON geometries are JSON objects with a type property (Point, LineString, Polygon, MultiPoint, MultiPolygon, * or MultiLineString) and an array of coordinates. The "multi" types simply have another level of nested arrays. * Geometries are usually nested into objects of type "Feature", which allows attaching properties. Features can be * further nested into a top-level object of type FeatureCollection. We only support GeoJSON whose top level object is * a FeatureCollection (not a single Feature or a single Geometry), and where every geometry is of the same type. * - * Section 4 of the GeoJSON RFC at https://datatracker.ietf.org/doc/html/rfc7946#section-4 defines the only acceptable - * coordinate reference system as WGS84. You may notice some versions of the GeoTools GeoJSON handler have CRS parsing - * capabilities. This is just support for an obsolete feature and should not be invoked. We instead range check all - * incoming coordinates (via a total bounding box check) to ensure they look reasonable in WGS84. + * For consistency with other vector data sources and our internal geometry representation, we are using the + * (unsupported) GeoTools module gt-geojsondatasource for loading GeoJSON into a FeatureCollection of OpenGIS Features. * - * In GeoTools FeatureSource is a read-only mechanism but it can apparently only return FeatureCollections, which load - * everything into memory. FeatureReader provides iterator-style access, but seems quite low-level and not intended - * for regular use. Because we limit the size of file uploads we can be fairly sure it will be harmless for the backend - * to load any data fully into memory. Streaming capabilities can be added later if the need arises. - * This is explained well at: https://docs.geotools.org/stable/userguide/tutorial/datastore/read.html - * The datastore.getFeatureReader() idiom used in our ShapefileReader class seems to be the right way to stream. - * But it seems unecessary to go through the steps we do steps - our ShapfileReader creates a FeatureSource and FeatureCollection - * in memory. Actually we're doing the same thing in ShapefileMain but worse - supplying a query when there is a - * parameter-less method to call. - * - * As of summer 2021, the unsupported module gt-geojson (package org.geotools.geojson) is deprecated and has been - * replaced with gt-geojsondatastore (package org.geotools.data.geojson), which is on track to supported module status. - * The newer module uses Jackson instead of an abandoned JSON library, and uses standard GeoTools DataStore interfaces. - * We also have our own com.conveyal.geojson.GeoJsonModule which should be phased out if GeoTools support is sufficient. + * This is somewhat problematic because GeoJSON does not adhere to some common GIS principles. For example, in a + * GeoJSON feature collection, every single object can have a different geometry type and different properties, or + * even properties with the same name but different data types. For simplicity we only support GeoJSON inputs that + * have a consistent schema across all features - the same geometry type and attribute types on every feature. We + * still need to verify these constraints ourselves, as GeoTools does not enforce them. * - * They've also got flatbuf and geobuf modules - can we replace our custom one? + * It is notable that GeoTools will not work correctly with GeoJSON input that does not respect these constraints, but + * it does not detect or report those problems - it just fails silently. For example, GeoTools will report the same + * property schema for every feature in a FeatureCollection. If a certain property is reported as having an integer + * numeric type, but a certain feature has text in the attribute of that same name, the reported value will be an + * Integer object with a value of zero, not a String. * - * Note that GeoTools featureReader queries have setCoordinateSystemReproject method - we don't need to do the - * manual reprojection in our ShapefileReader as we currently are. + * This behavior is odd, but remember that the gt-geojsondatastore module is unsupported (though apparently on the path + * to being supported) and was only recently included in Geotools releases. We may want to make code contributions to + * Geotools to improve JSON validation and error reporting. * - * Allow Attributes to be of "AMBIGUOUS" or null type, or just drop them if they're ambiguous. - * Flag them as hasMissingValues, or the quantity of missing values. + * Section 4 of the GeoJSON RFC at https://datatracker.ietf.org/doc/html/rfc7946#section-4 defines the only acceptable + * coordinate reference system as WGS84. You may notice older versions of the GeoTools GeoJSON handler have CRS parsing + * capabilities. This is just support for an obsolete feature and should not be invoked. We instead range check all + * incoming coordinates (via a total bounding box check) to ensure they look reasonable in WGS84. * - * Be careful, QGIS will happily export GeoJSON with a CRS property which is no longer allowed by the GeoJSON spec: - * "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::3857" } } - * If a CRS is present, make sure it matches one of the names for WGS84. Throw a warning if the field is present at all. + * Note that QGIS will happily and silently export GeoJSON with a crs field, which gt-geojsondatastore will happily + * read and report that it's in WGS84 without ever looking at the crs field. This is another case where Geotools would + * seriously benefit from added validation and error reporting, and where we need to add stopgap validation of our own. * - * See also: com.conveyal.r5.analyst.scenario.IndexedPolygonCollection#loadFromS3GeoJson() + * In GeoTools, FeatureSource is a read-only mechanism but it can apparently only return FeatureCollections which load + * everything into memory. FeatureReader provides iterator-style access, but seems quite low-level and not intended + * for regular use. Because we limit the size of file uploads we can be fairly sure it will be harmless for the backend + * to load any data fully into memory. Feature streaming capabilities and/or streaming JSON decoding can be added later + * if the need arises. The use of FeatureReader and FeatureSource are explained well at: + * https://docs.geotools.org/stable/userguide/tutorial/datastore/read.html */ public class GeoJsonDataSourceIngester extends DataSourceIngester { - public static final int MIN_GEOJSON_FILE_LENGTH = "{'type':'GeometryCollection','features':[]}".length(); + public static final int MIN_GEOJSON_FILE_LENGTH = "{'type':'FeatureCollection','features':[]}".length(); private final SpatialDataSource dataSource; @@ -97,10 +100,9 @@ public GeoJsonDataSourceIngester () { @Override public void ingest (File file, ProgressListener progressListener) { - progressListener.beginTask("Processing and validating uploaded GeoJSON", 2); + progressListener.beginTask("Processing and validating uploaded GeoJSON", 1); progressListener.setWorkProduct(dataSource.toWorkProduct()); - // Check that file exists and is not empty. - // Geotools GeoJson reader fails with stack overflow on empty/missing file. TODO: File GeoTools issue. + // Check that file exists and is not empty. Geotools reader fails with stack overflow on empty/missing file. if (!file.exists()) { throw new IllegalArgumentException("File does not exist: " + file.getPath()); } @@ -127,7 +129,7 @@ public void ingest (File file, ProgressListener progressListener) { } // The schema always reports the geometry type as the very generic "Geometry" class. // Check that all features have the same concrete Geometry type. - Class firstGeometryType = null; + Set> geometryClasses = new HashSet<>(); FeatureIterator iterator = wgsFeatureCollection.features(); while (iterator.hasNext()) { SimpleFeature feature = iterator.next(); @@ -136,36 +138,40 @@ public void ingest (File file, ProgressListener progressListener) { dataSource.addIssue(ERROR, "Geometry is null on feature: " + feature.getID()); continue; } - if (firstGeometryType == null) { - firstGeometryType = geometry.getClass(); - } else if (firstGeometryType != geometry.getClass()) { - dataSource.addIssue(ERROR, "Inconsistent geometry type on feature: " + feature.getID()); - continue; - } + geometryClasses.add(geometry.getClass()); } - progressListener.increment(); checkCrs(featureType); Envelope wgsEnvelope = wgsFeatureCollection.getBounds(); checkWgsEnvelopeSize(wgsEnvelope); + // Set SpatialDataSource fields (Conveyal metadata) from GeoTools model dataSource.wgsBounds = Bounds.fromWgsEnvelope(wgsEnvelope); - // Cannot set from FeatureType because it's always Geometry for GeoJson. - dataSource.geometryType = ShapefileReader.GeometryType.forBindingClass(firstGeometryType); dataSource.featureCount = wgsFeatureCollection.size(); + // Cannot set geometry type based on FeatureType.getGeometryDescriptor() because it's always just Geometry + // for GeoJson. We will leave the type null if there are zero or multiple geometry types present. + List geometryTypes = + geometryClasses.stream().map(GeometryType::forBindingClass).collect(Collectors.toList()); + if (geometryTypes.isEmpty()) { + dataSource.addIssue(ERROR, "No geometry types are present."); + } else if (geometryTypes.size() > 1) { + dataSource.addIssue(ERROR, "Multiple geometry types present: " + geometryTypes); + } else { + dataSource.geometryType = geometryTypes.get(0); + } progressListener.increment(); } catch (FactoryException | IOException e) { - // Unexpected errors cause immediate failure; predictable issues will be recorded on the DataSource object. - // Catch only checked exceptions to preserve the top-level exception type when possible. + // Catch only checked exceptions to avoid excessive wrapping of root cause exception when possible. throw new DataSourceException("Error parsing GeoJSON. Please ensure the files you uploaded are valid."); } } /** * GeoJSON used to allow CRS, but the RFC now says GeoJSON is always in WGS84 and no other CRS are allowed. - * QGIS and GeoTools both seem to support this, but it's an obsolete feature. - * FIXME this is never failing, even on projected input. The GeoTools reader seems to silently convert to WGS84. + * QGIS and GeoTools both seem to support crs fields, but it's an obsolete feature. */ private static void checkCrs (FeatureType featureType) throws FactoryException { + // FIXME newer GeoTools always reports WGS84 even when crs field is present. + // It doesn't report the problem or attempt any reprojection. CoordinateReferenceSystem crs = featureType.getCoordinateReferenceSystem(); if (crs != null && !DefaultGeographicCRS.WGS84.equals(crs) && !CRS.decode("CRS:84").equals(crs)) { throw new DataSourceException("GeoJSON should specify no coordinate reference system, and contain " + diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index d6376dc92..e248a1b75 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -11,9 +11,10 @@ import java.util.UUID; /** - * This is a draft for a more advanced task progress system. It is not yet complete. - * Task is intended for background tasks whose progress the end user should be aware of, such as file uploads. - * It should not be used for automatic internal actions (such as Events) which would clutter a user's active task list. + * This newer task progress system coexists with several older mechanisms. It is still evolving as we adapt it to new + * use cases. The Task class is intended to represent background tasks whose progress the end user should be aware of, + * such as processing uploaded files. It should not be used for automatic internal actions (such as Events) which would + * clutter a user's active task list. * * A Task (or some interface that it implements) could be used by the AsyncLoader to track progress. Together with some * AsyncLoader functionality it will be a bit like a Future with progress reporting. Use of AsyncLoader could then be From 616d9929e12a4cc4a49ca027f7e287ba68a4ca14 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 15:56:55 +0800 Subject: [PATCH 100/187] update javadoc and whitespace --- .../analysis/controllers/DataSourceController.java | 1 + .../conveyal/analysis/models/AggregationArea.java | 10 +++++++--- .../com/conveyal/analysis/models/DataGroup.java | 5 ++++- .../conveyal/analysis/models/SpatialDataSource.java | 13 ++----------- .../java/com/conveyal/r5/analyst/progress/Task.java | 8 ++++---- .../conveyal/r5/analyst/progress/TaskAction.java | 3 +++ .../conveyal/r5/analyst/progress/WorkProduct.java | 2 +- 7 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 47277d782..376cfafcb 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -58,6 +58,7 @@ public DataSourceController ( this.extractor = extractor; // We don't hold on to the AnalysisDB Component, just get one collection from it. // Register all the subclasses so the Mongo driver will recognize their discriminators. + // TODO should this be done once in AnalysisDB and the collection reused everywhere? Is that threadsafe? this.dataSourceCollection = database.getAnalysisCollection( "dataSources", DataSource.class, SpatialDataSource.class, OsmDataSource.class, GtfsDataSource.class ); diff --git a/src/main/java/com/conveyal/analysis/models/AggregationArea.java b/src/main/java/com/conveyal/analysis/models/AggregationArea.java index b1cdc9bc4..a0a34be19 100644 --- a/src/main/java/com/conveyal/analysis/models/AggregationArea.java +++ b/src/main/java/com/conveyal/analysis/models/AggregationArea.java @@ -8,9 +8,13 @@ import static com.conveyal.file.FileCategory.GRIDS; /** - * An aggregation area defines a set of origin points to be averaged together to produce an aggregate accessibility figure. - * It is defined by a geometry that is rasterized and stored as a grid, with pixels with values between 0 and 100,000 - * depending on how much of that pixel is overlapped by the mask. + * An aggregation area defines a set of origin points that can be combined to produce an aggregate accessibility figure. + * For example, if we have accessibility results for an entire city, we might want to calculate 25th percentile + * population-weighted accessibility for each administrative district. Each neighborhood would be an aggreagation area. + * + * An aggregation area is defined by a polygon that has been rasterized and stored as a grid, with each pixel value + * expressing how much of that pixel falls within the mask polygon. These values are in the range of 0 to 100,000 + * (rather than 0 to 1) because our serialized (on-disk) grid format can only store integers. */ public class AggregationArea extends BaseModel { diff --git a/src/main/java/com/conveyal/analysis/models/DataGroup.java b/src/main/java/com/conveyal/analysis/models/DataGroup.java index 6cf22a15c..6bad6307a 100644 --- a/src/main/java/com/conveyal/analysis/models/DataGroup.java +++ b/src/main/java/com/conveyal/analysis/models/DataGroup.java @@ -4,8 +4,11 @@ import org.bson.types.ObjectId; /** - * When deriving other data (layers, networks, etc.) from a DataSource, we sometimes produce many outputs at once from + * When deriving data (layers, networks, etc.) from a DataSource, we sometimes produce many outputs at once from * the same source and configuration options. We group all those derived products together using a DataGroup. + * The grouping is achieved simply by multiple other entities of the same type referencing the same dataGroupId. + * The DataGroups don't have many other characteristics of their own. They are materialized and stored in Mongo just + * to provide a user-editable name/description for the group. */ public class DataGroup extends BaseModel { diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java index d9c014423..1f29cf79f 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java @@ -39,7 +39,7 @@ public class SpatialDataSource extends DataSource { /** All features in this SpatialDataSource have an attached geometry of this type. */ public ShapefileReader.GeometryType geometryType; - /** Every feature has this set of Attributes - this is essentially a schema. */ + /** Every feature has this set of Attributes - this is essentially a schema giving attribute names and types. */ public List attributes; public SpatialDataSource (UserPermissions userPermissions, String name) { @@ -49,16 +49,7 @@ public SpatialDataSource (UserPermissions userPermissions, String name) { /** Zero-argument constructor required for Mongo automatic POJO deserialization. */ public SpatialDataSource () { } - /** - * Fluent methods to avoid constructors with lots of positional paramters. - * Not really a builder pattern since it doesn't construct immutable objects, - * but due to automatic POJO Mongo storage we can't have immutable objects anyway. - * Unfortunately these can't be placed on the superclass because they won't return the concrete subclass. - * Hence more absurd Java verbosity for things that should be built in language features like named parameters. - */ - // Given these restrictions I think I'd rather go with classic factory methods here instead of builders. - - public FileStorageKey storageKey() { + public FileStorageKey storageKey () { return new FileStorageKey(DATASOURCES, this._id.toString(), fileFormat.toString()); } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/Task.java b/src/main/java/com/conveyal/r5/analyst/progress/Task.java index e248a1b75..49990af7a 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/Task.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/Task.java @@ -11,10 +11,10 @@ import java.util.UUID; /** - * This newer task progress system coexists with several older mechanisms. It is still evolving as we adapt it to new - * use cases. The Task class is intended to represent background tasks whose progress the end user should be aware of, - * such as processing uploaded files. It should not be used for automatic internal actions (such as Events) which would - * clutter a user's active task list. + * This newer task progress system coexists with several older mechanisms, which it is intended to supersede. It is + * still evolving as we adapt it to new use cases. The Task class is intended to represent background tasks whose + * progress the end user should be aware of, such as processing uploaded files. It should not be used for automatic + * internal actions (such as Events) which would clutter a user's active task list. * * A Task (or some interface that it implements) could be used by the AsyncLoader to track progress. Together with some * AsyncLoader functionality it will be a bit like a Future with progress reporting. Use of AsyncLoader could then be diff --git a/src/main/java/com/conveyal/r5/analyst/progress/TaskAction.java b/src/main/java/com/conveyal/r5/analyst/progress/TaskAction.java index 9d0bb0d8c..79ccc661e 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/TaskAction.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/TaskAction.java @@ -5,6 +5,9 @@ * It's a single-method interface so it can be defined with lambda functions, or other objects can implement it. * When the action is run, it will receive an object implementing an interface through which it can report progress * and errors. + * + * Alteratively, TaskAction could have more methods: return a title, heaviness, user details, etc. to be seen by Task, + * instead of having only one method to make it a functional interface. I'd like to discourage anonymous functions anyway. */ public interface TaskAction { diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java index 1df162014..eb4da1d09 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java @@ -3,7 +3,7 @@ import com.conveyal.analysis.models.BaseModel; /** - * A unique identifier for the final product of a single task action. Currently this serves as both an internal data + * A unique identifier for the final product of a single TaskAction. Currently this serves as both an internal data * structure and an API model class, which should be harmless as it's an immutable data class. The id is unique within * the type, so the regionId is redundant information, but facilitates prefectches on the UI. If isGroup is true, the * id is not that of an individual record, but the dataGroupId of several records created in a single operation. From 8c3573b2887db3ad032adf9c6e02fed249789035 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 15:59:38 +0800 Subject: [PATCH 101/187] factor action out into AggregationAreaDerivation introduce interface DataDerivation. begin moving to eliminate anonymous functions and use clearer types. --- .../AggregationAreaController.java | 189 ++------------- .../derivation/AggregationAreaDerivation.java | 225 ++++++++++++++++++ .../datasource/derivation/DataDerivation.java | 24 ++ 3 files changed, 272 insertions(+), 166 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java create mode 100644 src/main/java/com/conveyal/analysis/datasource/derivation/DataDerivation.java diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 962baef69..694e34282 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -1,53 +1,29 @@ package com.conveyal.analysis.controllers; -import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; +import com.conveyal.analysis.datasource.derivation.AggregationAreaDerivation; +import com.conveyal.analysis.datasource.derivation.DataDerivation; import com.conveyal.analysis.models.AggregationArea; -import com.conveyal.analysis.models.DataGroup; import com.conveyal.analysis.models.DataSource; -import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.JsonUtil; import com.conveyal.file.FileStorage; -import com.conveyal.file.FileUtils; -import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.progress.Task; -import com.conveyal.r5.analyst.progress.WorkProduct; -import com.conveyal.r5.analyst.progress.WorkProductType; -import com.conveyal.r5.util.ShapefileReader; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Preconditions; import org.bson.conversions.Bson; -import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.operation.union.UnaryUnionOp; -import org.opengis.feature.simple.SimpleFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.text.MessageFormat; -import java.util.ArrayList; +import java.lang.invoke.MethodHandles; import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.conveyal.file.FileStorageFormat.GEOJSON; -import static com.conveyal.file.FileStorageFormat.SHP; -import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; -import static com.conveyal.r5.analyst.progress.WorkProductType.AGGREGATION_AREA; -import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -56,25 +32,14 @@ */ public class AggregationAreaController implements HttpController { - private static final Logger LOG = LoggerFactory.getLogger(AggregationAreaController.class); - - /** - * Arbitrary limit to prevent UI clutter from many aggregation areas (e.g. if someone uploads thousands of blocks). - * Someone might reasonably request an aggregation area for each of Chicago's 50 wards, so that's a good approximate - * limit for now. - */ - private static int MAX_FEATURES = 100; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final FileStorage fileStorage; + private final AnalysisDB analysisDb; private final TaskScheduler taskScheduler; - private final AnalysisCollection aggregationAreaCollection; - // FIXME Should we instead be using the same instance as the DataSourceController? - // Anyway the parameterized type is too specific. private final AnalysisCollection dataSourceCollection; - - // TODO review July 1 notes - private final AnalysisCollection dataGroupCollection; + private final AnalysisCollection aggregationAreaCollection; public AggregationAreaController ( FileStorage fileStorage, @@ -82,143 +47,35 @@ public AggregationAreaController ( TaskScheduler taskScheduler ) { this.fileStorage = fileStorage; + this.analysisDb = database; this.taskScheduler = taskScheduler; - this.aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); - this.dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); - this.dataGroupCollection = database.getAnalysisCollection("dataGroups", DataGroup.class); + dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); + aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); } /** - * Create binary .grid files for aggregation (aka mask) areas, save them to S3, and persist their details. - * @param req Must include a shapefile on which the aggregation area(s) will be based. - * --NOTE-- behavior has changed. union parameter is not specified. Now union = (nameProperty == null). - * If HTTP query parameter union is "true", features will be merged to a single aggregation area, named - * using the value of the "name" query parameter. If union is false or if the parameter is missing, each - * feature will be a separate aggregation area, named using the value for the shapefile property - * specified by the HTTP query parameter "nameAttribute." - * @return the Task representing the background action of creating the aggregation areas, or its ID. + * Create binary .grid files for aggregation (aka mask) areas, save them to FileStorage, and persist their metadata + * to Mongo. The supplied request (req) must include query parameters specifying the dataSourceId of a + * SpatialDataSoure containing the polygonal aggregation area geometries. If the nameProperty query parameter is + * non-null, it must be the name of a text attribute in that SpatialDataSource, and one aggregation area will be + * created for each polygon using those names. If the nameProperty is not supplied, all polygons will be merged into + * one large nameless (multi)polygon aggregation area. + * @return the Task representing the enqueued background action that will create the aggregation areas. */ private Task createAggregationAreas (Request req, Response res) throws Exception { - ArrayList aggregationAreas = new ArrayList<>(); - UserPermissions userPermissions = UserPermissions.from(req); - String dataSourceId = req.queryParams("dataSourceId"); - //String nameProperty = req.queryParams("nameProperty"); - final String nameProperty = "dist_name"; - final int zoom = parseZoom(req.queryParams("zoom")); - - - // 1. Get file from storage and read its features. ============================================================= - DataSource dataSource = dataSourceCollection.findById(dataSourceId); - checkArgument(dataSource instanceof SpatialDataSource, - "Only spatial data sets can be converted to aggregation areas."); - SpatialDataSource spatialDataSource = (SpatialDataSource) dataSource; - checkArgument(POLYGON.equals(spatialDataSource.geometryType), - "Only polygons can be converted to aggregation areas. DataSource is: " + spatialDataSource.geometryType); - checkArgument(SHP.equals(spatialDataSource.fileFormat), - "Currently, only shapefiles can be converted to aggregation areas."); - - File sourceFile; - List features = null; - - if (SHP.equals(spatialDataSource.fileFormat)) { - sourceFile = fileStorage.getFile(spatialDataSource.storageKey()); - ShapefileReader reader = null; - try { - reader = new ShapefileReader(sourceFile); - features = reader.wgs84Stream().collect(Collectors.toList()); - } finally { - if (reader != null) reader.close(); - } - } - - if (GEOJSON.equals(spatialDataSource.fileFormat)) { - // TODO implement - } - final List finalFeatures = features; - Task backgroundTask = Task.create("Aggregation area creation: " + spatialDataSource.name) - .forUser(userPermissions) + // Create and enqueue an asynchronous background action to derive aggreagation areas from spatial data source. + // The constructor will extract query parameters and range check them (not ideal separation, but it works). + DataDerivation derivation = AggregationAreaDerivation.fromRequest(req, fileStorage, analysisDb); + Task backgroundTask = Task.create("Aggregation area creation: " + derivation.dataSource().name) + .forUser(UserPermissions.from(req)) .setHeavy(true) - .withAction(progressListener -> { - String groupDescription = "Convert polygons to aggregation areas, " + - ((nameProperty == null) ? "merging all polygons." : "one area per polygon."); - DataGroup dataGroup = new DataGroup(userPermissions, spatialDataSource._id.toString(), groupDescription); - - progressListener.beginTask("Reading data source", finalFeatures.size() + 1); - Map areas = new HashMap<>(); - - if (nameProperty != null && finalFeatures.size() > MAX_FEATURES) { - throw AnalysisServerException.fileUpload( - MessageFormat.format("The uploaded shapefile has {0} features, " + - "which exceeds the limit of {1}", finalFeatures.size(), MAX_FEATURES) - ); - } - - if (nameProperty == null) { - // Union (single combined aggregation area) requested - List geometries = finalFeatures.stream().map(f -> - (Geometry) f.getDefaultGeometry()).collect(Collectors.toList() - ); - UnaryUnionOp union = new UnaryUnionOp(geometries); - // Name the area using the name in the request directly - areas.put(spatialDataSource.name, union.union()); - } else { - // Don't union. Name each area by looking up its value for the name property in the request. - finalFeatures.forEach(f -> areas.put( - readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry()) - ); - } - - // 2. Convert to raster grids, then store them. ==================================================== - areas.forEach((String name, Geometry geometry) -> { - if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); - Envelope env = geometry.getEnvelopeInternal(); - Grid maskGrid = new Grid(zoom, env); - progressListener.beginTask("Creating grid for " + name, 0); - - // Store the percentage each cell overlaps the mask, scaled as 0 to 100,000 - List weights = maskGrid.getPixelWeights(geometry, true); - weights.forEach(pixel -> { - maskGrid.grid[pixel.x][pixel.y] = pixel.weight * 100_000; - }); - - AggregationArea aggregationArea = new AggregationArea(userPermissions, name, spatialDataSource); - - try { - File gridFile = FileUtils.createScratchFile("grid"); - OutputStream os = new GZIPOutputStream(FileUtils.getOutputStream(gridFile)); - maskGrid.write(os); - os.close(); - aggregationArea.dataGroupId = dataGroup._id.toString(); - aggregationAreas.add(aggregationArea); - fileStorage.moveIntoStorage(aggregationArea.getStorageKey(), gridFile); - } catch (IOException e) { - throw new AnalysisServerException("Error processing/uploading aggregation area"); - } - progressListener.increment(); - }); - aggregationAreaCollection.insertMany(aggregationAreas); - dataGroupCollection.insert(dataGroup); - progressListener.setWorkProduct(WorkProduct.forDataGroup( - AGGREGATION_AREA, dataGroup._id.toString(), dataSource.regionId) - ); - progressListener.increment(); - }); + .withAction(derivation); taskScheduler.enqueue(backgroundTask); return backgroundTask; } - private String readProperty (SimpleFeature feature, String propertyName) { - try { - return feature.getProperty(propertyName).getValue().toString(); - } catch (NullPointerException e) { - String message = String.format("The specified property '%s' was not present on the uploaded features. " + - "Please verify that '%s' corresponds to a shapefile column.", propertyName, propertyName); - throw new AnalysisServerException(message); - } - } - private Collection getAggregationAreas (Request req, Response res) { Bson query = eq("regionId", req.queryParams("regionId")); String dataGroupId = req.queryParams("dataGroupId"); diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java new file mode 100644 index 000000000..b800e247d --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -0,0 +1,225 @@ +package com.conveyal.analysis.datasource.derivation; + +import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.UserPermissions; +import com.conveyal.analysis.datasource.DataSourceException; +import com.conveyal.analysis.models.AggregationArea; +import com.conveyal.analysis.models.DataGroup; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.analysis.persistence.AnalysisDB; +import com.conveyal.file.FileStorage; +import com.conveyal.file.FileUtils; +import com.conveyal.r5.analyst.Grid; +import com.conveyal.r5.analyst.progress.ProgressListener; +import com.conveyal.r5.analyst.progress.WorkProduct; +import com.conveyal.r5.util.ShapefileReader; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.operation.union.UnaryUnionOp; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; +import spark.Request; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; + +import static com.conveyal.file.FileStorageFormat.GEOJSON; +import static com.conveyal.file.FileStorageFormat.SHP; +import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.conveyal.r5.analyst.progress.WorkProductType.AGGREGATION_AREA; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Created by abyrd on 2021-09-03 + */ +public class AggregationAreaDerivation implements DataDerivation { + + /** + * Arbitrary limit to prevent UI clutter from many aggregation areas (e.g. if someone uploads thousands of blocks). + * Someone might reasonably request an aggregation area for each of Chicago's 50 wards, so that's a good approximate + * limit for now. + */ + private static final int MAX_FEATURES = 100; + + private final FileStorage fileStorage; + private final UserPermissions userPermissions; + private final String dataSourceId; + private final String nameProperty; + private final int zoom; + private final SpatialDataSource spatialDataSource; + private final List finalFeatures; + + // TODO derivations could return their model objects and DataGroups so they don't need direct database and fileStorage access. + // A DerivationProduct could be a collection of File, a Collection and a DataGroup. + private final AnalysisCollection aggregationAreaCollection; + private final AnalysisCollection dataGroupCollection; + + /** + * Extraction, validation and range checking of parameters. + * It's kind of a red flag that we're passing Components in here. The products should probably be returned by the + * Derivation and stored by some more general purpose wrapper so we can avoid direct file and database access here. + * It's also not great to pass in the full request - we only need to extract and validate query parameters. + */ + private AggregationAreaDerivation (FileStorage fileStorage, AnalysisDB database, Request req) { + + // Before kicking off asynchronous processing, range check inputs to fail fast on obvious problems. + userPermissions = UserPermissions.from(req); + dataSourceId = req.queryParams("dataSourceId"); + nameProperty = "dist_name"; // req.queryParams("nameProperty"); + zoom = parseZoom(req.queryParams("zoom")); + checkNotNull(dataSourceId); + checkNotNull(nameProperty); + // TODO range check zoom + + AnalysisCollection dataSourceCollection = + database.getAnalysisCollection("dataSources", DataSource.class); + DataSource dataSource = dataSourceCollection.findById(dataSourceId); + checkArgument(dataSource instanceof SpatialDataSource, + "Only spatial data sets can be converted to aggregation areas."); + spatialDataSource = (SpatialDataSource) dataSource; + checkArgument(POLYGON.equals(spatialDataSource.geometryType), + "Only polygons can be converted to aggregation areas. DataSource is: " + spatialDataSource.geometryType); + checkArgument(SHP.equals(spatialDataSource.fileFormat), + "Currently, only shapefiles can be converted to aggregation areas."); + + /* + Implementation notes: + Collecting all the Features to a List is a red flag for scalability, but the UnaryUnionOp used below (and the + CascadedPolygonUnion it depends on) appear to only operate on in-memory lists. The ShapefileReader and the + FeatureSource it contains also seem to always load all features at once. So for now we just have to tolerate + loading the whole files into memory at once. + If we do need to pre-process the file (here reading it and converting it to WGS84) that's not a + constant-time operation, so it should probably be done in the async task below instead of this synchronous + HTTP controller code. + We may not need to union the features at all. We could just iteratively rasterize all the polygons into a + single grid which would effectively union them. This would allow both the union and non-union case to be + handled in a streaming fashion (in constant memory). + This whole process needs to be audited though, it's strangely slow. + */ + File sourceFile; + if (SHP.equals(spatialDataSource.fileFormat)) { + sourceFile = fileStorage.getFile(spatialDataSource.storageKey()); + ShapefileReader reader = null; + try { + reader = new ShapefileReader(sourceFile); + finalFeatures = reader.wgs84Stream().collect(Collectors.toList()); + } catch (FactoryException | RuntimeException | IOException | TransformException e) { + throw new DataSourceException("Failed to load shapefile.", e); + } finally { + if (reader != null) reader.close(); + } + } else { + // GeoJSON, GeoPackage etc. + throw new UnsupportedOperationException("To be implemented."); + } + if (nameProperty != null && finalFeatures.size() > MAX_FEATURES) { + String message = MessageFormat.format( + "The uploaded shapefile has {0} features, exceeding the limit of {1}", + finalFeatures.size(), MAX_FEATURES + ); + throw new DataSourceException(message); + } + + this.fileStorage = fileStorage; + // Do not retain AnalysisDB reference, but grab the collections we need. + // TODO cache AnalysisCollection instances and reuse? Are they threadsafe? + aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); + dataGroupCollection = database.getAnalysisCollection("dataGroups", DataGroup.class); + } + + @Override + public void action (ProgressListener progressListener) throws Exception { + + ArrayList aggregationAreas = new ArrayList<>(); + String groupDescription = "Convert polygons to aggregation areas, " + + ((nameProperty == null) ? "merging all polygons." : "one area per polygon."); + DataGroup dataGroup = new DataGroup(userPermissions, spatialDataSource._id.toString(), groupDescription); + + progressListener.beginTask("Reading data source", finalFeatures.size() + 1); + Map areaGeometries = new HashMap<>(); + + if (nameProperty == null) { + // Union (single combined aggregation area) requested + List geometries = finalFeatures.stream().map(f -> + (Geometry) f.getDefaultGeometry()).collect(Collectors.toList() + ); + UnaryUnionOp union = new UnaryUnionOp(geometries); + // Name the area using the name in the request directly + areaGeometries.put(spatialDataSource.name, union.union()); + } else { + // Don't union. Name each area by looking up its value for the name property in the request. + finalFeatures.forEach(f -> areaGeometries.put( + readProperty(f, nameProperty), (Geometry) f.getDefaultGeometry()) + ); + } + + // Convert to raster grids, then store them. + areaGeometries.forEach((String name, Geometry geometry) -> { + if (geometry == null) throw new AnalysisServerException("Invalid geometry uploaded."); + Envelope env = geometry.getEnvelopeInternal(); + Grid maskGrid = new Grid(zoom, env); + progressListener.beginTask("Creating grid for " + name, 0); + + // Store the percentage each cell overlaps the mask, scaled as 0 to 100,000 + List weights = maskGrid.getPixelWeights(geometry, true); + weights.forEach(pixel -> { + maskGrid.grid[pixel.x][pixel.y] = pixel.weight * 100_000; + }); + + AggregationArea aggregationArea = new AggregationArea(userPermissions, name, spatialDataSource); + + try { + File gridFile = FileUtils.createScratchFile("grid"); + OutputStream os = new GZIPOutputStream(FileUtils.getOutputStream(gridFile)); + maskGrid.write(os); + os.close(); + aggregationArea.dataGroupId = dataGroup._id.toString(); + aggregationAreas.add(aggregationArea); + fileStorage.moveIntoStorage(aggregationArea.getStorageKey(), gridFile); + } catch (IOException e) { + throw new AnalysisServerException("Error processing/uploading aggregation area"); + } + progressListener.increment(); + }); + aggregationAreaCollection.insertMany(aggregationAreas); + dataGroupCollection.insert(dataGroup); + progressListener.setWorkProduct(WorkProduct.forDataGroup( + AGGREGATION_AREA, dataGroup._id.toString(), spatialDataSource.regionId) + ); + progressListener.increment(); + + } + + private static String readProperty (SimpleFeature feature, String propertyName) { + try { + return feature.getProperty(propertyName).getValue().toString(); + } catch (NullPointerException e) { + String message = String.format("The specified property '%s' was not present on the uploaded features. " + + "Please verify that '%s' corresponds to a shapefile column.", propertyName, propertyName); + throw new AnalysisServerException(message); + } + } + + public static AggregationAreaDerivation fromRequest (Request req, FileStorage fileStorage, AnalysisDB database) { + return new AggregationAreaDerivation(fileStorage, database, req); + } + + @Override + public SpatialDataSource dataSource () { + return spatialDataSource; + } + +} diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/DataDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/DataDerivation.java new file mode 100644 index 000000000..9e0ea8515 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/DataDerivation.java @@ -0,0 +1,24 @@ +package com.conveyal.analysis.datasource.derivation; + +import com.conveyal.analysis.models.BaseModel; +import com.conveyal.analysis.models.DataGroup; +import com.conveyal.analysis.models.DataSource; +import com.conveyal.r5.analyst.progress.TaskAction; + +import java.util.Collection; + +/** + * An interface for unary operators mapping DataSources into other data sets represented in our Mongo database. + * An asynchronous function from D to M. + */ +public interface DataDerivation extends TaskAction { + + public D dataSource (); + +// public Collection outputs(); + +// public DataGroup outputGroup(); + +// or single output method: public DataGroup output(); + +} From 88cffa617f894a3c489bc770c54feace1b18d22d Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 17:58:21 +0800 Subject: [PATCH 102/187] move getFormField to HttpUtil. --- .../OpportunityDatasetController.java | 41 ++++--------------- .../datasource/DataSourceUploadAction.java | 5 +-- .../com/conveyal/analysis/util/HttpUtils.java | 24 +++++++++++ 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 62665b4b8..aa705e83b 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -9,6 +9,7 @@ import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.FileItemInputStreamProvider; +import com.conveyal.analysis.util.HttpUtils; import com.conveyal.analysis.util.JsonUtil; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageFormat; @@ -41,7 +42,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; @@ -292,29 +292,6 @@ private List createFreeFormPointSetsFromCsv(FileItem csvFileIt } - /** - * Get the specified field from a map representing a multipart/form-data POST request, as a UTF-8 String. - * FileItems represent any form item that was received within a multipart/form-data POST request, not just files. - * This is a static utility method that should be reusable across different HttpControllers. - */ - public static String getFormField(Map> formFields, String fieldName, boolean required) { - try { - List fileItems = formFields.get(fieldName); - if (fileItems == null || fileItems.isEmpty()) { - if (required) { - throw AnalysisServerException.badRequest("Missing required field: " + fieldName); - } else { - return null; - } - } - String value = fileItems.get(0).getString("UTF-8"); - return value; - } catch (UnsupportedEncodingException e) { - throw AnalysisServerException.badRequest(String.format("Multipart form field '%s' had unsupported encoding", - fieldName)); - } - } - /** * Handle many types of file upload. Returns a OpportunityDatasetUploadStatus which has a handle to request status. * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. @@ -331,9 +308,9 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res } // Parse required fields. Will throw a ServerException on failure. - final String sourceName = getFormField(formFields, "Name", true); - final String regionId = getFormField(formFields, "regionId", true); - final int zoom = parseZoom(getFormField(formFields, "zoom", false)); + final String sourceName = HttpUtils.getFormField(formFields, "Name", true); + final String regionId = HttpUtils.getFormField(formFields, "regionId", true); + final int zoom = parseZoom(HttpUtils.getFormField(formFields, "zoom", false)); // Create a region-wide status object tracking the processing of opportunity data. // Create the status object before doing anything including input and parameter validation, so that any problems @@ -485,14 +462,14 @@ private List createGridsFromCsv(FileItem csvFileItem, int zoom, OpportunityDatasetUploadStatus status) throws Exception { - String latField = getFormField(query, "latField", true); - String lonField = getFormField(query, "lonField", true); - String idField = getFormField(query, "idField", false); + String latField = HttpUtils.getFormField(query, "latField", true); + String lonField = HttpUtils.getFormField(query, "lonField", true); + String idField = HttpUtils.getFormField(query, "idField", false); // Optional fields to run grid construction twice with two different sets of points. // This is only really useful when creating grids to visualize freeform pointsets for one-to-one analyses. - String latField2 = getFormField(query, "latField2", false); - String lonField2 = getFormField(query, "lonField2", false); + String latField2 = HttpUtils.getFormField(query, "latField2", false); + String lonField2 = HttpUtils.getFormField(query, "lonField2", false); List ignoreFields = Arrays.asList(idField, latField2, lonField2); InputStreamProvider csvStreamProvider = new FileItemInputStreamProvider(csvFileItem); diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 24a433ec4..6c1563620 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -11,7 +11,6 @@ import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.io.FilenameUtils; -import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,12 +20,10 @@ import java.util.Map; import java.util.stream.Collectors; -import static com.conveyal.analysis.controllers.OpportunityDatasetController.getFormField; +import static com.conveyal.analysis.util.HttpUtils.getFormField; import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; import static com.conveyal.file.FileCategory.DATASOURCES; -import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.SHP; -import static com.conveyal.file.FileStorageFormat.TIFF; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; diff --git a/src/main/java/com/conveyal/analysis/util/HttpUtils.java b/src/main/java/com/conveyal/analysis/util/HttpUtils.java index 5073a00ab..2e3737724 100644 --- a/src/main/java/com/conveyal/analysis/util/HttpUtils.java +++ b/src/main/java/com/conveyal/analysis/util/HttpUtils.java @@ -8,6 +8,7 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import javax.servlet.http.HttpServletRequest; +import java.io.UnsupportedEncodingException; import java.util.List; import java.util.Map; @@ -30,4 +31,27 @@ public static Map> getRequestFiles (HttpServletRequest re throw AnalysisServerException.badRequest(ExceptionUtils.stackTraceString(e)); } } + + /** + * Get the specified field from a map representing a multipart/form-data POST request, as a UTF-8 String. + * FileItems represent any form item that was received within a multipart/form-data POST request, not just files. + * This is a static utility method that should be reusable across different HttpControllers. + */ + public static String getFormField(Map> formFields, String fieldName, boolean required) { + try { + List fileItems = formFields.get(fieldName); + if (fileItems == null || fileItems.isEmpty()) { + if (required) { + throw AnalysisServerException.badRequest("Missing required field: " + fieldName); + } else { + return null; + } + } + String value = fileItems.get(0).getString("UTF-8"); + return value; + } catch (UnsupportedEncodingException e) { + throw AnalysisServerException.badRequest(String.format("Multipart form field '%s' had unsupported encoding", + fieldName)); + } + } } From 68889b41e3e941b98bc1884258bdbd9b1064a45f Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 21:14:06 +0800 Subject: [PATCH 103/187] GeoTIFF ingester Store files with more standard lowercase extensions. Make enum value more specific GEOTIFF instead of TIFF. Use nameProperty query parameter when deriving aggregation areas. --- .../AggregationAreaController.java | 6 +- .../controllers/DataSourceController.java | 4 +- .../OpportunityDatasetController.java | 4 +- .../RegionalAnalysisController.java | 5 +- .../datasource/DataSourceIngester.java | 6 +- .../datasource/DataSourceUploadAction.java | 2 +- .../datasource/GeoTiffDataSourceIngester.java | 67 +++++++++++++++++-- .../analysis/datasource/SpatialAttribute.java | 9 ++- .../analysis/datasource/SpatialLayers.java | 2 +- .../derivation/AggregationAreaDerivation.java | 2 +- .../analysis/models/SpatialDataSource.java | 5 +- .../persistence/AnalysisCollection.java | 2 + .../com/conveyal/file/FileStorageFormat.java | 7 +- .../com/conveyal/r5/util/ShapefileReader.java | 2 +- 14 files changed, 96 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 694e34282..4413fcc9e 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -60,9 +60,9 @@ public AggregationAreaController ( * non-null, it must be the name of a text attribute in that SpatialDataSource, and one aggregation area will be * created for each polygon using those names. If the nameProperty is not supplied, all polygons will be merged into * one large nameless (multi)polygon aggregation area. - * @return the Task representing the enqueued background action that will create the aggregation areas. + * @return the ID of the Task representing the enqueued background action that will create the aggregation areas. */ - private Task createAggregationAreas (Request req, Response res) throws Exception { + private String createAggregationAreas (Request req, Response res) throws Exception { // Create and enqueue an asynchronous background action to derive aggreagation areas from spatial data source. // The constructor will extract query parameters and range check them (not ideal separation, but it works). @@ -73,7 +73,7 @@ private Task createAggregationAreas (Request req, Response res) throws Exception .withAction(derivation); taskScheduler.enqueue(backgroundTask); - return backgroundTask; + return backgroundTask.id.toString(); } private Collection getAggregationAreas (Request req, Response res) { diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 376cfafcb..019a188d9 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -108,8 +108,8 @@ private SpatialDataSource downloadLODES(Request req, Response res) { /** * A file is posted to this endpoint to create a new DataSource. It is validated and metadata are extracted. * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. - * In a standard REST API, a post would return the ID of the newly created DataSource. Here we're starting an async - * background process, so we return the task ID or the ID its work product (the DataSource)? + * In standard REST API style, a POST would return the ID of the newly created DataSource. Here we're starting an + * async background process, so we return the ID of the enqueued Task (rather than its work product, the DataSource). */ private String handleUpload (Request req, Response res) { final UserPermissions userPermissions = UserPermissions.from(req); diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index aa705e83b..cf3d3cb73 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -446,7 +446,7 @@ private OpportunityDataset deleteDataset(String id, UserPermissions userPermissi } else { fileStorage.delete(dataset.getStorageKey(FileStorageFormat.GRID)); fileStorage.delete(dataset.getStorageKey(FileStorageFormat.PNG)); - fileStorage.delete(dataset.getStorageKey(FileStorageFormat.TIFF)); + fileStorage.delete(dataset.getStorageKey(FileStorageFormat.GEOTIFF)); } return dataset; } @@ -582,7 +582,7 @@ private Object downloadOpportunityDataset (Request req, Response res) throws IOE if (FileStorageFormat.PNG.equals(downloadFormat)) { grid.writePng(fos); - } else if (FileStorageFormat.TIFF.equals(downloadFormat)) { + } else if (FileStorageFormat.GEOTIFF.equals(downloadFormat)) { grid.writeGeotiff(fos); } diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 485820ef2..80e2ac0fd 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -21,7 +21,6 @@ import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.cluster.RegionalTask; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.primitives.Ints; import com.mongodb.QueryBuilder; import gnu.trove.list.array.TIntArrayList; @@ -264,7 +263,7 @@ private Object getRegionalResults (Request req, Response res) throws IOException LOG.debug("Returning {} minute accessibility to pointset {} (percentile {}) for regional analysis {}.", cutoffMinutes, destinationPointSetId, percentile, regionalAnalysisId); FileStorageFormat format = FileStorageFormat.valueOf(fileFormatExtension.toUpperCase()); - if (!FileStorageFormat.GRID.equals(format) && !FileStorageFormat.PNG.equals(format) && !FileStorageFormat.TIFF.equals(format)) { + if (!FileStorageFormat.GRID.equals(format) && !FileStorageFormat.PNG.equals(format) && !FileStorageFormat.GEOTIFF.equals(format)) { throw AnalysisServerException.badRequest("Format \"" + format + "\" is invalid. Request format must be \"grid\", \"png\", or \"tiff\"."); } @@ -310,7 +309,7 @@ private Object getRegionalResults (Request req, Response res) throws IOException case PNG: grid.writePng(fos); break; - case TIFF: + case GEOTIFF: grid.writeGeotiff(fos); break; } diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java index e7e37cce0..9e3943c99 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceIngester.java @@ -4,16 +4,14 @@ import com.conveyal.analysis.models.DataSource; import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; -import org.apache.commons.fileupload.FileItem; import org.bson.types.ObjectId; import java.io.File; -import java.util.stream.Collectors; import static com.conveyal.file.FileStorageFormat.GEOJSON; import static com.conveyal.file.FileStorageFormat.GEOPACKAGE; import static com.conveyal.file.FileStorageFormat.SHP; -import static com.conveyal.file.FileStorageFormat.TIFF; +import static com.conveyal.file.FileStorageFormat.GEOTIFF; /** * Logic for loading and validating a specific kind of input file, yielding a specific subclass of DataSource. @@ -68,7 +66,7 @@ public static DataSourceIngester forFormat (FileStorageFormat format) { return new ShapefileDataSourceIngester(); } else if (format == GEOJSON) { return new GeoJsonDataSourceIngester(); - } else if (format == TIFF) { // really this enum value should be GEOTIFF rather than just TIFF. + } else if (format == GEOTIFF) { return new GeoTiffDataSourceIngester(); } else if (format == GEOPACKAGE) { return new GeoPackageDataSourceIngester(); diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 6c1563620..308bc0de9 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -98,7 +98,7 @@ private final void moveFilesIntoStorage (ProgressListener progressListener) { DiskFileItem dfi = (DiskFileItem) fileItem; // TODO use canonical extensions from filetype enum // TODO upper case? should we be using lower case? - String extension = FilenameUtils.getExtension(fileItem.getName()).toUpperCase(Locale.ROOT); + String extension = FilenameUtils.getExtension(fileItem.getName()).toLowerCase(Locale.ROOT); FileStorageKey key = new FileStorageKey(DATASOURCES, dataSourceId, extension); fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); if (fileItems.size() == 1 || extension.equalsIgnoreCase(SHP.extension)) { diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java index 97b72969f..e7a18c575 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java @@ -1,10 +1,31 @@ package com.conveyal.analysis.datasource; +import com.conveyal.analysis.models.Bounds; import com.conveyal.analysis.models.DataSource; import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.r5.analyst.progress.ProgressListener; +import org.geotools.coverage.GridSampleDimension; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.grid.GridEnvelope2D; +import org.geotools.coverage.grid.io.AbstractGridFormat; +import org.geotools.coverage.grid.io.GridFormatFinder; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.CRS; +import org.geotools.referencing.crs.DefaultGeographicCRS; +import org.geotools.util.factory.Hints; +import org.opengis.coverage.SampleDimensionType; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.opengis.referencing.operation.MathTransform; +import org.opengis.referencing.operation.TransformException; import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static com.conveyal.file.FileStorageFormat.GEOTIFF; +import static com.conveyal.r5.util.ShapefileReader.GeometryType.PIXEL; /** * GoeTIFFs are used as inputs in network building as digital elevation profiles, and eventually expected to @@ -14,6 +35,12 @@ public class GeoTiffDataSourceIngester extends DataSourceIngester { private final SpatialDataSource dataSource; + public GeoTiffDataSourceIngester () { + this.dataSource = new SpatialDataSource(); + dataSource.geometryType = PIXEL; + dataSource.fileFormat = GEOTIFF; // Should be GEOTIFF specifically + } + @Override protected DataSource dataSource () { return dataSource; @@ -21,11 +48,43 @@ protected DataSource dataSource () { @Override public void ingest (File file, ProgressListener progressListener) { - throw new UnsupportedOperationException(); - } + AbstractGridFormat format = GridFormatFinder.findFormat(file); + Hints hints = new Hints(Hints.FORCE_LONGITUDE_FIRST_AXIS_ORDER, Boolean.TRUE); + var coverageReader = format.getReader(file, hints); + GridCoverage2D coverage; + try { + coverage = coverageReader.read(null); + } catch (IOException e) { + throw new DataSourceException("Could not read GeoTiff.", e); + } + // Transform to WGS84 to ensure this will not trigger any errors downstream. + CoordinateReferenceSystem coverageCrs = coverage.getCoordinateReferenceSystem2D(); + MathTransform wgsToCoverage, coverageToWgs; + ReferencedEnvelope wgsEnvelope; + try { + wgsToCoverage = CRS.findMathTransform(DefaultGeographicCRS.WGS84, coverageCrs); + coverageToWgs = wgsToCoverage.inverse(); + // Envelope in coverage CRS is not necessarily aligned with axes when transformed to WGS84. + // As far as I can tell those cases are handled by this call, but I'm not completely sure. + wgsEnvelope = new ReferencedEnvelope(coverage.getEnvelope2D().toBounds(DefaultGeographicCRS.WGS84)); + } catch (FactoryException | TransformException e) { + throw new DataSourceException("Could not create coordinate transform to and from WGS84."); + } - public GeoTiffDataSourceIngester () { - this.dataSource = new SpatialDataSource(); + List attributes = new ArrayList<>(); + for (int d = 0; d < coverage.getNumSampleDimensions(); d++) { + GridSampleDimension sampleDimension = coverage.getSampleDimension(d); + SampleDimensionType type = sampleDimension.getSampleDimensionType(); + attributes.add(new SpatialAttribute(type, d)); + } + // Get the dimensions of the pixel grid so we can record the number of pixels. + // Total number of pixels can be huge, cast it to 64 bits. + GridEnvelope2D gridEnv = coverage.getGridGeometry().getGridRange2D(); + dataSource.wgsBounds = Bounds.fromWgsEnvelope(wgsEnvelope); + dataSource.featureCount = (long)gridEnv.width * (long)gridEnv.height; + dataSource.geometryType = PIXEL; + dataSource.attributes = attributes; + dataSource.coordinateSystem = coverage.getCoordinateReferenceSystem2D().getName().toString(); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java index 4cecbc103..6d20302c1 100644 --- a/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java @@ -1,6 +1,7 @@ package com.conveyal.analysis.datasource; import org.locationtech.jts.geom.Geometry; +import org.opengis.coverage.SampleDimensionType; import org.opengis.feature.type.AttributeDescriptor; import org.opengis.feature.type.AttributeType; @@ -46,10 +47,16 @@ public SpatialAttribute(String name, AttributeType type) { this.type = Type.forBindingClass(type.getBinding()); } - public SpatialAttribute(AttributeDescriptor descriptor) { + public SpatialAttribute (AttributeDescriptor descriptor) { this(descriptor.getLocalName(), descriptor.getType()); } + public SpatialAttribute (SampleDimensionType dimensionType, int bandNumber) { + name = "Band " + bandNumber; + label = String.format("%s (%s)", name, dimensionType.name()); + type = Type.NUMBER; + } + /** No-arg constructor required for Mongo POJO deserialization. */ public SpatialAttribute () { } diff --git a/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java index b0b7e27c6..33a733b0d 100644 --- a/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java @@ -81,7 +81,7 @@ public static FileStorageFormat detectUploadFormatAndValidate (List fi } else if (fileExtensions.contains("GPKG")) { uploadFormat = FileStorageFormat.GEOPACKAGE; } else if (fileExtensions.contains("TIFF") || fileExtensions.contains("TIF")) { - uploadFormat = FileStorageFormat.TIFF; + uploadFormat = FileStorageFormat.GEOTIFF; } if (uploadFormat == null) { diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index b800e247d..9d24e2d4e 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -78,7 +78,7 @@ private AggregationAreaDerivation (FileStorage fileStorage, AnalysisDB database, // Before kicking off asynchronous processing, range check inputs to fail fast on obvious problems. userPermissions = UserPermissions.from(req); dataSourceId = req.queryParams("dataSourceId"); - nameProperty = "dist_name"; // req.queryParams("nameProperty"); + nameProperty = req.queryParams("nameProperty"); //"dist_name"; // zoom = parseZoom(req.queryParams("zoom")); checkNotNull(dataSourceId); checkNotNull(nameProperty); diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java index 1f29cf79f..0dd4d97aa 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java @@ -34,11 +34,14 @@ public class SpatialDataSource extends DataSource { /** The number of features in this SpatialDataSource. */ - public int featureCount; + public long featureCount; /** All features in this SpatialDataSource have an attached geometry of this type. */ public ShapefileReader.GeometryType geometryType; + /** An EPSG code for the source's native coordinate system, or a WKT projection string. */ + public String coordinateSystem; + /** Every feature has this set of Attributes - this is essentially a schema giving attribute names and types. */ public List attributes; diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index ed85cd847..f58cc0105 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -92,6 +92,8 @@ public T create(T newModel, UserPermissions userPermissions) { /** * Note that if the supplied model has _id = null, the Mongo insertOne method will overwrite it with a new * ObjectId(). We consider it good practice to set the _id for any model object ourselves, avoiding this behavior. + * It looks like we could remove the OBJECT_ID_GENERATORS convention to force explicit ID creation. + * https://mongodb.github.io/mongo-java-driver/3.11/bson/pojos/#conventions */ public void insert (T model) { collection.insertOne(model); diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index ef07e0723..53fcd9d93 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -11,14 +11,14 @@ public enum FileStorageFormat { GRID("grid", "application/octet-stream"), POINTSET("pointset", "application/octet-stream"), // Why is this "pointset" extension duplicated? PNG("png", "image/png"), - TIFF("tiff", "image/tiff"), + GEOTIFF("tif", "image/tiff"), CSV("csv", "text/csv"), // SHP implies .dbf and .prj, and optionally .shx SHP("shp", "application/octet-stream"), - // These final ones are not yet used. - // In our internal storage, we may want to force less ambiguous .gtfs.zip .osm.pbf and .geo.json. + // Some of these are not yet used. + // In our internal storage, we may want to force less ambiguous .gtfs.zip .osm.pbf and .geojson. GTFS("zip", "application/zip"), OSMPBF("pbf", "application/octet-stream"), // Also can be application/geo+json, see https://www.iana.org/assignments/media-types/application/geo+json @@ -29,6 +29,7 @@ public enum FileStorageFormat { // These should not be serialized into Mongo. Default Enum codec uses String name() and valueOf(String). // TODO clarify whether the extension is used for backend storage, or for detecting type up uploaded files. + // TODO array of file extensions, with the first one used canonically in FileStorage and the others for detection. public final String extension; public final String mimeType; diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index c674a3e15..ab4e2276a 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -172,7 +172,7 @@ public static List attributes (SimpleFeatureType schema) { /** These are very broad. For example, line includes linestring and multilinestring. */ public enum GeometryType { - POLYGON, POINT, LINE; + POLYGON, POINT, LINE, PIXEL; public static GeometryType forBindingClass (Class binding) { if (Polygonal.class.isAssignableFrom(binding)) return POLYGON; if (Puntal.class.isAssignableFrom(binding)) return POINT; From 8c9dc645251b709d82765850a28ec35910f722df Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 21:39:51 +0800 Subject: [PATCH 104/187] Record CRS metadata for all SpatialDataSources. CRS name and code are often the same, but are often not EPSG codes. CRS.lookupIdentifier fails in some cases. --- .../analysis/datasource/GeoJsonDataSourceIngester.java | 1 + .../analysis/datasource/GeoPackageDataSourceIngester.java | 6 +++++- .../analysis/datasource/GeoTiffDataSourceIngester.java | 2 +- .../analysis/datasource/ShapefileDataSourceIngester.java | 4 ++++ src/main/java/com/conveyal/r5/util/ShapefileReader.java | 2 +- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 79d2d9987..540796a29 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -158,6 +158,7 @@ public void ingest (File file, ProgressListener progressListener) { } else { dataSource.geometryType = geometryTypes.get(0); } + dataSource.coordinateSystem = DefaultGeographicCRS.WGS84.getName().getCode(); progressListener.increment(); } catch (FactoryException | IOException e) { // Catch only checked exceptions to avoid excessive wrapping of root cause exception when possible. diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java index 704c5c4a2..215451223 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -15,11 +15,13 @@ import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.referencing.FactoryException; import java.io.File; import java.io.IOException; @@ -89,9 +91,11 @@ public void ingest (File file, ProgressListener progressListener) { dataSource.attributes = attributes(wgsFeatureCollection.getSchema()); dataSource.geometryType = geometryType(wgsFeatureCollection); dataSource.featureCount = wgsFeatureCollection.size(); + dataSource.coordinateSystem = + featureSource.getSchema().getCoordinateReferenceSystem().getName().getCode(); progressListener.increment(); } catch (IOException e) { - throw new RuntimeException("Error reading GeoPackage due to IOException.", e); + throw new RuntimeException("Error reading GeoPackage.", e); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java index e7a18c575..f591e785a 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java @@ -84,7 +84,7 @@ public void ingest (File file, ProgressListener progressListener) { dataSource.featureCount = (long)gridEnv.width * (long)gridEnv.height; dataSource.geometryType = PIXEL; dataSource.attributes = attributes; - dataSource.coordinateSystem = coverage.getCoordinateReferenceSystem2D().getName().toString(); + dataSource.coordinateSystem = coverage.getCoordinateReferenceSystem().getName().getCode(); } } diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index 157983a1c..a513a6bee 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -6,6 +6,7 @@ import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader; +import org.geotools.referencing.CRS; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.opengis.referencing.FactoryException; @@ -54,6 +55,9 @@ public void ingest (File file, ProgressListener progressListener) { dataSource.attributes = reader.attributes(); dataSource.geometryType = reader.geometryType(); dataSource.featureCount = reader.featureCount(); + dataSource.coordinateSystem = + reader.crs.getName().getCode(); + progressListener.increment(); } catch (FactoryException | TransformException e) { throw new DataSourceException("Shapefile transform error. " + diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index ab4e2276a..164e7ba26 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -52,7 +52,7 @@ public class ShapefileReader { private final FeatureCollection features; private final DataStore store; private final FeatureSource source; - private final CoordinateReferenceSystem crs; + public final CoordinateReferenceSystem crs; private final MathTransform transform; public ShapefileReader (File shapefile) throws IOException, FactoryException { From 9354c46b40effa862bf36eca4c7775456abf625c Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 21:42:38 +0800 Subject: [PATCH 105/187] report progress on geoTiff import --- .../conveyal/analysis/datasource/GeoTiffDataSourceIngester.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java index f591e785a..c63468ac1 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java @@ -48,6 +48,7 @@ protected DataSource dataSource () { @Override public void ingest (File file, ProgressListener progressListener) { + progressListener.beginTask("Processing uploaded GeoTIFF", 1); AbstractGridFormat format = GridFormatFinder.findFormat(file); Hints hints = new Hints(Hints.FORCE_LONGITUDE_FIRST_AXIS_ORDER, Boolean.TRUE); var coverageReader = format.getReader(file, hints); @@ -85,6 +86,7 @@ public void ingest (File file, ProgressListener progressListener) { dataSource.geometryType = PIXEL; dataSource.attributes = attributes; dataSource.coordinateSystem = coverage.getCoordinateReferenceSystem().getName().getCode(); + progressListener.increment(); } } From ae5b602640110dda3e0b1b8e9356a1f61687bf7e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 3 Sep 2021 21:46:58 +0800 Subject: [PATCH 106/187] placeholder for full delete --- .../analysis/controllers/DataSourceController.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 019a188d9..50679bd60 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -12,6 +12,7 @@ import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.HttpUtils; import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.Task; import org.apache.commons.fileupload.FileItem; import org.slf4j.Logger; @@ -79,11 +80,9 @@ private DataSource getOneDataSourceById (Request req, Response res) { /** HTTP DELETE: Delete a single DataSource record and associated files in FileStorage by supplied ID parameter. */ private String deleteOneDataSourceById (Request request, Response response) { long nDeleted = dataSourceCollection.deleteByIdParamIfPermitted(request).getDeletedCount(); + // TODO normalize to canonical file extensions so we can find them to delete them. + // fileStorage.delete(new FileStorageKey(DATA_SOURCE, _id + extension)); return "DELETE " + nDeleted; - // TODO delete files from storage - // TODO delete referencing database records - // Shouldn't this be deleting by ID instead of sending the whole document? - // TODO why do our delete methods return a list of documents? Can we just return the ID or HTTP status code? } private SpatialDataSource downloadLODES(Request req, Response res) { From b51ecbd6ec26488a9061c87195d0e93a72606bb7 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 6 Sep 2021 20:15:39 +0800 Subject: [PATCH 107/187] Remove unused GraphQL code and dependency --- build.gradle | 4 - .../analysis/AnalysisServerException.java | 14 -- .../com/conveyal/analysis/BackendMain.java | 1 - .../analysis/controllers/WrappedFeedInfo.java | 21 --- .../java/com/conveyal/gtfs/api/ApiMain.java | 58 ------ .../gtfs/api/graphql/GeoJsonCoercing.java | 64 ------- .../gtfs/api/graphql/GraphQLGtfsSchema.java | 113 ------------ .../graphql/WrappedEntityFieldFetcher.java | 37 ---- .../gtfs/api/graphql/WrappedGTFSEntity.java | 19 -- .../api/graphql/fetchers/FeedFetcher.java | 53 ------ .../api/graphql/fetchers/PatternFetcher.java | 100 ---------- .../api/graphql/fetchers/RouteFetcher.java | 113 ------------ .../api/graphql/fetchers/StopFetcher.java | 135 -------------- .../api/graphql/fetchers/StopTimeFetcher.java | 64 ------- .../api/graphql/fetchers/TripDataFetcher.java | 171 ------------------ .../gtfs/api/graphql/types/FeedType.java | 65 ------- .../gtfs/api/graphql/types/PatternType.java | 86 --------- .../gtfs/api/graphql/types/RouteType.java | 75 -------- .../gtfs/api/graphql/types/StopTimeType.java | 36 ---- .../gtfs/api/graphql/types/StopType.java | 65 ------- .../gtfs/api/graphql/types/TripType.java | 57 ------ .../com/conveyal/gtfs/api/util/GeomUtil.java | 30 --- .../conveyal/gtfs/api/util/GraphQLUtil.java | 90 --------- 23 files changed, 1471 deletions(-) delete mode 100644 src/main/java/com/conveyal/analysis/controllers/WrappedFeedInfo.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/ApiMain.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/GeoJsonCoercing.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/GraphQLGtfsSchema.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/WrappedEntityFieldFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/WrappedGTFSEntity.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/fetchers/FeedFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/fetchers/PatternFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/fetchers/RouteFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopTimeFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/fetchers/TripDataFetcher.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/types/FeedType.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/types/PatternType.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/types/RouteType.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/types/StopTimeType.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/types/StopType.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/graphql/types/TripType.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/util/GeomUtil.java delete mode 100644 src/main/java/com/conveyal/gtfs/api/util/GraphQLUtil.java diff --git a/build.gradle b/build.gradle index 50390bd7d..9f28c0f3c 100644 --- a/build.gradle +++ b/build.gradle @@ -201,10 +201,6 @@ dependencies { // Now used only for Seamless Census TODO eliminate this final AWS dependency implementation 'com.amazonaws:aws-java-sdk-s3:1.11.341' - // Old version of GraphQL-Java used by legacy gtfs-api embedded in analysis-backend. - // TODO eliminate GraphQL in future API revisions - implementation 'com.graphql-java:graphql-java:2.1.0' - // Commons Math gives us FastMath, MersenneTwister, and low-discrepancy vector generators. implementation 'org.apache.commons:commons-math3:3.0' diff --git a/src/main/java/com/conveyal/analysis/AnalysisServerException.java b/src/main/java/com/conveyal/analysis/AnalysisServerException.java index de160c667..de573c531 100644 --- a/src/main/java/com/conveyal/analysis/AnalysisServerException.java +++ b/src/main/java/com/conveyal/analysis/AnalysisServerException.java @@ -1,12 +1,9 @@ package com.conveyal.analysis; import com.conveyal.r5.util.ExceptionUtils; -import graphql.GraphQLError; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - public class AnalysisServerException extends RuntimeException { private static final Logger LOG = LoggerFactory.getLogger(AnalysisServerException.class); @@ -40,17 +37,6 @@ public static AnalysisServerException forbidden(String message) { return new AnalysisServerException(Type.FORBIDDEN, message, 403); } - public static AnalysisServerException graphQL(List errors) { - return new AnalysisServerException( - Type.GRAPHQL, - errors - .stream() - .map(e -> e.getMessage()) - .reduce("", (a, b) -> a + " " + b), - 400 - ); - } - public static AnalysisServerException nonce() { return new AnalysisServerException(Type.NONCE, "The data you attempted to change is out of date and could not be " + "updated. This project may be open by another user or in another browser tab.", 400); diff --git a/src/main/java/com/conveyal/analysis/BackendMain.java b/src/main/java/com/conveyal/analysis/BackendMain.java index c20239711..431537095 100644 --- a/src/main/java/com/conveyal/analysis/BackendMain.java +++ b/src/main/java/com/conveyal/analysis/BackendMain.java @@ -3,7 +3,6 @@ import com.conveyal.analysis.components.BackendComponents; import com.conveyal.analysis.components.LocalBackendComponents; import com.conveyal.analysis.persistence.Persistence; -import com.conveyal.gtfs.api.ApiMain; import com.conveyal.r5.SoftwareVersion; import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.WorkerCategory; diff --git a/src/main/java/com/conveyal/analysis/controllers/WrappedFeedInfo.java b/src/main/java/com/conveyal/analysis/controllers/WrappedFeedInfo.java deleted file mode 100644 index 08f5c5ed3..000000000 --- a/src/main/java/com/conveyal/analysis/controllers/WrappedFeedInfo.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.conveyal.analysis.controllers; - -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.FeedInfo; - -/** - * Wrap feed info with GTFS feed checksum and feed unique ID. - */ -public class WrappedFeedInfo extends WrappedGTFSEntity { - public long checksum; - - - /** - * Wrap the given GTFS entity with the unique Feed ID specified (this is not generally a GTFS feed ID as they - * are not unique between different versions of the same feed. Also pass in feed checksum. - */ - public WrappedFeedInfo(String feedUniqueID, FeedInfo entity, long checksum) { - super(feedUniqueID, entity); - this.checksum = checksum; - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/ApiMain.java b/src/main/java/com/conveyal/gtfs/api/ApiMain.java deleted file mode 100644 index 494b35033..000000000 --- a/src/main/java/com/conveyal/gtfs/api/ApiMain.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.conveyal.gtfs.api; - -import com.conveyal.gtfs.GTFSCache; -import com.conveyal.gtfs.GTFSFeed; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by landon on 2/3/16. - * TODO convert ApiMain into a Component once it's very simple. - */ -public class ApiMain { - - private static GTFSCache cache; - - public static final Logger LOG = LoggerFactory.getLogger(ApiMain.class); - - public static void initialize (GTFSCache cache) { - ApiMain.cache = cache; - } - - // TODO rename methods, we no longer have FeedSource. - private static GTFSFeed getFeedSource (String uniqueId) { - GTFSFeed feed = cache.get(uniqueId); - // The feedId of the GTFSFeed objects may not be unique - we can have multiple versions of the same feed - // covering different time periods, uploaded by different users. Therefore we record another ID here that is - // known to be unique across the whole application - the ID used to fetch the feed. - // TODO setting this field could be pushed down into cache.get() or even into the CacheLoader, but I'm doing - // it here to keep this a pure refactor for now. - feed.uniqueId = uniqueId; - return feed; - } - - /** - * Convenience function to get a feed source without throwing checked exceptions, for example for use in lambdas. - * @return the GTFSFeed for the given ID, or null if an exception occurs. - */ - public static GTFSFeed getFeedSourceWithoutExceptions (String id) { - try { - return getFeedSource(id); - } catch (Exception e) { - LOG.error("Error retrieving from cache feed " + id, e); - return null; - } - } - - // TODO verify that this is not used to fetch so many feeds that it will cause some of them to be closed by eviction - // TODO introduce checks on quantity of feeds, against max cache size, and fail hard if too many are requested. - public static List getFeedSources (List feedIds) { - return feedIds.stream() - .map(ApiMain::getFeedSourceWithoutExceptions) - .filter(fs -> fs != null) - .collect(Collectors.toList()); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/GeoJsonCoercing.java b/src/main/java/com/conveyal/gtfs/api/graphql/GeoJsonCoercing.java deleted file mode 100644 index 65f297967..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/GeoJsonCoercing.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.conveyal.gtfs.api.graphql; - -import graphql.schema.Coercing; -import org.locationtech.jts.geom.LineString; -import org.locationtech.jts.geom.MultiPolygon; - -import java.util.stream.Stream; - -/** - * Created by matthewc on 3/9/16. - */ -public class GeoJsonCoercing implements Coercing { - @Override - public Object serialize(Object input) { - // Handle newer org.locationtech JTS LineStrings - if (input instanceof LineString) { - GeoJsonLineString ret = new GeoJsonLineString(); - ret.coordinates = Stream.of(((LineString)input).getCoordinates()) - .map(c -> new double[] { c.x, c.y }) - .toArray(i -> new double[i][]); - - return ret; - } - // Also handle legacy com.vividsolutons JTS LineStrings, which are serialized into our MapDBs - else if (input instanceof com.vividsolutions.jts.geom.LineString) { - GeoJsonLineString ret = new GeoJsonLineString(); - ret.coordinates = Stream.of(((com.vividsolutions.jts.geom.LineString) input).getCoordinates()) - .map(c -> new double[] { c.x, c.y }) - .toArray(i -> new double[i][]); - - return ret; - } - else if (input instanceof MultiPolygon) { - MultiPolygon g = (MultiPolygon) input; - GeoJsonMultiPolygon ret = new GeoJsonMultiPolygon(); - ret.coordinates = Stream.of(g.getCoordinates()) - .map(c -> new double[] { c.x, c.y }) - .toArray(i -> new double[i][]); - - return ret; - } - else return null; - } - - @Override - public Object parseValue(Object o) { - return null; - } - - @Override - public Object parseLiteral(Object o) { - return null; - } - - private static class GeoJsonLineString { - public final String type = "LineString"; - public double[][] coordinates; - } - - private static class GeoJsonMultiPolygon { - public final String type = "MultiPolygon"; - public double[][] coordinates; - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/GraphQLGtfsSchema.java b/src/main/java/com/conveyal/gtfs/api/graphql/GraphQLGtfsSchema.java deleted file mode 100644 index ccd1274b3..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/GraphQLGtfsSchema.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.conveyal.gtfs.api.graphql; - -import com.conveyal.gtfs.api.graphql.fetchers.FeedFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.PatternFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.RouteFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.StopFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.StopTimeFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.TripDataFetcher; -import com.conveyal.gtfs.api.graphql.types.FeedType; -import com.conveyal.gtfs.api.graphql.types.PatternType; -import com.conveyal.gtfs.api.graphql.types.RouteType; -import com.conveyal.gtfs.api.graphql.types.StopTimeType; -import com.conveyal.gtfs.api.graphql.types.StopType; -import com.conveyal.gtfs.api.graphql.types.TripType; -import graphql.schema.*; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.*; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by matthewc on 3/9/16. - */ -public class GraphQLGtfsSchema { - public static GraphQLObjectType stopType = StopType.build(); - - public static GraphQLObjectType stopTimeType = StopTimeType.build(); - - public static GraphQLObjectType tripType = TripType.build(); - - public static GraphQLObjectType patternType = PatternType.build(); - - public static GraphQLObjectType routeType = RouteType.build(); - - public static GraphQLObjectType feedType = FeedType.build(); - - public static GraphQLObjectType rootQuery = newObject() - .name("rootQuery") - .description("Root level query for routes, stops, feeds, patterns, trips, and stopTimes within GTFS feeds.") - .field(newFieldDefinition() - .name("routes") - .description("List of GTFS routes optionally queried by route_id (feed_id required).") - .type(new GraphQLList(routeType)) - .argument(multiStringArg("route_id")) - .argument(multiStringArg("feed_id")) - .dataFetcher(RouteFetcher::apex) - .build() - ) - .field(newFieldDefinition() - .name("stops") - .type(new GraphQLList(stopType)) - .argument(multiStringArg("feed_id")) - .argument(multiStringArg("stop_id")) - .argument(multiStringArg("route_id")) - .argument(multiStringArg("pattern_id")) - .argument(floatArg("lat")) - .argument(floatArg("lon")) - .argument(floatArg("radius")) - .argument(floatArg("max_lat")) - .argument(floatArg("max_lon")) - .argument(floatArg("min_lat")) - .argument(floatArg("min_lon")) - .dataFetcher(StopFetcher::apex) - .build() - ) - .field(newFieldDefinition() - .name("feeds") - .argument(multiStringArg("feed_id")) - .dataFetcher(FeedFetcher::apex) - .type(new GraphQLList(feedType)) - .build() - ) - // TODO: determine if there's a better way to get at the refs for patterns, trips, and stopTimes than injecting them at the root. - .field(newFieldDefinition() - .name("patterns") - .type(new GraphQLList(patternType)) - .argument(multiStringArg("feed_id")) - .argument(multiStringArg("pattern_id")) - .argument(floatArg("lat")) - .argument(floatArg("lon")) - .argument(floatArg("radius")) - .argument(floatArg("max_lat")) - .argument(floatArg("max_lon")) - .argument(floatArg("min_lat")) - .argument(floatArg("min_lon")) - .dataFetcher(PatternFetcher::apex) - .build() - ) - .field(newFieldDefinition() - .name("trips") - .argument(multiStringArg("feed_id")) - .argument(multiStringArg("trip_id")) - .argument(multiStringArg("route_id")) - .dataFetcher(TripDataFetcher::apex) - .type(new GraphQLList(tripType)) - .build() - ) - .field(newFieldDefinition() - .name("stopTimes") - .argument(multiStringArg("feed_id")) - .argument(multiStringArg("stop_id")) - .argument(multiStringArg("trip_id")) - .dataFetcher(StopTimeFetcher::apex) - .type(new GraphQLList(stopTimeType)) - .build() - ) - .build(); - - - - public static GraphQLSchema schema = GraphQLSchema.newSchema().query(rootQuery).build(); - -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/WrappedEntityFieldFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/WrappedEntityFieldFetcher.java deleted file mode 100644 index dc707f9fd..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/WrappedEntityFieldFetcher.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.conveyal.gtfs.api.graphql; - -import graphql.schema.DataFetcher; -import graphql.schema.DataFetchingEnvironment; - -import java.lang.reflect.Field; - -/** - * Fetch data from wrapped GTFS entities. Modeled after graphql-java FieldDataFetcher. - */ -public class WrappedEntityFieldFetcher implements DataFetcher { - private final String field; - - public WrappedEntityFieldFetcher (String field) { - this.field = field; - } - - @Override - public Object get(DataFetchingEnvironment dataFetchingEnvironment) { - Object source = dataFetchingEnvironment.getSource(); - - if (source instanceof WrappedGTFSEntity) source = ((WrappedGTFSEntity) source).entity; - - Field field = null; - try { - field = source.getClass().getField(this.field); - } catch (NoSuchFieldException e) { - return null; - } - - try { - return field.get(source); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/WrappedGTFSEntity.java b/src/main/java/com/conveyal/gtfs/api/graphql/WrappedGTFSEntity.java deleted file mode 100644 index 70763e7b6..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/WrappedGTFSEntity.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.conveyal.gtfs.api.graphql; - -/** - * Wraps a GTFS entity, whose own ID may only be unique within the feed, decorating it with the unique ID of the feed - * it came from. - */ -public class WrappedGTFSEntity { - public T entity; - public String feedUniqueId; - - /** - * Wrap the given GTFS entity with the unique Feed ID specified (this is not generally a GTFS feed ID as they - * are not unique between different versions of the same feed. - */ - public WrappedGTFSEntity (String feedUniqueID, T entity) { - this.feedUniqueId = feedUniqueID; - this.entity = entity; - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/FeedFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/FeedFetcher.java deleted file mode 100644 index 61bf9339a..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/FeedFetcher.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.conveyal.gtfs.api.graphql.fetchers; - -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.ApiMain; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.FeedInfo; -import graphql.schema.DataFetchingEnvironment; -import org.locationtech.jts.geom.Geometry; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by matthewc on 3/10/16. - */ -public class FeedFetcher { - public static List> apex(DataFetchingEnvironment environment) { - List feedId = environment.getArgument("feed_id"); - return ApiMain.getFeedSources(feedId).stream() - .map(fs -> getFeedInfo(fs)) - .collect(Collectors.toList()); - - } - - private static WrappedGTFSEntity getFeedInfo(GTFSFeed feed) { - FeedInfo ret; - if (feed.feedInfo.size() > 0) { - ret = feed.feedInfo.values().iterator().next(); - } else { - ret = new FeedInfo(); - } - // NONE is a special value used in GTFS Lib feed info - if (ret.feed_id == null || "NONE".equals(ret.feed_id)) { - ret = ret.clone(); - ret.feed_id = feed.feedId; - } - return new WrappedGTFSEntity<>(feed.uniqueId, ret); - } - - public static Geometry getMergedBuffer(DataFetchingEnvironment env) { - WrappedGTFSEntity feedInfo = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(feedInfo.feedUniqueId); - if (feed == null) return null; - return feed.getMergedBuffers(); - } - - public static WrappedGTFSEntity forWrappedGtfsEntity (DataFetchingEnvironment env) { - WrappedGTFSEntity feedInfo = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(feedInfo.feedUniqueId); - if (feed == null) return null; - return getFeedInfo(feed); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/PatternFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/PatternFetcher.java deleted file mode 100644 index 43f960ac3..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/PatternFetcher.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.conveyal.gtfs.api.graphql.fetchers; - -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.ApiMain; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.Pattern; -import com.conveyal.gtfs.model.Route; -import com.conveyal.gtfs.model.Trip; -import graphql.schema.DataFetchingEnvironment; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * - * Created by matthewc on 3/9/16. - */ -public class PatternFetcher { - private static final Double DEFAULT_RADIUS = 1.0; // default 1 km search radius - - public static List> apex(DataFetchingEnvironment env) { - Collection feeds; - - List feedId = env.getArgument("feed_id"); - feeds = ApiMain.getFeedSources(feedId); - Map args = env.getArguments(); - List> patterns = new ArrayList<>(); - - for (GTFSFeed feed : feeds) { - if (env.getArgument("pattern_id") != null) { - List patternId = env.getArgument("pattern_id"); - patternId.stream() - .filter(feed.patterns::containsKey) - .map(feed.patterns::get) - .map(pattern -> new WrappedGTFSEntity(feed.uniqueId, pattern)) - .forEach(patterns::add); - } - else if (env.getArgument("route_id") != null) { - List routeId = (List) env.getArgument("route_id"); - feed.patterns.values().stream() - .filter(p -> routeId.contains(p.route_id)) - .map(pattern -> new WrappedGTFSEntity(feed.uniqueId, pattern)) - .forEach(patterns::add); - } - } - - return patterns; - } - public static List> fromRoute(DataFetchingEnvironment env) { - WrappedGTFSEntity route = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(route.feedUniqueId); - if (feed == null) return null; - - List stopIds = env.getArgument("stop_id"); - List patternId = env.getArgument("pattern_id"); - Long limit = env.getArgument("limit"); - - List> patterns = feed.patterns.values().stream() - .filter(p -> p.route_id.equals(route.entity.route_id)) - .map(p -> new WrappedGTFSEntity<>(feed.uniqueId, p)) - .collect(Collectors.toList()); - if (patternId != null) { - patterns.stream() - .filter(p -> patternId.contains(p.entity.pattern_id)) - .collect(Collectors.toList()); - } - if (stopIds != null) { - patterns.stream() - .filter(p -> !Collections.disjoint(p.entity.orderedStops, stopIds)) // disjoint returns true if no elements in common - .collect(Collectors.toList()); - } - if (limit != null) { - return patterns.stream().limit(limit).collect(Collectors.toList()); - } - return patterns; - } - - public static Long fromRouteCount(DataFetchingEnvironment env) { - WrappedGTFSEntity route = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(route.feedUniqueId); - if (feed == null) return null; - - return feed.patterns.values().stream() - .filter(p -> p.route_id.equals(route.entity.route_id)) - .count(); - } - - public static WrappedGTFSEntity fromTrip(DataFetchingEnvironment env) { - WrappedGTFSEntity trip = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(trip.feedUniqueId); - if (feed == null) return null; - - Pattern patt = feed.patterns.get(feed.patternForTrip.get(trip.entity.trip_id)); - return new WrappedGTFSEntity<>(feed.uniqueId, patt); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/RouteFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/RouteFetcher.java deleted file mode 100644 index 6c989e3a5..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/RouteFetcher.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.conveyal.gtfs.api.graphql.fetchers; - -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.ApiMain; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.FeedInfo; -import com.conveyal.gtfs.model.Pattern; -import com.conveyal.gtfs.model.Route; -import com.conveyal.gtfs.model.Stop; -import graphql.schema.DataFetchingEnvironment; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Created by matthewc on 3/10/16. - */ -public class RouteFetcher { - public static List> apex (DataFetchingEnvironment environment) { - Map args = environment.getArguments(); - - Collection feeds; - - List feedId = (List) args.get("feed_id"); - feeds = ApiMain.getFeedSources(feedId); - - List> routes = new ArrayList<>(); - - // TODO: clear up possible scope issues feed and route IDs - for (GTFSFeed feed : feeds) { - if (args.get("route_id") != null) { - List routeId = (List) args.get("route_id"); - routeId.stream() - .filter(feed.routes::containsKey) - .map(feed.routes::get) - .map(r -> new WrappedGTFSEntity(feed.uniqueId, r)) - .forEach(routes::add); - } - else { - feed.routes.values().stream().map(r -> new WrappedGTFSEntity<>(feed.uniqueId, r)).forEach(routes::add); - } - } - - return routes; - } - - public static List> fromStop(DataFetchingEnvironment environment) { - WrappedGTFSEntity stop = (WrappedGTFSEntity) environment.getSource(); - List routeIds = environment.getArgument("route_id"); - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(stop.feedUniqueId); - if (feed == null) return null; - - List> routes = feed.patterns.values().stream() - .filter(p -> p.orderedStops.contains(stop.entity.stop_id)) - .map(p -> feed.routes.get(p.route_id)) - .distinct() - .map(r -> new WrappedGTFSEntity<>(feed.uniqueId, r)) - .collect(Collectors.toList()); - - if (routeIds != null) { - return routes.stream() - .filter(r -> routeIds.contains(r.entity.route_id)) - .collect(Collectors.toList()); - } - else { - return routes; - } - } - - public static WrappedGTFSEntity fromPattern(DataFetchingEnvironment env) { - WrappedGTFSEntity pattern = (WrappedGTFSEntity) env.getSource(); - List routeIds = env.getArgument("route_id"); - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(pattern.feedUniqueId); - if (feed == null) return null; - - return new WrappedGTFSEntity<>(feed.uniqueId, feed.routes.get(pattern.entity.route_id)); - } - - public static List> fromFeed(DataFetchingEnvironment environment) { - WrappedGTFSEntity fi = (WrappedGTFSEntity) environment.getSource(); - List routeIds = environment.getArgument("route_id"); - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(fi.feedUniqueId); - if (feed == null) return null; - - if (routeIds != null) { - return routeIds.stream() - .filter(id -> id != null && feed.routes.containsKey(id)) - .map(feed.routes::get) - .map(r -> new WrappedGTFSEntity<>(feed.uniqueId, r)) - .collect(Collectors.toList()); - } - else { - return feed.routes.values().stream() - .map(r -> new WrappedGTFSEntity<>(feed.uniqueId, r)) - .collect(Collectors.toList()); - } - } - - public static Long fromFeedCount(DataFetchingEnvironment environment) { - WrappedGTFSEntity fi = (WrappedGTFSEntity) environment.getSource(); - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(fi.feedUniqueId); - if (feed == null) return null; - - return feed.routes.values().stream().count(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopFetcher.java deleted file mode 100644 index 3f7cb3b35..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopFetcher.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.conveyal.gtfs.api.graphql.fetchers; - -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.ApiMain; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.FeedInfo; -import com.conveyal.gtfs.model.Pattern; -import com.conveyal.gtfs.model.Stop; -import graphql.schema.DataFetchingEnvironment; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Created by matthewc on 3/9/16. - */ -public class StopFetcher { - private static final Logger LOG = LoggerFactory.getLogger(StopFetcher.class); - private static final Double DEFAULT_RADIUS = 1.0; // default 1 km search radius - - /** top level stops query (i.e. not inside a stoptime etc) */ - public static List> apex(DataFetchingEnvironment env) { - Map args = env.getArguments(); - - Collection feeds; - - List feedId = (List) args.get("feed_id"); - feeds = ApiMain.getFeedSources(feedId); - - List> stops = new ArrayList<>(); - - // TODO: clear up possible scope issues feed and stop IDs - for (GTFSFeed feed : feeds) { - if (args.get("stop_id") != null) { - List stopId = (List) args.get("stop_id"); - stopId.stream() - .filter(id -> id != null && feed.stops.containsKey(id)) - .map(feed.stops::get) - .map(s -> new WrappedGTFSEntity(feed.uniqueId, s)) - .forEach(stops::add); - } - // TODO: should pattern pre-empt route or should they operate together? - else if (args.get("pattern_id") != null) { - List patternId = (List) args.get("pattern_id"); - feed.patterns.values().stream() - .filter(p -> patternId.contains(p.pattern_id)) - .map(p -> feed.getOrderedStopListForTrip(p.associatedTrips.get(0))) - .flatMap(List::stream) - .map(feed.stops::get) - .distinct() - .map(stop -> new WrappedGTFSEntity(feed.uniqueId, stop)) - .forEach(stops::add); - } - else if (args.get("route_id") != null) { - List routeId = (List) args.get("route_id"); - feed.patterns.values().stream() - .filter(p -> routeId.contains(p.route_id)) - .map(p -> feed.getOrderedStopListForTrip(p.associatedTrips.get(0))) - .flatMap(List::stream) - .map(feed.stops::get) - .distinct() - .map(stop -> new WrappedGTFSEntity(feed.uniqueId, stop)) - .forEach(stops::add); - } - } - return stops; - } - - public static List> fromPattern(DataFetchingEnvironment environment) { - WrappedGTFSEntity pattern = (WrappedGTFSEntity) environment.getSource(); - - if (pattern.entity.associatedTrips.isEmpty()) { - LOG.warn("Empty pattern!"); - return Collections.emptyList(); - } - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(pattern.feedUniqueId); - if (feed == null) return null; - - return feed.getOrderedStopListForTrip(pattern.entity.associatedTrips.get(0)) - .stream() - .map(feed.stops::get) - .map(s -> new WrappedGTFSEntity<>(feed.uniqueId, s)) - .collect(Collectors.toList()); - } - - public static Long fromPatternCount(DataFetchingEnvironment environment) { - WrappedGTFSEntity pattern = (WrappedGTFSEntity) environment.getSource(); - - if (pattern.entity.associatedTrips.isEmpty()) { - LOG.warn("Empty pattern!"); - return 0L; - } - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(pattern.feedUniqueId); - if (feed == null) return null; - - return feed.getOrderedStopListForTrip(pattern.entity.associatedTrips.get(0)) - .stream().count(); - } - - public static List> fromFeed(DataFetchingEnvironment env) { - WrappedGTFSEntity fi = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(fi.feedUniqueId); - if (feed == null) return null; - - Collection stops = feed.stops.values(); - List stopIds = env.getArgument("stop_id"); - - if (stopIds != null) { - return stopIds.stream() - .filter(id -> id != null && feed.stops.containsKey(id)) - .map(feed.stops::get) - .map(s -> new WrappedGTFSEntity<>(feed.uniqueId, s)) - .collect(Collectors.toList()); - } - return stops.stream() - .map(s -> new WrappedGTFSEntity<>(feed.uniqueId, s)) - .collect(Collectors.toList()); - } - - public static Long fromFeedCount(DataFetchingEnvironment env) { - WrappedGTFSEntity fi = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(fi.feedUniqueId); - if (feed == null) return null; - Collection stops = feed.stops.values(); - return stops.stream().count(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopTimeFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopTimeFetcher.java deleted file mode 100644 index 2ce2c90e1..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/StopTimeFetcher.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.conveyal.gtfs.api.graphql.fetchers; - -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.ApiMain; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.StopTime; -import com.conveyal.gtfs.model.Trip; -import graphql.schema.DataFetchingEnvironment; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -/** - * Created by matthewc on 3/9/16. - */ -public class StopTimeFetcher { - public static List> apex(DataFetchingEnvironment env) { - Collection feeds; - - List feedId = env.getArgument("feed_id"); - feeds = ApiMain.getFeedSources(feedId); - - List> stopTimes = new ArrayList<>(); - - // TODO: clear up possible scope issues feed and stop IDs - for (GTFSFeed feed : feeds) { - if (env.getArgument("trip_id") != null) { - List tripId = env.getArgument("trip_id"); - tripId.stream() - .map(id -> feed.getOrderedStopTimesForTrip(id)) - .map(st -> new WrappedGTFSEntity(feed.uniqueId, st)) - .forEach(stopTimes::add); - } - } - - return stopTimes; - } - public static List> fromTrip(DataFetchingEnvironment env) { - WrappedGTFSEntity trip = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(trip.feedUniqueId); - if (feed == null) return null; - - List stopIds = env.getArgument("stop_id"); - - // get stop_times in order - Stream stopTimes = StreamSupport.stream(feed.getOrderedStopTimesForTrip(trip.entity.trip_id).spliterator(), false); - if (stopIds != null) { - return stopTimes - .filter(stopTime -> stopIds.contains(stopTime.stop_id)) - .map(st -> new WrappedGTFSEntity<>(feed.uniqueId, st)) - .collect(Collectors.toList()); - } - else { - return stopTimes - .map(st -> new WrappedGTFSEntity<>(feed.uniqueId, st)) - .collect(Collectors.toList()); - } - } - -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/TripDataFetcher.java b/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/TripDataFetcher.java deleted file mode 100644 index 6b242e938..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/fetchers/TripDataFetcher.java +++ /dev/null @@ -1,171 +0,0 @@ -package com.conveyal.gtfs.api.graphql.fetchers; - -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.api.ApiMain; -import com.conveyal.gtfs.api.graphql.WrappedGTFSEntity; -import com.conveyal.gtfs.model.Agency; -import com.conveyal.gtfs.model.Pattern; -import com.conveyal.gtfs.model.Route; -import com.conveyal.gtfs.model.StopTime; -import com.conveyal.gtfs.model.Trip; -import graphql.schema.DataFetchingEnvironment; -import org.mapdb.Fun; - -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static spark.Spark.halt; - -public class TripDataFetcher { - public static List> apex(DataFetchingEnvironment env) { - Collection feeds; - - List feedId = (List) env.getArgument("feed_id"); - feeds = ApiMain.getFeedSources(feedId); - - List> trips = new ArrayList<>(); - - // TODO: clear up possible scope issues feed and trip IDs - for (GTFSFeed feed : feeds) { - if (env.getArgument("trip_id") != null) { - List tripId = (List) env.getArgument("trip_id"); - tripId.stream() - .filter(feed.trips::containsKey) - .map(feed.trips::get) - .map(trip -> new WrappedGTFSEntity(feed.uniqueId, trip)) - .forEach(trips::add); - } - else if (env.getArgument("route_id") != null) { - List routeId = (List) env.getArgument("route_id"); - feed.trips.values().stream() - .filter(t -> routeId.contains(t.route_id)) - .map(trip -> new WrappedGTFSEntity(feed.uniqueId, trip)) - .forEach(trips::add); - } - else { - feed.trips.values().stream() - .map(trip -> new WrappedGTFSEntity(feed.uniqueId, trip)) - .forEach(trips::add); - } - } - - return trips; - } - - /** - * Fetch trip data given a route. - */ - public static List> fromRoute(DataFetchingEnvironment dataFetchingEnvironment) { - WrappedGTFSEntity route = (WrappedGTFSEntity) dataFetchingEnvironment.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(route.feedUniqueId); - if (feed == null) return null; - - return feed.trips.values().stream() - .filter(t -> t.route_id.equals(route.entity.route_id)) - .map(t -> new WrappedGTFSEntity<>(feed.uniqueId, t)) - .collect(Collectors.toList()); - } - - public static Long fromRouteCount(DataFetchingEnvironment dataFetchingEnvironment) { - WrappedGTFSEntity route = (WrappedGTFSEntity) dataFetchingEnvironment.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(route.feedUniqueId); - if (feed == null) return null; - - return feed.trips.values().stream() - .filter(t -> t.route_id.equals(route.entity.route_id)) - .count(); - } - - public static WrappedGTFSEntity fromStopTime (DataFetchingEnvironment env) { - WrappedGTFSEntity stopTime = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(stopTime.feedUniqueId); - if (feed == null) return null; - - Trip trip = feed.trips.get(stopTime.entity.trip_id); - - return new WrappedGTFSEntity<>(stopTime.feedUniqueId, trip); - } - - public static List> fromPattern (DataFetchingEnvironment env) { - WrappedGTFSEntity pattern = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(pattern.feedUniqueId); - if (feed == null) return null; - - Long beginTime = env.getArgument("begin_time"); - Long endTime = env.getArgument("end_time"); - - if (beginTime != null && endTime != null) { - String agencyId = feed.routes.get(pattern.entity.route_id).agency_id; - Agency agency = agencyId != null ? feed.agency.get(agencyId) : null; - if (beginTime >= endTime) { - halt(404, "end_time must be greater than begin_time."); - } - LocalDateTime beginDateTime = LocalDateTime.ofEpochSecond(beginTime, 0, ZoneOffset.UTC); - int beginSeconds = beginDateTime.getSecond(); - LocalDateTime endDateTime = LocalDateTime.ofEpochSecond(endTime, 0, ZoneOffset.UTC); - int endSeconds = endDateTime.getSecond(); - long days = ChronoUnit.DAYS.between(beginDateTime, endDateTime); - ZoneId zone = agency != null ? ZoneId.of(agency.agency_timezone) : ZoneId.systemDefault(); - Set services = feed.services.values().stream() - .filter(s -> { - for (int i = 0; i < days; i++) { - LocalDate date = beginDateTime.toLocalDate().plusDays(i); - if (s.activeOn(date)) { - return true; - } - } - return false; - }) - .map(s -> s.service_id) - .collect(Collectors.toSet()); - return pattern.entity.associatedTrips.stream().map(feed.trips::get) - .filter(t -> services.contains(t.service_id)) - .map(t -> new WrappedGTFSEntity<>(feed.uniqueId, t)) - .collect(Collectors.toList()); - } - else { - return pattern.entity.associatedTrips.stream().map(feed.trips::get) - .map(t -> new WrappedGTFSEntity<>(feed.uniqueId, t)) - .collect(Collectors.toList()); - } - } - - public static Long fromPatternCount (DataFetchingEnvironment env) { - WrappedGTFSEntity pattern = (WrappedGTFSEntity) env.getSource(); - - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(pattern.feedUniqueId); - if (feed == null) return null; - - return pattern.entity.associatedTrips.stream().map(feed.trips::get).count(); - } - - public static Integer getStartTime(DataFetchingEnvironment env) { - WrappedGTFSEntity trip = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(trip.feedUniqueId); - if (feed == null) return null; - - Map.Entry st = feed.stop_times.ceilingEntry(new Fun.Tuple2(trip.entity.trip_id, null)); - return st != null ? st.getValue().departure_time : null; - } - - public static Integer getDuration(DataFetchingEnvironment env) { - WrappedGTFSEntity trip = (WrappedGTFSEntity) env.getSource(); - GTFSFeed feed = ApiMain.getFeedSourceWithoutExceptions(trip.feedUniqueId); - if (feed == null) return null; - - Integer startTime = getStartTime(env); - Map.Entry endStopTime = feed.stop_times.floorEntry(new Fun.Tuple2(trip.entity.trip_id, Fun.HI)); - - if (startTime == null || endStopTime == null || endStopTime.getValue().arrival_time < startTime) return null; - else return endStopTime.getValue().arrival_time - startTime; - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/types/FeedType.java b/src/main/java/com/conveyal/gtfs/api/graphql/types/FeedType.java deleted file mode 100644 index 2385b0dd6..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/types/FeedType.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.conveyal.gtfs.api.graphql.types; - -import com.conveyal.gtfs.api.graphql.fetchers.FeedFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.RouteFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.StopFetcher; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLTypeReference; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.lineString; -import static com.conveyal.gtfs.api.util.GraphQLUtil.multiStringArg; -import static com.conveyal.gtfs.api.util.GraphQLUtil.string; -import static graphql.Scalars.GraphQLLong; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by landon on 10/3/16. - */ -public class FeedType { - public static GraphQLObjectType build () { - return newObject() - .name("feed") - .description("Provides information for a GTFS feed and access to the entities it contains") - .field(string("feed_id")) - .field(string("feed_publisher_name")) - .field(string("feed_publisher_url")) - .field(string("feed_lang")) - .field(string("feed_version")) - .field(newFieldDefinition() - .name("routes") - .type(new GraphQLList(new GraphQLTypeReference("route"))) - .argument(multiStringArg("route_id")) - .dataFetcher(RouteFetcher::fromFeed) - .build() - ) - .field(newFieldDefinition() - .type(GraphQLLong) - .name("route_count") - .dataFetcher(RouteFetcher::fromFeedCount) - .build() - ) - .field(newFieldDefinition() - .name("stops") - .type(new GraphQLList(new GraphQLTypeReference("stop"))) - .argument(multiStringArg("stop_id")) - .dataFetcher(StopFetcher::fromFeed) - .build() - ) - .field(newFieldDefinition() - .type(GraphQLLong) - .name("stop_count") - .dataFetcher(StopFetcher::fromFeedCount) - .build() - ) - .field(newFieldDefinition() - .name("mergedBuffer") - .type(lineString()) - .description("Merged buffers around all stops in feed") - .dataFetcher(FeedFetcher::getMergedBuffer) - .build() - ) - .build(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/types/PatternType.java b/src/main/java/com/conveyal/gtfs/api/graphql/types/PatternType.java deleted file mode 100644 index ccf054eed..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/types/PatternType.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.conveyal.gtfs.api.graphql.types; - -import com.conveyal.gtfs.api.graphql.WrappedEntityFieldFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.RouteFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.StopFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.TripDataFetcher; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLTypeReference; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.doublee; -import static com.conveyal.gtfs.api.util.GraphQLUtil.feed; -import static com.conveyal.gtfs.api.util.GraphQLUtil.lineString; -import static com.conveyal.gtfs.api.util.GraphQLUtil.longArg; -import static com.conveyal.gtfs.api.util.GraphQLUtil.string; -import static graphql.Scalars.GraphQLLong; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by landon on 10/3/16. - */ -public class PatternType { - public static GraphQLObjectType build () { - GraphQLObjectType patternStats = newObject() - .name("patternStats") - .description("Statistics about a pattern") - .field(doublee("headway")) - .field(doublee("avgSpeed")) - .field(doublee("stopSpacing")) - .build(); - - return newObject() - .name("pattern") - .description("A unique sequence of stops that a GTFS route visits") - .field(string("pattern_id")) - .field(string("name")) - .field(string("route_id")) - .field(feed()) - .field(newFieldDefinition() - .name("route") - .description("Route that pattern operates along") - .dataFetcher(RouteFetcher::fromPattern) - .type(new GraphQLTypeReference("route")) - .build() - ) - .field(newFieldDefinition() - .name("stops") - .description("Stops that pattern serves") - .dataFetcher(StopFetcher::fromPattern) - .type(new GraphQLList(new GraphQLTypeReference("stop"))) - .build() - ) - .field(newFieldDefinition() - .type(GraphQLLong) - .description("Count of stops that pattern serves") - .name("stop_count") - .dataFetcher(StopFetcher::fromPatternCount) - .build() - ) - .field(newFieldDefinition() - .type(lineString()) - .description("Geometry that pattern operates along") - .dataFetcher(new WrappedEntityFieldFetcher("geometry")) - .name("geometry") - .build() - ) - .field(newFieldDefinition() - .name("trips") - .description("Trips associated with pattern") - .type(new GraphQLList(new GraphQLTypeReference("trip"))) - .dataFetcher(TripDataFetcher::fromPattern) - .argument(longArg("begin_time")) - .argument(longArg("end_time")) - .build() - ) - .field(newFieldDefinition() - .type(GraphQLLong) - .description("Count of trips associated with pattern") - .name("trip_count") - .dataFetcher(TripDataFetcher::fromPatternCount) - .build() - ) - .build(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/types/RouteType.java b/src/main/java/com/conveyal/gtfs/api/graphql/types/RouteType.java deleted file mode 100644 index 92fea837b..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/types/RouteType.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.conveyal.gtfs.api.graphql.types; - -import com.conveyal.gtfs.api.graphql.fetchers.PatternFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.TripDataFetcher; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLTypeReference; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.doublee; -import static com.conveyal.gtfs.api.util.GraphQLUtil.feed; -import static com.conveyal.gtfs.api.util.GraphQLUtil.intt; -import static com.conveyal.gtfs.api.util.GraphQLUtil.longArg; -import static com.conveyal.gtfs.api.util.GraphQLUtil.multiStringArg; -import static com.conveyal.gtfs.api.util.GraphQLUtil.string; -import static graphql.Scalars.GraphQLLong; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by landon on 10/3/16. - */ -public class RouteType { - public static GraphQLObjectType build () { - // routeStats should be modeled after com.conveyal.gtfs.stats.model.RouteStatistic - GraphQLObjectType routeStats = newObject() - .name("routeStats") - .description("Statistics about a route") - .field(doublee("headway")) - .field(doublee("avgSpeed")) - .field(doublee("stopSpacing")) - .build(); - - return newObject() - .name("route") - .description("A GTFS route object") - .field(string("route_id")) - // TODO agency - .field(string("route_short_name")) - .field(string("route_long_name")) - .field(string("route_desc")) - .field(string("route_url")) - // TODO route_type as enum - .field(intt("route_type")) - .field(string("route_color")) - .field(string("route_text_color")) - .field(feed()) - .field(newFieldDefinition() - .type(new GraphQLList(new GraphQLTypeReference("trip"))) - .name("trips") - .dataFetcher(TripDataFetcher::fromRoute) - .build() - ) - .field(newFieldDefinition() - .type(GraphQLLong) - .name("trip_count") - .dataFetcher(TripDataFetcher::fromRouteCount) - .build() - ) - .field(newFieldDefinition() - .type(new GraphQLList(new GraphQLTypeReference("pattern"))) - .name("patterns") - .argument(multiStringArg("stop_id")) - .argument(multiStringArg("pattern_id")) - .argument(longArg("limit")) - .dataFetcher(PatternFetcher::fromRoute) - .build() - ) - .field(newFieldDefinition() - .type(GraphQLLong) - .name("pattern_count") - .dataFetcher(PatternFetcher::fromRouteCount) - .build() - ) - .build(); - }} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/types/StopTimeType.java b/src/main/java/com/conveyal/gtfs/api/graphql/types/StopTimeType.java deleted file mode 100644 index c1877592d..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/types/StopTimeType.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.conveyal.gtfs.api.graphql.types; - -import com.conveyal.gtfs.api.graphql.fetchers.TripDataFetcher; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLTypeReference; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.*; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by landon on 10/3/16. - */ -public class StopTimeType { - public static GraphQLObjectType build () { - return newObject() - .name("stopTime") - .field(intt("arrival_time")) - .field(intt("departure_time")) - .field(intt("stop_sequence")) - .field(string("stop_id")) - .field(string("stop_headsign")) - .field(doublee("shape_dist_traveled")) - .field(feed()) - .field(newFieldDefinition() - .name("trip") - .type(new GraphQLTypeReference("trip")) - .dataFetcher(TripDataFetcher::fromStopTime) - .argument(stringArg("date")) - .argument(longArg("from")) - .argument(longArg("to")) - .build() - ) - .build(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/types/StopType.java b/src/main/java/com/conveyal/gtfs/api/graphql/types/StopType.java deleted file mode 100644 index a14149179..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/types/StopType.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.conveyal.gtfs.api.graphql.types; - -import com.conveyal.gtfs.api.graphql.fetchers.RouteFetcher; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLTypeReference; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.doublee; -import static com.conveyal.gtfs.api.util.GraphQLUtil.feed; -import static com.conveyal.gtfs.api.util.GraphQLUtil.intt; -import static com.conveyal.gtfs.api.util.GraphQLUtil.multiStringArg; -import static com.conveyal.gtfs.api.util.GraphQLUtil.string; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by landon on 10/3/16. - */ -public class StopType { - public static GraphQLObjectType build () { - - // transferPerformance should be modeled after com.conveyal.gtfs.stats.model.TransferPerformanceSummary - GraphQLObjectType transferPerformance = newObject() - .name("transferPerformance") - .description("Transfer performance for a stop") - .field(string("fromRoute")) - .field(string("toRoute")) - .field(intt("bestCase")) - .field(intt("worstCase")) - .field(intt("typicalCase")) - .build(); - - // stopStats should be modeled after com.conveyal.gtfs.stats.model.StopStatistic - GraphQLObjectType stopStats = newObject() - .name("stopStats") - .description("Statistics about a stop") - .field(doublee("headway")) - .field(intt("tripCount")) - .field(intt("routeCount")) - .build(); - - return newObject() - .name("stop") - .description("A GTFS stop object") - .field(string("stop_id")) - .field(string("stop_name")) - .field(string("stop_code")) - .field(string("stop_desc")) - .field(doublee("stop_lon")) - .field(doublee("stop_lat")) - .field(string("zone_id")) - .field(string("stop_url")) - .field(string("stop_timezone")) - .field(feed()) - .field(newFieldDefinition() - .name("routes") - .description("The list of routes that serve a stop") - .type(new GraphQLList(new GraphQLTypeReference("route"))) - .argument(multiStringArg("route_id")) - .dataFetcher(RouteFetcher::fromStop) - .build() - ) - .build(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/graphql/types/TripType.java b/src/main/java/com/conveyal/gtfs/api/graphql/types/TripType.java deleted file mode 100644 index fcf1c2bad..000000000 --- a/src/main/java/com/conveyal/gtfs/api/graphql/types/TripType.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.conveyal.gtfs.api.graphql.types; - -import com.conveyal.gtfs.api.graphql.fetchers.PatternFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.StopTimeFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.TripDataFetcher; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLTypeReference; - -import static com.conveyal.gtfs.api.util.GraphQLUtil.*; -import static graphql.Scalars.GraphQLInt; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; -import static graphql.schema.GraphQLObjectType.newObject; - -/** - * Created by landon on 10/3/16. - */ -public class TripType { - public static GraphQLObjectType build () { - return newObject() - .name("trip") - .field(string("trip_id")) - .field(string("trip_headsign")) - .field(string("trip_short_name")) - .field(string("block_id")) - .field(intt("direction_id")) - .field(string("route_id")) - .field(feed()) - .field(newFieldDefinition() - .name("pattern") - .type(new GraphQLTypeReference("pattern")) - .dataFetcher(PatternFetcher::fromTrip) - .build() - ) - .field(newFieldDefinition() - .name("stop_times") - .type(new GraphQLList(new GraphQLTypeReference("stopTime"))) - .argument(multiStringArg("stop_id")) - .dataFetcher(StopTimeFetcher::fromTrip) - .build() - ) - // some pseudo-fields to reduce the amount of data that has to be fetched - .field(newFieldDefinition() - .name("start_time") - .type(GraphQLInt) - .dataFetcher(TripDataFetcher::getStartTime) - .build() - ) - .field(newFieldDefinition() - .name("duration") - .type(GraphQLInt) - .dataFetcher(TripDataFetcher::getDuration) - .build() - ) - .build(); - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/util/GeomUtil.java b/src/main/java/com/conveyal/gtfs/api/util/GeomUtil.java deleted file mode 100644 index 731144cfa..000000000 --- a/src/main/java/com/conveyal/gtfs/api/util/GeomUtil.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.conveyal.gtfs.api.util; - -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.CoordinateList; -import org.locationtech.jts.geom.Envelope; - -/** - * Created by landon on 2/8/16. - */ -public class GeomUtil { - public static Envelope getBoundingBox(Coordinate coordinate, Double radius){ - Envelope boundingBox; - - double R = 6371; // earth radius in km - - // radius argument is also in km - - double x1 = coordinate.x - Math.toDegrees(radius/R/Math.cos(Math.toRadians(coordinate.y))); - - double x2 = coordinate.x + Math.toDegrees(radius/R/Math.cos(Math.toRadians(coordinate.y))); - - double y1 = coordinate.y + Math.toDegrees(radius/R); - - double y2 = coordinate.y - Math.toDegrees(radius/R); - - boundingBox = new Envelope(x1, x2, y1, y2); - - return boundingBox; - } -} diff --git a/src/main/java/com/conveyal/gtfs/api/util/GraphQLUtil.java b/src/main/java/com/conveyal/gtfs/api/util/GraphQLUtil.java deleted file mode 100644 index de321b8bc..000000000 --- a/src/main/java/com/conveyal/gtfs/api/util/GraphQLUtil.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.conveyal.gtfs.api.util; - -import com.conveyal.gtfs.api.graphql.GeoJsonCoercing; -import com.conveyal.gtfs.api.graphql.WrappedEntityFieldFetcher; -import com.conveyal.gtfs.api.graphql.fetchers.FeedFetcher; -import graphql.schema.DataFetchingEnvironment; -import graphql.schema.GraphQLArgument; -import graphql.schema.GraphQLFieldDefinition; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLScalarType; -import graphql.schema.GraphQLTypeReference; - -import static graphql.Scalars.*; -import static graphql.schema.GraphQLArgument.newArgument; -import static graphql.schema.GraphQLFieldDefinition.newFieldDefinition; - -/** - * Created by landon on 10/3/16. - */ -public class GraphQLUtil { - - public static GraphQLScalarType lineString () { - return new GraphQLScalarType("GeoJSON", "GeoJSON", new GeoJsonCoercing()); - } - - public static GraphQLFieldDefinition string (String name) { - return newFieldDefinition() - .name(name) - .type(GraphQLString) - .dataFetcher(new WrappedEntityFieldFetcher(name)) - .build(); - } - - public static GraphQLFieldDefinition intt (String name) { - return newFieldDefinition() - .name(name) - .type(GraphQLInt) - .dataFetcher(new WrappedEntityFieldFetcher(name)) - .build(); - } - - public static GraphQLFieldDefinition doublee (String name) { - return newFieldDefinition() - .name(name) - .type(GraphQLFloat) - .dataFetcher(new WrappedEntityFieldFetcher(name)) - .build(); - } - - public static GraphQLFieldDefinition feed () { - return newFieldDefinition() - .name("feed") - .description("Containing feed") - .dataFetcher(FeedFetcher::forWrappedGtfsEntity) - .type(new GraphQLTypeReference("feed")) - .build(); - } - - public static GraphQLArgument stringArg (String name) { - return newArgument() - .name(name) - .type(GraphQLString) - .build(); - } - - public static GraphQLArgument multiStringArg (String name) { - return newArgument() - .name(name) - .type(new GraphQLList(GraphQLString)) - .build(); - } - - public static GraphQLArgument floatArg (String name) { - return newArgument() - .name(name) - .type(GraphQLFloat) - .build(); - } - - public static GraphQLArgument longArg (String name) { - return newArgument() - .name(name) - .type(GraphQLLong) - .build(); - } - - public static boolean argumentDefined(DataFetchingEnvironment env, String name) { - return (env.containsArgument(name) && env.getArgument(name) != null); - } -} From b654f7abef8bd5a61a6bbe85c4c786e9b09c6ff2 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 6 Sep 2021 21:45:37 +0800 Subject: [PATCH 108/187] Do not close the feed after using --- .../java/com/conveyal/analysis/controllers/GTFSController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index 12a8b2ed1..d43cdfe3e 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -181,7 +181,6 @@ private List getAllStops (Request req, Response res) { String bundleScopedFeedId = Bundle.bundleScopeFeedId(feedSummary.feedId, feedGroupId); GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); allStopsByFeed.add(new AllStopsAPIResponse(feed)); - feed.close(); } return allStopsByFeed; } From 8c34a4bb1ed05cd9a2329b7524b6892f68fa13a7 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 6 Sep 2021 21:51:03 +0800 Subject: [PATCH 109/187] Correct comment --- .../com/conveyal/analysis/components/broker/WorkerTags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java b/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java index ceb564694..cab86bdb4 100644 --- a/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java +++ b/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java @@ -18,7 +18,7 @@ public class WorkerTags { /** The UUID for the project. */ public final String projectId; - /** The UUID for the project. */ + /** The UUID for the region. */ public final String regionId; public WorkerTags (String group, String user, String projectId, String regionId) { From fb703524cbb24c4230498320584ea50af792159c Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 6 Sep 2021 21:56:33 +0800 Subject: [PATCH 110/187] Update names of methods and classes for clarity --- .../analysis/controllers/GTFSController.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java index d43cdfe3e..f58783083 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GTFSController.java @@ -52,11 +52,11 @@ private void addImmutableResponseHeader (Response res) { res.header("Cache-Control", cacheControlImmutable); } - private static class BaseIdentifier { + private static class BaseAPIResponse { public final String _id; public final String name; - BaseIdentifier (String _id, String name) { + BaseAPIResponse(String _id, String name) { this._id = _id; this.name = name; } @@ -72,7 +72,7 @@ private GTFSFeed getFeedFromRequest (Request req) { return gtfsCache.get(bundleScopedFeedId); } - static class RouteAPIResponse extends BaseIdentifier { + static class RouteAPIResponse extends BaseAPIResponse { public final int type; public final String color; @@ -106,7 +106,7 @@ private List getRoutes(Request req, Response res) { .collect(Collectors.toList()); } - static class PatternAPIResponse extends BaseIdentifier { + static class PatternAPIResponse extends BaseAPIResponse { public final GeoJSONLineString geometry; public final List orderedStopIds; public final List associatedTripIds; @@ -140,7 +140,7 @@ private List getPatternsForRoute (Request req, Response res) .collect(Collectors.toList()); } - static class StopAPIResponse extends BaseIdentifier { + static class StopAPIResponse extends BaseAPIResponse { public final double lat; public final double lon; @@ -151,23 +151,23 @@ static class StopAPIResponse extends BaseIdentifier { } } - private List getStops (Request req, Response res) { + private List getAllStopsForOneFeed(Request req, Response res) { addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); return feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); } - static class AllStopsAPIResponse { + static class FeedGroupStopsAPIResponse { public final String feedId; public final List stops; - AllStopsAPIResponse(GTFSFeed feed) { + FeedGroupStopsAPIResponse(GTFSFeed feed) { this.feedId = feed.feedId; this.stops = feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); } } - private List getAllStops (Request req, Response res) { + private List getAllStopsForFeedGroup(Request req, Response res) { addImmutableResponseHeader(res); String feedGroupId = req.params("feedGroupId"); DBCursor cursor = Persistence.bundles.find(QueryBuilder.start("feedGroupId").is(feedGroupId).get()); @@ -175,17 +175,17 @@ private List getAllStops (Request req, Response res) { throw AnalysisServerException.notFound("Bundle could not be found for the given feed group ID."); } - List allStopsByFeed = new ArrayList<>(); + List allStopsByFeed = new ArrayList<>(); Bundle bundle = cursor.next(); for (Bundle.FeedSummary feedSummary : bundle.feeds) { String bundleScopedFeedId = Bundle.bundleScopeFeedId(feedSummary.feedId, feedGroupId); GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); - allStopsByFeed.add(new AllStopsAPIResponse(feed)); + allStopsByFeed.add(new FeedGroupStopsAPIResponse(feed)); } return allStopsByFeed; } - static class TripAPIResponse extends BaseIdentifier { + static class TripAPIResponse extends BaseAPIResponse { public final String headsign; public final Integer startTime; public final Integer duration; @@ -223,11 +223,11 @@ private List getTripsForRoute (Request req, Response res) { @Override public void registerEndpoints (spark.Service sparkService) { - sparkService.get("/api/gtfs/:feedGroupId/stops", this::getAllStops, toJson); + sparkService.get("/api/gtfs/:feedGroupId/stops", this::getAllStopsForFeedGroup, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes", this::getRoutes, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId", this::getRoute, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId/patterns", this::getPatternsForRoute, toJson); sparkService.get("/api/gtfs/:feedGroupId/:feedId/routes/:routeId/trips", this::getTripsForRoute, toJson); - sparkService.get("/api/gtfs/:feedGroupId/:feedId/stops", this::getStops, toJson); + sparkService.get("/api/gtfs/:feedGroupId/:feedId/stops", this::getAllStopsForOneFeed, toJson); } } From 13fcb8ade3d1da60b42c7834b71e20a84bae1824 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 6 Sep 2021 21:45:38 +0800 Subject: [PATCH 111/187] update ErrorEvent to include http path also factored out code to prepend user and group to stacktrace --- .../conveyal/analysis/components/HttpApi.java | 2 +- .../components/eventbus/ErrorEvent.java | 49 ++++++++++++++++++- .../components/eventbus/ErrorLogger.java | 3 +- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index ab0e7ef81..72659cb78 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -163,7 +163,7 @@ private void respondToException(Exception e, Request request, Response response, AnalysisServerException.Type type, String message, int code) { // Stacktrace in ErrorEvent reused below to avoid repeatedly generating String of stacktrace. - ErrorEvent errorEvent = new ErrorEvent(e); + ErrorEvent errorEvent = new ErrorEvent(e, request.pathInfo()); eventBus.send(errorEvent.forUser(request.attribute(USER_PERMISSIONS_ATTRIBUTE))); JSONObject body = new JSONObject(); diff --git a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java index bc86bebc0..1a695d133 100644 --- a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java +++ b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java @@ -17,11 +17,58 @@ public class ErrorEvent extends Event { public final String summary; + /** + * The path portion of the HTTP URL, if the error has occurred while responding to an HTTP request from a user. + * May be null if this information is unavailable or unknown (in components where user information is not retained). + */ + public final String httpPath; + public final String stackTrace; - public ErrorEvent (Throwable throwable) { + public ErrorEvent (Throwable throwable, String httpPath) { this.summary = ExceptionUtils.shortCauseString(throwable); this.stackTrace = ExceptionUtils.stackTraceString(throwable); + this.httpPath = httpPath; + } + + public ErrorEvent (Throwable throwable) { + this(throwable, null); + } + + /** Return a string intended for logging on Slack or the console. */ + public String traceWithContext (boolean verbose) { + StringBuilder builder = new StringBuilder(); + builder.append("User "); + builder.append(user); + builder.append(" of group "); + builder.append(accessGroup); + if (httpPath != null) { + builder.append(" accessing "); + builder.append(httpPath); + } + builder.append(": "); + if (verbose) { + builder.append(stackTrace); + } else { + builder.append(filterStackTrace(stackTrace)); + } + return builder.toString(); + } + + private static String filterStackTrace (String stackTrace) { + if (stackTrace == null) return null; + final String unknownFrame = "at unknown frame"; + String error = stackTrace.lines().findFirst().get(); + String frame = stackTrace.lines() + .map(String::strip) + .filter(s -> s.startsWith("at ")) + .findFirst().orElse(unknownFrame); + String conveyalFrame = stackTrace.lines() + .map(String::strip) + .filter(s -> s.startsWith("at com.conveyal.")) + .filter(frame::equals) + .findFirst().orElse(""); + return String.join("\n", error, frame, conveyalFrame); } } diff --git a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorLogger.java b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorLogger.java index 539753aec..e1be9b004 100644 --- a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorLogger.java +++ b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorLogger.java @@ -15,7 +15,8 @@ public class ErrorLogger implements EventHandler { public void handleEvent (Event event) { if (event instanceof ErrorEvent) { ErrorEvent errorEvent = (ErrorEvent) event; - LOG.error("User {} of {}: {}", errorEvent.user, errorEvent.accessGroup, errorEvent.stackTrace); + // Verbose message (full stack traces) for console logs. + LOG.error(errorEvent.traceWithContext(true)); } } From 8218cfeb98962c25841daf7db0791443a743b707 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 6 Sep 2021 22:05:52 +0800 Subject: [PATCH 112/187] Rename GTFS -> Gtfs and API -> Api --- .../components/BackendComponents.java | 4 +- ...TFSController.java => GtfsController.java} | 66 +++++++++---------- 2 files changed, 35 insertions(+), 35 deletions(-) rename src/main/java/com/conveyal/analysis/controllers/{GTFSController.java => GtfsController.java} (79%) diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 59cdb9f07..71afe7876 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -7,7 +7,7 @@ import com.conveyal.analysis.controllers.BrokerController; import com.conveyal.analysis.controllers.BundleController; import com.conveyal.analysis.controllers.FileStorageController; -import com.conveyal.analysis.controllers.GTFSController; +import com.conveyal.analysis.controllers.GtfsController; import com.conveyal.analysis.controllers.GtfsTileController; import com.conveyal.analysis.controllers.HttpController; import com.conveyal.analysis.controllers.OpportunityDatasetController; @@ -83,7 +83,7 @@ public List standardHttpControllers () { return Lists.newArrayList( // These handlers are at paths beginning with /api // and therefore subject to authentication and authorization. - new GTFSController(gtfsCache), + new GtfsController(gtfsCache), new BundleController(this), new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor), new RegionalAnalysisController(broker, fileStorage), diff --git a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java b/src/main/java/com/conveyal/analysis/controllers/GtfsController.java similarity index 79% rename from src/main/java/com/conveyal/analysis/controllers/GTFSController.java rename to src/main/java/com/conveyal/analysis/controllers/GtfsController.java index f58783083..151345c25 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GTFSController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GtfsController.java @@ -33,9 +33,9 @@ * CDN in the future. Everything retrieved is immutable. Once it's retrieved and stored in the CDN, it doesn't need to * be pulled from the cache again. */ -public class GTFSController implements HttpController { +public class GtfsController implements HttpController { private final GTFSCache gtfsCache; - public GTFSController (GTFSCache gtfsCache) { + public GtfsController(GTFSCache gtfsCache) { this.gtfsCache = gtfsCache; } @@ -52,17 +52,17 @@ private void addImmutableResponseHeader (Response res) { res.header("Cache-Control", cacheControlImmutable); } - private static class BaseAPIResponse { + private static class BaseApiResponse { public final String _id; public final String name; - BaseAPIResponse(String _id, String name) { + BaseApiResponse(String _id, String name) { this._id = _id; this.name = name; } } - private static class GeoJSONLineString { + private static class GeoJsonLineString { public final String type = "LineString"; public double[][] coordinates; } @@ -72,7 +72,7 @@ private GTFSFeed getFeedFromRequest (Request req) { return gtfsCache.get(bundleScopedFeedId); } - static class RouteAPIResponse extends BaseAPIResponse { + static class RouteApiResponse extends BaseApiResponse { public final int type; public final String color; @@ -83,43 +83,43 @@ static String getRouteName (Route route) { return tempName.trim(); } - RouteAPIResponse(Route route) { + RouteApiResponse(Route route) { super(route.route_id, getRouteName(route)); color = route.route_color; type = route.route_type; } } - private RouteAPIResponse getRoute(Request req, Response res) { + private RouteApiResponse getRoute(Request req, Response res) { addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); - return new RouteAPIResponse(feed.routes.get(req.params("routeId"))); + return new RouteApiResponse(feed.routes.get(req.params("routeId"))); } - private List getRoutes(Request req, Response res) { + private List getRoutes(Request req, Response res) { addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); return feed.routes .values() .stream() - .map(RouteAPIResponse::new) + .map(RouteApiResponse::new) .collect(Collectors.toList()); } - static class PatternAPIResponse extends BaseAPIResponse { - public final GeoJSONLineString geometry; + static class PatternApiResponse extends BaseApiResponse { + public final GeoJsonLineString geometry; public final List orderedStopIds; public final List associatedTripIds; - PatternAPIResponse(Pattern pattern) { + PatternApiResponse(Pattern pattern) { super(pattern.pattern_id, pattern.name); geometry = serialize(pattern.geometry); orderedStopIds = pattern.orderedStops; associatedTripIds = pattern.associatedTrips; } - static GeoJSONLineString serialize (com.vividsolutions.jts.geom.LineString geometry) { - GeoJSONLineString ret = new GeoJSONLineString(); + static GeoJsonLineString serialize (com.vividsolutions.jts.geom.LineString geometry) { + GeoJsonLineString ret = new GeoJsonLineString(); ret.coordinates = Stream.of(geometry.getCoordinates()) .map(c -> new double[] { c.x, c.y }) .toArray(double[][]::new); @@ -128,7 +128,7 @@ static GeoJSONLineString serialize (com.vividsolutions.jts.geom.LineString geome } } - private List getPatternsForRoute (Request req, Response res) { + private List getPatternsForRoute (Request req, Response res) { addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); final String routeId = req.params("routeId"); @@ -136,38 +136,38 @@ private List getPatternsForRoute (Request req, Response res) .values() .stream() .filter(p -> Objects.equals(p.route_id, routeId)) - .map(PatternAPIResponse::new) + .map(PatternApiResponse::new) .collect(Collectors.toList()); } - static class StopAPIResponse extends BaseAPIResponse { + static class StopApiResponse extends BaseApiResponse { public final double lat; public final double lon; - StopAPIResponse(Stop stop) { + StopApiResponse(Stop stop) { super(stop.stop_id, stop.stop_name); lat = stop.stop_lat; lon = stop.stop_lon; } } - private List getAllStopsForOneFeed(Request req, Response res) { + private List getAllStopsForOneFeed(Request req, Response res) { addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); - return feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); + return feed.stops.values().stream().map(StopApiResponse::new).collect(Collectors.toList()); } - static class FeedGroupStopsAPIResponse { + static class FeedGroupStopsApiResponse { public final String feedId; - public final List stops; + public final List stops; - FeedGroupStopsAPIResponse(GTFSFeed feed) { + FeedGroupStopsApiResponse(GTFSFeed feed) { this.feedId = feed.feedId; - this.stops = feed.stops.values().stream().map(StopAPIResponse::new).collect(Collectors.toList()); + this.stops = feed.stops.values().stream().map(StopApiResponse::new).collect(Collectors.toList()); } } - private List getAllStopsForFeedGroup(Request req, Response res) { + private List getAllStopsForFeedGroup(Request req, Response res) { addImmutableResponseHeader(res); String feedGroupId = req.params("feedGroupId"); DBCursor cursor = Persistence.bundles.find(QueryBuilder.start("feedGroupId").is(feedGroupId).get()); @@ -175,23 +175,23 @@ private List getAllStopsForFeedGroup(Request req, Res throw AnalysisServerException.notFound("Bundle could not be found for the given feed group ID."); } - List allStopsByFeed = new ArrayList<>(); + List allStopsByFeed = new ArrayList<>(); Bundle bundle = cursor.next(); for (Bundle.FeedSummary feedSummary : bundle.feeds) { String bundleScopedFeedId = Bundle.bundleScopeFeedId(feedSummary.feedId, feedGroupId); GTFSFeed feed = gtfsCache.get(bundleScopedFeedId); - allStopsByFeed.add(new FeedGroupStopsAPIResponse(feed)); + allStopsByFeed.add(new FeedGroupStopsApiResponse(feed)); } return allStopsByFeed; } - static class TripAPIResponse extends BaseAPIResponse { + static class TripApiResponse extends BaseApiResponse { public final String headsign; public final Integer startTime; public final Integer duration; public final int directionId; - TripAPIResponse(GTFSFeed feed, Trip trip) { + TripApiResponse(GTFSFeed feed, Trip trip) { super(trip.trip_id, trip.trip_short_name); headsign = trip.trip_headsign; directionId = trip.direction_id; @@ -209,14 +209,14 @@ static class TripAPIResponse extends BaseAPIResponse { } } - private List getTripsForRoute (Request req, Response res) { + private List getTripsForRoute (Request req, Response res) { addImmutableResponseHeader(res); final GTFSFeed feed = getFeedFromRequest(req); final String routeId = req.params("routeId"); return feed.trips .values().stream() .filter(t -> Objects.equals(t.route_id, routeId)) - .map(t -> new TripAPIResponse(feed, t)) + .map(t -> new TripApiResponse(feed, t)) .sorted(Comparator.comparingInt(t -> t.startTime)) .collect(Collectors.toList()); } From 22210b9b0035c8a10df63446c078d7a81fa87c2a Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Mon, 6 Sep 2021 22:18:53 +0800 Subject: [PATCH 113/187] Set Cypress to a working UI branch --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index 31072191a..940822a0c 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: a869cd11919343a163e110e812b5d27f3a4ad4c8 + ref: 8218cfeb98962c25841daf7db0791443a743b707 path: ui - uses: actions/checkout@v2 with: From 6521eafa1633886fcf4d8fc0d695836c330bbdd9 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 6 Sep 2021 23:20:36 +0800 Subject: [PATCH 114/187] clarify logging when user is unknown --- .../conveyal/analysis/AnalysisServerException.java | 2 ++ .../com/conveyal/analysis/components/HttpApi.java | 1 + .../analysis/components/eventbus/ErrorEvent.java | 12 ++++++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/AnalysisServerException.java b/src/main/java/com/conveyal/analysis/AnalysisServerException.java index de160c667..3d2d04290 100644 --- a/src/main/java/com/conveyal/analysis/AnalysisServerException.java +++ b/src/main/java/com/conveyal/analysis/AnalysisServerException.java @@ -60,6 +60,8 @@ public static AnalysisServerException notFound(String message) { return new AnalysisServerException(Type.NOT_FOUND, message, 404); } + // Note that there is a naming mistake in the HTTP codes. 401 "unauthorized" actually means "unauthenticated". + // 403 "forbidden" is what is usually referred to as "unauthorized" in other contexts. public static AnalysisServerException unauthorized(String message) { return new AnalysisServerException(Type.UNAUTHORIZED, message, 401); } diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 72659cb78..c760d7f20 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -23,6 +23,7 @@ import static com.conveyal.analysis.AnalysisServerException.Type.BAD_REQUEST; import static com.conveyal.analysis.AnalysisServerException.Type.RUNTIME; +import static com.conveyal.analysis.AnalysisServerException.Type.UNAUTHORIZED; import static com.conveyal.analysis.AnalysisServerException.Type.UNKNOWN; /** diff --git a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java index 1a695d133..600adfac9 100644 --- a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java +++ b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java @@ -38,10 +38,14 @@ public ErrorEvent (Throwable throwable) { /** Return a string intended for logging on Slack or the console. */ public String traceWithContext (boolean verbose) { StringBuilder builder = new StringBuilder(); - builder.append("User "); - builder.append(user); - builder.append(" of group "); - builder.append(accessGroup); + if (user == null && accessGroup == null) { + builder.append("Unknown/unauthenticated user"); + } else { + builder.append("User "); + builder.append(user); + builder.append(" of group "); + builder.append(accessGroup); + } if (httpPath != null) { builder.append(" accessing "); builder.append(httpPath); From 64f110ef3d7438786b3e6587d8a29dc977af48a1 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 6 Sep 2021 23:40:43 +0800 Subject: [PATCH 115/187] invert filtering to avoid repeated conveyal frames --- .../com/conveyal/analysis/components/eventbus/ErrorEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java index 600adfac9..fcf87243f 100644 --- a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java +++ b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java @@ -70,7 +70,7 @@ private static String filterStackTrace (String stackTrace) { String conveyalFrame = stackTrace.lines() .map(String::strip) .filter(s -> s.startsWith("at com.conveyal.")) - .filter(frame::equals) + .filter(s -> !frame.equals(s)) .findFirst().orElse(""); return String.join("\n", error, frame, conveyalFrame); } From d96c16cea3a819b7afeaee725f412878987d1721 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 7 Sep 2021 13:32:42 +0800 Subject: [PATCH 116/187] Check for the "all" scenarioId --- .../analysis/controllers/BrokerController.java | 11 ++++++----- .../controllers/RegionalAnalysisController.java | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 455c339c4..ec1131116 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; +import com.mongodb.DBObject; import com.mongodb.QueryBuilder; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -133,11 +134,11 @@ private Object singlePoint(Request request, Response response) { final String userEmail = request.attribute("email"); final long startTimeMsec = System.currentTimeMillis(); - final AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); - - final List modifications = Persistence.modifications.findPermitted( - QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(), - accessGroup); + AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); + DBObject query = "all".equals(analysisRequest.scenarioId) + ? QueryBuilder.start("projectId").is(analysisRequest.projectId).get() + : QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(); + List modifications = Persistence.modifications.findPermitted(query, accessGroup); // Transform the analysis UI/backend task format into a slightly different type for R5 workers. TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest.populateTask( diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 0c835207f..98d24920d 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -21,6 +21,7 @@ import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.cluster.RegionalTask; import com.google.common.primitives.Ints; +import com.mongodb.DBObject; import com.mongodb.QueryBuilder; import gnu.trove.list.array.TIntArrayList; import org.json.simple.JSONObject; @@ -378,10 +379,10 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro analysisRequest.percentiles = DEFAULT_REGIONAL_PERCENTILES; } - List modifications = Persistence.modifications.findPermitted( - QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(), - accessGroup - ); + DBObject query = "all".equals(analysisRequest.scenarioId) + ? QueryBuilder.start("projectId").is(analysisRequest.projectId).get() + : QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(); + List modifications = Persistence.modifications.findPermitted(query, accessGroup); // Create an internal RegionalTask and RegionalAnalysis from the AnalysisRequest sent by the client. From b61392f81ed476c030f21bbc158285218cc506d8 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 7 Sep 2021 15:09:00 +0800 Subject: [PATCH 117/187] fix: set permissions on stored (not source) file reset permissions to read/write before delete --- .../OpportunityDatasetController.java | 1 + .../java/com/conveyal/file/FileStorage.java | 2 ++ .../com/conveyal/file/LocalFileStorage.java | 26 ++++++++++++++----- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 93b25938a..cb4a35d61 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -582,6 +582,7 @@ private OpportunityDataset deleteDataset(String id, String accessGroup) { if (dataset == null) { throw AnalysisServerException.notFound("Opportunity dataset could not be found."); } else { + // Several of these files may not exist. FileStorage::delete contract states this will be handled cleanly. fileStorage.delete(dataset.getStorageKey(FileStorageFormat.GRID)); fileStorage.delete(dataset.getStorageKey(FileStorageFormat.PNG)); fileStorage.delete(dataset.getStorageKey(FileStorageFormat.TIFF)); diff --git a/src/main/java/com/conveyal/file/FileStorage.java b/src/main/java/com/conveyal/file/FileStorage.java index 41a417a44..d3dc55299 100644 --- a/src/main/java/com/conveyal/file/FileStorage.java +++ b/src/main/java/com/conveyal/file/FileStorage.java @@ -60,6 +60,8 @@ public interface FileStorage { /** * Delete the File identified by the FileStorageKey, in both the local cache and any remote mirror. + * Due to some pre-existing code, implementations must tolerate calling this method on files that don't exist + * without throwing exceptions. */ void delete(FileStorageKey fileStorageKey); diff --git a/src/main/java/com/conveyal/file/LocalFileStorage.java b/src/main/java/com/conveyal/file/LocalFileStorage.java index 1e20ffe55..04bd8e8a9 100644 --- a/src/main/java/com/conveyal/file/LocalFileStorage.java +++ b/src/main/java/com/conveyal/file/LocalFileStorage.java @@ -12,6 +12,9 @@ import java.nio.file.attribute.PosixFilePermission; import java.util.Set; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; + /** * This implementation of FileStorage stores files in a local directory hierarchy and does not mirror anything to * cloud storage. @@ -40,7 +43,7 @@ public LocalFileStorage (Config config) { * Move the File into the FileStorage by moving the passed in file to the Path represented by the FileStorageKey. */ @Override - public void moveIntoStorage(FileStorageKey key, File file) { + public void moveIntoStorage(FileStorageKey key, File sourceFile) { // Get a pointer to the local file File storedFile = getFile(key); // Ensure the directories exist @@ -48,18 +51,18 @@ public void moveIntoStorage(FileStorageKey key, File file) { try { try { // Move the temporary file to the permanent file location. - Files.move(file.toPath(), storedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move(sourceFile.toPath(), storedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (FileSystemException e) { // The default Windows filesystem (NTFS) does not unlock memory-mapped files, so certain files (e.g. // mapdb) cannot be moved or deleted. This workaround may cause temporary files to accumulate, but it // should not be triggered for default Linux filesystems (ext). // See https://github.com/jankotek/MapDB/issues/326 - Files.copy(file.toPath(), storedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(sourceFile.toPath(), storedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); LOG.info("Could not move {} because of FileSystem restrictions (probably NTFS). Copying instead.", - file.getName()); + sourceFile.getName()); } // Set the file to be read-only and accessible only by the current user. - Files.setPosixFilePermissions(file.toPath(), Set.of(PosixFilePermission.OWNER_READ)); + Files.setPosixFilePermissions(storedFile.toPath(), Set.of(OWNER_READ)); } catch (IOException e) { throw new RuntimeException(e); } @@ -87,7 +90,18 @@ public String getURL (FileStorageKey key) { @Override public void delete (FileStorageKey key) { - getFile(key).delete(); + try { + File storedFile = getFile(key); + if (storedFile.exists()) { + // File permissions are set read-only to prevent corruption, so must be changed to allow deletion. + Files.setPosixFilePermissions(storedFile.toPath(), Set.of(OWNER_READ, OWNER_WRITE)); + storedFile.delete(); + } else { + LOG.warn("Attempted to delete non-existing file: " + storedFile); + } + } catch (Exception e) { + throw new RuntimeException("Exception while deleting stored file.", e); + } } @Override From f4519cdeb8c048ae6895d80d3de002bd7eec690d Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 7 Sep 2021 15:12:11 +0800 Subject: [PATCH 118/187] cypress tests against newer UI branch --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index 31072191a..827763cd5 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: a869cd11919343a163e110e812b5d27f3a4ad4c8 + ref: ed2c18444864e537daeda0d82f208d6a295625b0 path: ui - uses: actions/checkout@v2 with: From 31069336936ccd8708aa248ad5d6623341a2ea61 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 7 Sep 2021 15:38:47 +0800 Subject: [PATCH 119/187] delete datasource files, not just database records --- .../controllers/DataSourceController.java | 26 ++++++++++++++----- .../controllers/FileStorageController.java | 8 +++--- .../datasource/DataSourceUploadAction.java | 1 - .../conveyal/analysis/models/DataSource.java | 6 +++++ .../persistence/AnalysisCollection.java | 5 ++-- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 50679bd60..820a1ffeb 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -11,23 +11,26 @@ import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.HttpUtils; +import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.Task; +import com.mongodb.client.result.DeleteResult; import org.apache.commons.fileupload.FileItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; -import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.util.List; import java.util.Map; import static com.conveyal.analysis.util.JsonUtil.toJson; +import static com.conveyal.file.FileCategory.DATASOURCES; +import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; -import static com.conveyal.r5.analyst.progress.WorkProductType.DATA_SOURCE; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -74,15 +77,24 @@ private List getAllDataSourcesForRegion (Request req, Response res) /** HTTP GET: Retrieve a single DataSource record by the ID supplied in the URL path parameter. */ private DataSource getOneDataSourceById (Request req, Response res) { - return dataSourceCollection.findPermittedByRequestParamId(req, res); + return dataSourceCollection.findPermittedByRequestParamId(req); } /** HTTP DELETE: Delete a single DataSource record and associated files in FileStorage by supplied ID parameter. */ private String deleteOneDataSourceById (Request request, Response response) { - long nDeleted = dataSourceCollection.deleteByIdParamIfPermitted(request).getDeletedCount(); - // TODO normalize to canonical file extensions so we can find them to delete them. - // fileStorage.delete(new FileStorageKey(DATA_SOURCE, _id + extension)); - return "DELETE " + nDeleted; + DataSource dataSource = dataSourceCollection.findPermittedByRequestParamId(request); + DeleteResult deleteResult = dataSourceCollection.delete(dataSource); + long nDeleted = deleteResult.getDeletedCount(); + // This will not delete the file if its extension when uploaded did not match the canonical one. + // Ideally we should normalize file extensions when uploaded, but it's a little tricky to handle SHP sidecars. + fileStorage.delete(dataSource.fileStorageKey()); + // This is so ad-hoc but it's not necessarily worth generalizing since SHP is the only format with sidecars. + if (dataSource.fileFormat == SHP) { + fileStorage.delete(new FileStorageKey(DATASOURCES, dataSource._id.toString(), "shx")); + fileStorage.delete(new FileStorageKey(DATASOURCES, dataSource._id.toString(), "dbf")); + fileStorage.delete(new FileStorageKey(DATASOURCES, dataSource._id.toString(), "prj")); + } + return "Deleted " + nDeleted; } private SpatialDataSource downloadLODES(Request req, Response res) { diff --git a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java index 905d12e5c..220f8be18 100644 --- a/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java +++ b/src/main/java/com/conveyal/analysis/controllers/FileStorageController.java @@ -82,7 +82,7 @@ private FileInfo createFileInfo(Request req, Response res) throws IOException { * Remove the FileInfo record from the database and the file from the FileStorage. */ private boolean deleteFile(Request req, Response res) { - FileInfo file = fileCollection.findPermittedByRequestParamId(req, res); + FileInfo file = fileCollection.findPermittedByRequestParamId(req); fileStorage.delete(file.getKey()); return fileCollection.delete(file).wasAcknowledged(); } @@ -92,7 +92,7 @@ private boolean deleteFile(Request req, Response res) { * file. */ private String generateDownloadURL(Request req, Response res) { - FileInfo file = fileCollection.findPermittedByRequestParamId(req, res); + FileInfo file = fileCollection.findPermittedByRequestParamId(req); res.type("text/plain"); return fileStorage.getURL(file.getKey()); } @@ -101,7 +101,7 @@ private String generateDownloadURL(Request req, Response res) { * Find FileInfo by passing in and _id and download the corresponding file by returning an InputStream. */ private InputStream downloadFile(Request req, Response res) throws IOException { - FileInfo fileInfo = fileCollection.findPermittedByRequestParamId(req, res); + FileInfo fileInfo = fileCollection.findPermittedByRequestParamId(req); File file = fileStorage.getFile(fileInfo.getKey()); res.type(fileInfo.format.mimeType); if (FileUtils.isGzip(file)) { @@ -115,7 +115,7 @@ private InputStream downloadFile(Request req, Response res) throws IOException { * file. */ private FileInfo uploadFile(Request req, Response res) throws Exception { - FileInfo fileInfo = fileCollection.findPermittedByRequestParamId(req, res); + FileInfo fileInfo = fileCollection.findPermittedByRequestParamId(req); File file = FileUtils.createScratchFile(req.raw().getInputStream()); fileStorage.moveIntoStorage(fileInfo.getKey(), file); diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 308bc0de9..193382308 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -97,7 +97,6 @@ private final void moveFilesIntoStorage (ProgressListener progressListener) { for (FileItem fileItem : fileItems) { DiskFileItem dfi = (DiskFileItem) fileItem; // TODO use canonical extensions from filetype enum - // TODO upper case? should we be using lower case? String extension = FilenameUtils.getExtension(fileItem.getName()).toLowerCase(Locale.ROOT); FileStorageKey key = new FileStorageKey(DATASOURCES, dataSourceId, extension); fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); diff --git a/src/main/java/com/conveyal/analysis/models/DataSource.java b/src/main/java/com/conveyal/analysis/models/DataSource.java index 8586a361c..c6fbbff8f 100644 --- a/src/main/java/com/conveyal/analysis/models/DataSource.java +++ b/src/main/java/com/conveyal/analysis/models/DataSource.java @@ -1,7 +1,9 @@ package com.conveyal.analysis.models; import com.conveyal.analysis.UserPermissions; +import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorageFormat; +import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.WorkProduct; import org.bson.codecs.pojo.annotations.BsonDiscriminator; @@ -70,4 +72,8 @@ public void addIssue (DataSourceValidationIssue.Level level, String message) { issues.add(new DataSourceValidationIssue(level, message)); } + public FileStorageKey fileStorageKey () { + return new FileStorageKey(FileCategory.DATASOURCES, _id.toString(), fileFormat.extension); + } + } diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java index f58cc0105..7de3f8a60 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisCollection.java @@ -148,10 +148,9 @@ public T create(Request req, Response res) throws IOException { } /** - * Controller find by id helper. - * TODO remove unused second parameter. + * Helper for HttpControllers - find a document by the _id path parameter in the request, checking permissions. */ - public T findPermittedByRequestParamId(Request req, Response res) { + public T findPermittedByRequestParamId (Request req) { UserPermissions user = UserPermissions.from(req); T value = findById(req.params("_id")); // Throw if or does not have permission From af190a9cc84fe2fa320914d791e00fba6af4c35e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 7 Sep 2021 15:57:03 +0800 Subject: [PATCH 120/187] store datasource files under canonical extension --- .../datasource/DataSourceUploadAction.java | 13 +++++++++---- .../com/conveyal/analysis/datasource/Lines.java | 7 ------- .../com/conveyal/analysis/datasource/Points.java | 15 --------------- 3 files changed, 9 insertions(+), 26 deletions(-) delete mode 100644 src/main/java/com/conveyal/analysis/datasource/Lines.java delete mode 100644 src/main/java/com/conveyal/analysis/datasource/Points.java diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 193382308..67907c1e3 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -90,14 +90,19 @@ public final void action (ProgressListener progressListener) throws Exception { * stage. If so, then this logic needs to change a bit. */ private final void moveFilesIntoStorage (ProgressListener progressListener) { - // Loop through uploaded files, registering the extensions and writing to storage (with filenames that - // correspond to the source id) + // Loop through uploaded files, registering the extensions and writing to storage + // (with filenames that correspond to the source id) progressListener.beginTask("Moving files into storage...", 1); final String dataSourceId = ingester.dataSource()._id.toString(); for (FileItem fileItem : fileItems) { DiskFileItem dfi = (DiskFileItem) fileItem; - // TODO use canonical extensions from filetype enum - String extension = FilenameUtils.getExtension(fileItem.getName()).toLowerCase(Locale.ROOT); + // Use canonical extension from file type - files may be uploaded with e.g. tif instead of tiff or geotiff. + String extension = ingester.dataSource().fileFormat.extension; + if (fileItems.size() > 1) { + // If we have multiple files, as with Shapefile, just keep the original extension for each file. + // This could lead to orphaned files after a deletion, we might want to implement wildcard deletion. + extension = FilenameUtils.getExtension(fileItem.getName()).toLowerCase(Locale.ROOT); + } FileStorageKey key = new FileStorageKey(DATASOURCES, dataSourceId, extension); fileStorage.moveIntoStorage(key, dfi.getStoreLocation()); if (fileItems.size() == 1 || extension.equalsIgnoreCase(SHP.extension)) { diff --git a/src/main/java/com/conveyal/analysis/datasource/Lines.java b/src/main/java/com/conveyal/analysis/datasource/Lines.java deleted file mode 100644 index 144c5eeed..000000000 --- a/src/main/java/com/conveyal/analysis/datasource/Lines.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.conveyal.analysis.datasource; - -public class Lines { - - // TODO transit alignment modification - -} diff --git a/src/main/java/com/conveyal/analysis/datasource/Points.java b/src/main/java/com/conveyal/analysis/datasource/Points.java deleted file mode 100644 index abb699af4..000000000 --- a/src/main/java/com/conveyal/analysis/datasource/Points.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.conveyal.analysis.datasource; - -import com.conveyal.analysis.models.FileInfo; - -public class Points { - - public static void toFreeform (FileInfo source) { - // TODO implement - } - - public static void toGrid (FileInfo source) { - // TODO implement - } - -} From c1b6e6dd955e7f34f3a22e44d522630e9242aa9c Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 7 Sep 2021 15:58:28 +0800 Subject: [PATCH 121/187] use newer UI dev commit in cypress tests --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index cc839c334..9a445c308 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: 12c62421dfe376c471303d85b302427cdcf2a17f + ref: 72d88067da78f774959f221094eaa0d20d2aa02c path: ui - uses: actions/checkout@v2 with: From 247549a5109680172b7f1136c0064a5d58781acf Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 7 Sep 2021 17:00:38 +0800 Subject: [PATCH 122/187] clarify missing stack frame message --- .../com/conveyal/analysis/components/eventbus/ErrorEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java index fcf87243f..24dc542f1 100644 --- a/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java +++ b/src/main/java/com/conveyal/analysis/components/eventbus/ErrorEvent.java @@ -61,7 +61,7 @@ public String traceWithContext (boolean verbose) { private static String filterStackTrace (String stackTrace) { if (stackTrace == null) return null; - final String unknownFrame = "at unknown frame"; + final String unknownFrame = "Unknown stack frame, probably optimized out by JVM."; String error = stackTrace.lines().findFirst().get(); String frame = stackTrace.lines() .map(String::strip) From 326a58d0d2c4f577df6926ddeebd44c6b87efcc2 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 7 Sep 2021 22:02:56 +0800 Subject: [PATCH 123/187] Add back in the similar modification creation to the request object --- .../controllers/BrokerController.java | 14 +----- .../RegionalAnalysisController.java | 15 +----- .../analysis/models/AnalysisRequest.java | 47 +++++++++---------- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index ec1131116..9be6f12f7 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -10,7 +10,6 @@ import com.conveyal.analysis.components.eventbus.SinglePointEvent; import com.conveyal.analysis.models.AnalysisRequest; import com.conveyal.analysis.models.Bundle; -import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.HttpStatus; @@ -26,7 +25,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; -import com.mongodb.DBObject; import com.mongodb.QueryBuilder; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -49,7 +47,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static com.conveyal.r5.common.Util.notNullOrEmpty; import static com.google.common.base.Preconditions.checkNotNull; @@ -135,16 +132,9 @@ private Object singlePoint(Request request, Response response) { final long startTimeMsec = System.currentTimeMillis(); AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); - DBObject query = "all".equals(analysisRequest.scenarioId) - ? QueryBuilder.start("projectId").is(analysisRequest.projectId).get() - : QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(); - List modifications = Persistence.modifications.findPermitted(query, accessGroup); - // Transform the analysis UI/backend task format into a slightly different type for R5 workers. - TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest.populateTask( - new TravelTimeSurfaceTask(), - modifications.stream().map(Modification::toR5).collect(Collectors.toList()) - ); + TravelTimeSurfaceTask task = new TravelTimeSurfaceTask(); + analysisRequest.populateTask(task, accessGroup); // If destination opportunities are supplied, prepare to calculate accessibility worker-side if (notNullOrEmpty(analysisRequest.destinationPointSetIds)){ diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 98d24920d..23d0e775f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -5,7 +5,6 @@ import com.conveyal.analysis.components.broker.Broker; import com.conveyal.analysis.components.broker.JobStatus; import com.conveyal.analysis.models.AnalysisRequest; -import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.RegionalAnalysis; import com.conveyal.analysis.persistence.Persistence; @@ -21,7 +20,6 @@ import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.cluster.RegionalTask; import com.google.common.primitives.Ints; -import com.mongodb.DBObject; import com.mongodb.QueryBuilder; import gnu.trove.list.array.TIntArrayList; import org.json.simple.JSONObject; @@ -41,7 +39,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.util.JsonUtil.toJson; @@ -379,19 +376,11 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro analysisRequest.percentiles = DEFAULT_REGIONAL_PERCENTILES; } - DBObject query = "all".equals(analysisRequest.scenarioId) - ? QueryBuilder.start("projectId").is(analysisRequest.projectId).get() - : QueryBuilder.start("_id").in(analysisRequest.modificationIds).get(); - List modifications = Persistence.modifications.findPermitted(query, accessGroup); - // Create an internal RegionalTask and RegionalAnalysis from the AnalysisRequest sent by the client. - // TODO now this is setting cutoffs and percentiles in the regional (template) task. // why is some stuff set in this populate method, and other things set here in the caller? - RegionalTask task = (RegionalTask) analysisRequest.populateTask( - new RegionalTask(), - modifications.stream().map(Modification::toR5).collect(Collectors.toList()) - ); + RegionalTask task = new RegionalTask(); + analysisRequest.populateTask(task, accessGroup); // Set the destination PointSets, which are required for all non-Taui regional requests. if (!analysisRequest.makeTauiSite) { diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 52e845b8f..0d80a2f20 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -1,16 +1,18 @@ package com.conveyal.analysis.models; import com.conveyal.analysis.AnalysisServerException; +import com.conveyal.analysis.persistence.Persistence; import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; import com.conveyal.r5.analyst.decay.DecayFunction; import com.conveyal.r5.analyst.decay.StepDecayFunction; import com.conveyal.r5.analyst.fare.InRoutingFareCalculator; -import com.conveyal.r5.analyst.scenario.Modification; import com.conveyal.r5.analyst.scenario.Scenario; import com.conveyal.r5.api.util.LegMode; import com.conveyal.r5.api.util.TransitModes; -import com.conveyal.r5.common.JsonUtilities; +import com.mongodb.DBObject; +import com.mongodb.QueryBuilder; +import org.apache.commons.codec.digest.DigestUtils; import java.time.LocalDate; import java.util.ArrayList; @@ -18,7 +20,6 @@ import java.util.EnumSet; import java.util.List; import java.util.stream.Collectors; -import java.util.zip.CRC32; /** * Request sent from the UI to the backend. It is actually distinct from the task that the broker @@ -149,6 +150,23 @@ public class AnalysisRequest { */ public DecayFunction decayFunction; + /** + * Create the R5 `Scenario` from this request. + */ + public Scenario createScenario (String accessGroup) { + DBObject query = "all".equals(scenarioId) + ? QueryBuilder.start("projectId").is(projectId).get() + : QueryBuilder.start("_id").in(modificationIds).get(); + List modifications = Persistence.modifications.findPermitted(query, accessGroup); + // `findPermitted` sorts by creation time by default. Nonces will be in the same order each time. + String nonces = Arrays.toString(modifications.stream().map(m -> m.nonce).toArray()); + String scenarioId = String.format("%s-%s", bundleId, DigestUtils.sha1Hex(nonces)); + Scenario scenario = new Scenario(); + scenario.id = scenarioId; + scenario.modifications = modifications.stream().map(com.conveyal.analysis.models.Modification::toR5).collect(Collectors.toList()); + return scenario; + } + /** * Finds the modifications for the specified project and variant, maps them to their * corresponding R5 modification types, creates a checksum from those modifications, and adds @@ -162,27 +180,10 @@ public class AnalysisRequest { * TODO arguably this should be done by a method on the task classes themselves, with common parts factored out * to the same method on the superclass. */ - public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, List modifications) { + public void populateTask (AnalysisWorkerTask task, String accessGroup) { if (bounds == null) throw AnalysisServerException.badRequest("Analysis bounds must be set."); - // The CRC of the modifications in this scenario is appended to the bundle ID to identify a unique set of - // modifications allowing the worker to cache and reuse networks built by applying that exact revision of the - // scenario to a base network. - CRC32 crc = new CRC32(); - crc.update(JsonUtilities.objectToJsonBytes(modifications)); - long crcValue = crc.getValue(); - // FIXME Job IDs need to be unique. Why are we setting this to the project and variant? - // This only works because the job ID is overwritten when the job is enqueued. - // Its main effect is to cause the scenario ID to have this same pattern! - // We should probably leave the JobID null on single point tasks. Needed: polymorphic task initialization. - task.jobId = String.format("%s-%s", bundleId, crcValue); - - Scenario scenario = new Scenario(); - scenario.id = task.scenarioId = task.jobId; - scenario.modifications = modifications; - // task.scenario.description = scenarioName; - - task.scenario = scenario; + task.scenario = createScenario(accessGroup); task.graphId = bundleId; task.workerVersion = workerVersion; task.maxFare = maxFare; @@ -245,8 +246,6 @@ public AnalysisWorkerTask populateTask (AnalysisWorkerTask task, List Date: Wed, 8 Sep 2021 16:57:55 +0800 Subject: [PATCH 124/187] Apply suggestions from code review Co-authored-by: Anson Stewart --- .../controllers/AggregationAreaController.java | 2 -- .../analysis/controllers/DataSourceController.java | 3 --- .../analysis/controllers/HttpController.java | 3 --- .../datasource/GeoJsonDataSourceIngester.java | 3 --- .../analysis/datasource/SpatialAttribute.java | 2 +- .../conveyal/analysis/models/SpatialDataSource.java | 12 ------------ .../conveyal/analysis/persistence/AnalysisDB.java | 2 -- 7 files changed, 1 insertion(+), 26 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 4413fcc9e..cf9aa6603 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -22,8 +22,6 @@ import java.util.Collection; import static com.conveyal.analysis.util.JsonUtil.toJson; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 820a1ffeb..171952fc3 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -11,9 +11,7 @@ import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.util.HttpUtils; -import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorage; -import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; import com.conveyal.r5.analyst.progress.Task; import com.mongodb.client.result.DeleteResult; @@ -31,7 +29,6 @@ import static com.conveyal.file.FileCategory.DATASOURCES; import static com.conveyal.file.FileStorageFormat.SHP; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; -import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; /** diff --git a/src/main/java/com/conveyal/analysis/controllers/HttpController.java b/src/main/java/com/conveyal/analysis/controllers/HttpController.java index a6a7e1c47..41843990e 100644 --- a/src/main/java/com/conveyal/analysis/controllers/HttpController.java +++ b/src/main/java/com/conveyal/analysis/controllers/HttpController.java @@ -1,8 +1,5 @@ package com.conveyal.analysis.controllers; -import com.conveyal.analysis.UserPermissions; -import spark.Request; - /** * All of our classes defining HTTP API endpoints implement this interface. * It has a single method that registers all the endpoints. diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 540796a29..9b6f648ab 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -5,14 +5,11 @@ import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.file.FileStorageFormat; import com.conveyal.r5.analyst.progress.ProgressListener; -import com.conveyal.r5.util.ShapefileReader; import com.conveyal.r5.util.ShapefileReader.GeometryType; import org.geotools.data.Query; -import org.geotools.data.geojson.GeoJSONDataStore; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; -import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultGeographicCRS; import org.locationtech.jts.geom.Envelope; diff --git a/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java index 6d20302c1..b33aa2348 100644 --- a/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java +++ b/src/main/java/com/conveyal/analysis/datasource/SpatialAttribute.java @@ -6,7 +6,7 @@ import org.opengis.feature.type.AttributeType; /** - * In OpenGIS terminology, SpatialResources contain features, each of which has attributes. This class represents a + * In OpenGIS terminology, SpatialDataSources contain features, each of which has attributes. This class represents a * single attribute present on all the features in a resource - it's basically the schema metadata for a GIS layer. * Users can specify their own name for any attribute in the source file, so this also associates these user-specified * names with the original attribute name. diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java index 0dd4d97aa..ce94f0522 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java @@ -1,26 +1,14 @@ package com.conveyal.analysis.models; -import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.datasource.SpatialAttribute; -import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; import com.conveyal.r5.util.ShapefileReader; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.io.FilenameUtils; import org.bson.codecs.pojo.annotations.BsonDiscriminator; -import org.locationtech.jts.geom.Envelope; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.operation.TransformException; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static com.conveyal.file.FileCategory.DATASOURCES; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; /** * A SpatialDataSource is metadata about a user-uploaded file containing geospatial features (e.g. shapefile, GeoJSON, diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 956f6ba53..571038788 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -1,8 +1,6 @@ package com.conveyal.analysis.persistence; import com.conveyal.analysis.models.BaseModel; -import com.conveyal.analysis.models.SpatialDataSource; -import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; From 014db0689035d259d2443be86f8ee731ed87a75e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 8 Sep 2021 17:49:06 +0800 Subject: [PATCH 125/187] rename and refactor SpatialLayers utility class throw DataSourceExceptions compare against lower case extns to match canonical form on filesystem enforce single uploaded file for all formats but shapefile move toward checks performed in a loop over FileStorageFormat add Javadoc to methods --- .../OpportunityDatasetController.java | 2 +- .../datasource/DataSourceUploadAction.java | 2 +- .../analysis/datasource/DataSourceUtil.java | 107 +++++++++++++++ .../datasource/GeoJsonDataSourceIngester.java | 1 + .../analysis/datasource/SpatialLayers.java | 125 ------------------ 5 files changed, 110 insertions(+), 127 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java delete mode 100644 src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index fb382c79a..17882917f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -55,7 +55,7 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; +import static com.conveyal.analysis.datasource.DataSourceUtil.detectUploadFormatAndValidate; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java index 67907c1e3..39f6740ab 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUploadAction.java @@ -21,7 +21,7 @@ import java.util.stream.Collectors; import static com.conveyal.analysis.util.HttpUtils.getFormField; -import static com.conveyal.analysis.datasource.SpatialLayers.detectUploadFormatAndValidate; +import static com.conveyal.analysis.datasource.DataSourceUtil.detectUploadFormatAndValidate; import static com.conveyal.file.FileCategory.DATASOURCES; import static com.conveyal.file.FileStorageFormat.SHP; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java new file mode 100644 index 000000000..983cb55e1 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java @@ -0,0 +1,107 @@ +package com.conveyal.analysis.datasource; + +import com.conveyal.analysis.AnalysisServerException; + +import com.conveyal.file.FileStorageFormat; +import com.google.common.collect.Sets; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.io.FilenameUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * Utility class with common static methods for validating and processing uploaded spatial data files. + */ +public abstract class DataSourceUtil { + + /** + * Detect the format of a batch of user-uploaded files. Once the intended file type has been established, we + * validate the list of uploaded files, making sure certain preconditions are met. Some kinds of uploads must + * contain multiple files (.shp) while most others must contain only a single file (.csv, .gpkg etc.). + * Note that this does not perform structural or semantic validation of file contents, just the high-level + * characteristics of the set of file names. + * @throws DataSourceException if the type of the upload can't be detected or preconditions are violated. + * @return the expected type of the uploaded file or files, never null. + */ + public static FileStorageFormat detectUploadFormatAndValidate (List fileItems) { + if (fileItems == null || fileItems.isEmpty()) { + throw new DataSourceException("You must include some files to create an opportunity dataset."); + } + Set fileExtensions = extractFileExtensions(fileItems); + if (fileExtensions.contains("zip")) { + throw new DataSourceException("Upload of spatial .zip files not yet supported"); + // TODO unzip and process unzipped files - will need to peek inside to detect GTFS uploads first. + // detectUploadFormatAndValidate(unzipped) + } + // There was at least one file with an extension, the set must now contain at least one extension. + if (fileExtensions.isEmpty()) { + throw new DataSourceException("No file extensions seen, cannot detect upload type."); + } + // Check that if upload contains any of the Shapefile sidecar files, it contains all of the required ones. + final Set shapefileExtensions = Sets.newHashSet("shp", "dbf", "prj"); + if ( ! Sets.intersection(fileExtensions, shapefileExtensions).isEmpty()) { + if (fileExtensions.containsAll(shapefileExtensions)) { + verifyBaseNamesSame(fileItems); + // TODO check that any additional file is .shx, and that there are no more than 4 files. + } else { + throw new DataSourceException("You must multi-select at least SHP, DBF, and PRJ files for shapefile upload."); + } + return FileStorageFormat.SHP; + } + // The upload was not a Shapefile. All other formats should contain one single file. + if (fileExtensions.size() != 1) { + throw new DataSourceException("For any format but Shapefile, upload only one file at a time."); + } + final String extension = fileExtensions.stream().findFirst().get(); + // TODO replace with iteration over FileStorageFormat.values() and their lists of extensions + if (extension.equals("grid")) { + return FileStorageFormat.GRID; + } else if (extension.equals("csv")) { + return FileStorageFormat.CSV; + } else if (extension.equals("geojson") || extension.equals("json")) { + return FileStorageFormat.GEOJSON; + } else if (extension.equals("gpkg")) { + return FileStorageFormat.GEOPACKAGE; + } else if (extension.equals("tif") || extension.equals("tiff") || extension.equals("geotiff")) { + return FileStorageFormat.GEOTIFF; + } + throw new DataSourceException("Could not detect format of uploaded spatial data."); + } + + /** Given a list of FileItems, return a set of all unique file extensions present, normalized to lower case. */ + private static Set extractFileExtensions (List fileItems) { + Set fileExtensions = new HashSet<>(); + for (FileItem fileItem : fileItems) { + String fileName = fileItem.getName(); + String extension = FilenameUtils.getExtension(fileName); + if (extension.isEmpty()) { + new DataSourceException("Filename has no extension: " + fileName); + } + fileExtensions.add(extension.toLowerCase(Locale.ROOT)); + } + return fileExtensions; + } + + /** In uploads containing more than one file, all files are expected to have the same name before the extension. */ + private static void verifyBaseNamesSame (List fileItems) { + String firstBaseName = null; + for (FileItem fileItem : fileItems) { + String fileName = fileItem.getName(); + // Special case for .shp.xml files, which will otherwise fail this check + if ("xml".equalsIgnoreCase(FilenameUtils.getExtension(fileName))) { + fileName = FilenameUtils.getBaseName(fileName); + } + String baseName = FilenameUtils.getBaseName(fileName); + if (firstBaseName == null) { + firstBaseName = baseName; + } + if (!firstBaseName.equals(baseName)) { + throw new DataSourceException("In a shapefile upload, all files must have the same base name."); + } + } + } + +} diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 9b6f648ab..5e6e49b0f 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -7,6 +7,7 @@ import com.conveyal.r5.analyst.progress.ProgressListener; import com.conveyal.r5.util.ShapefileReader.GeometryType; import org.geotools.data.Query; +import org.geotools.data.geojson.GeoJSONDataStore; import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; diff --git a/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java b/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java deleted file mode 100644 index 33a733b0d..000000000 --- a/src/main/java/com/conveyal/analysis/datasource/SpatialLayers.java +++ /dev/null @@ -1,125 +0,0 @@ -package com.conveyal.analysis.datasource; - -import com.conveyal.analysis.AnalysisServerException; - -import com.conveyal.file.FileStorageFormat; -import com.google.common.collect.Sets; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.io.FilenameUtils; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Utility class with common static methods for validating and processing uploaded spatial data files. - */ -public abstract class SpatialLayers { - - /** - * FIXME Originally used for OpportunityDataset upload, moved to SpatialLayers but should be named DataSource - * Detect from a batch of uploaded files whether the user has uploaded a Shapefile, a CSV, or one or more binary - * grids. In the process we validate the list of uploaded files, making sure certain preconditions are met. - * Some kinds of uploads must contain multiple files (.shp) or can contain multiple files (.grid) while others - * must have only a single file (.csv). Scan the list of uploaded files to ensure it makes sense before acting. - * Note that this does not validate the contents of the files semantically, just the high-level characteristics of - * the set of files. - * @throws AnalysisServerException if the type of the upload can't be detected or preconditions are violated. - * @return the expected type of the uploaded file or files, never null. - */ - public static FileStorageFormat detectUploadFormatAndValidate (List fileItems) { - if (fileItems == null || fileItems.isEmpty()) { - throw AnalysisServerException.fileUpload("You must include some files to create an opportunity dataset."); - } - - Set fileExtensions = extractFileExtensions(fileItems); - - if (fileExtensions.contains("ZIP")) { - throw AnalysisServerException.fileUpload("Upload of spatial .zip files not yet supported"); - // TODO unzip - // detectUploadFormatAndValidate(unzipped) - } - - // There was at least one file with an extension, the set must now contain at least one extension. - if (fileExtensions.isEmpty()) { - throw AnalysisServerException.fileUpload("No file extensions seen, cannot detect upload type."); - } - - FileStorageFormat uploadFormat = null; - - // Check that if upload contains any of the Shapefile sidecar files, it contains all of the required ones. - final Set shapefileExtensions = Sets.newHashSet("SHP", "DBF", "PRJ"); - if ( ! Sets.intersection(fileExtensions, shapefileExtensions).isEmpty()) { - if (fileExtensions.containsAll(shapefileExtensions)) { - uploadFormat = FileStorageFormat.SHP; - verifyBaseNamesSame(fileItems); - // TODO check that any additional file is SHX, and that there are no more than 4 files. - } else { - final String message = "You must multi-select at least SHP, DBF, and PRJ files for shapefile upload."; - throw AnalysisServerException.fileUpload(message); - } - } - - // Even if we've already detected a shapefile, run the other tests to check for a bad mixture of file types. - // TODO factor the size == 1 check out of all cases - if (fileExtensions.contains("GRID")) { - if (fileExtensions.size() == 1) { - uploadFormat = FileStorageFormat.GRID; - } else { - String message = "When uploading grids you may upload multiple files, but they must all be grids."; - throw AnalysisServerException.fileUpload(message); - } - } else if (fileExtensions.contains("CSV")) { - if (fileItems.size() == 1) { - uploadFormat = FileStorageFormat.CSV; - } else { - String message = "When uploading CSV you may only upload one file at a time."; - throw AnalysisServerException.fileUpload(message); - } - } else if (fileExtensions.contains("GEOJSON") || fileExtensions.contains("JSON")) { - uploadFormat = FileStorageFormat.GEOJSON; - } else if (fileExtensions.contains("GPKG")) { - uploadFormat = FileStorageFormat.GEOPACKAGE; - } else if (fileExtensions.contains("TIFF") || fileExtensions.contains("TIF")) { - uploadFormat = FileStorageFormat.GEOTIFF; - } - - if (uploadFormat == null) { - throw AnalysisServerException.fileUpload("Could not detect format of uploaded spatial data."); - } - return uploadFormat; - } - - private static Set extractFileExtensions (List fileItems) { - - Set fileExtensions = new HashSet<>(); - - for (FileItem fileItem : fileItems) { - String fileName = fileItem.getName(); - String extension = FilenameUtils.getExtension(fileName); - if (extension.isEmpty()) { - throw AnalysisServerException.fileUpload("Filename has no extension: " + fileName); - } - fileExtensions.add(extension.toUpperCase()); - } - - return fileExtensions; - } - - private static void verifyBaseNamesSame (List fileItems) { - String firstBaseName = null; - for (FileItem fileItem : fileItems) { - // Ignore .shp.xml files, which will fail the verifyBaseNamesSame check - if ("xml".equalsIgnoreCase(FilenameUtils.getExtension(fileItem.getName()))) continue; - String baseName = FilenameUtils.getBaseName(fileItem.getName()); - if (firstBaseName == null) { - firstBaseName = baseName; - } - if (!firstBaseName.equals(baseName)) { - String message = "In a shapefile upload, all files must have the same base name."; - throw AnalysisServerException.fileUpload(message); - } - } - } - -} From ed207ea982c6889e1469ab15292a190acfe51d27 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 8 Sep 2021 18:08:47 +0800 Subject: [PATCH 126/187] additional checks on file uploads nonzero size, readable, etc. do not tolerate .xml, UI selection dialog should not allow .xml anyway --- .../analysis/datasource/DataSourceUtil.java | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java index 983cb55e1..514c5443a 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java @@ -5,13 +5,18 @@ import com.conveyal.file.FileStorageFormat; import com.google.common.collect.Sets; import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.io.FilenameUtils; +import java.io.File; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; +import static com.conveyal.r5.common.Util.isNullOrEmpty; +import static com.google.common.base.Preconditions.checkState; + /** * Utility class with common static methods for validating and processing uploaded spatial data files. */ @@ -27,19 +32,19 @@ public abstract class DataSourceUtil { * @return the expected type of the uploaded file or files, never null. */ public static FileStorageFormat detectUploadFormatAndValidate (List fileItems) { - if (fileItems == null || fileItems.isEmpty()) { - throw new DataSourceException("You must include some files to create an opportunity dataset."); + if (isNullOrEmpty(fileItems)) { + throw new DataSourceException("You must select some files to upload."); } Set fileExtensions = extractFileExtensions(fileItems); + if (fileExtensions.isEmpty()) { + throw new DataSourceException("No file extensions seen, cannot detect upload type."); + } + checkFileCharacteristics(fileItems); if (fileExtensions.contains("zip")) { throw new DataSourceException("Upload of spatial .zip files not yet supported"); // TODO unzip and process unzipped files - will need to peek inside to detect GTFS uploads first. // detectUploadFormatAndValidate(unzipped) } - // There was at least one file with an extension, the set must now contain at least one extension. - if (fileExtensions.isEmpty()) { - throw new DataSourceException("No file extensions seen, cannot detect upload type."); - } // Check that if upload contains any of the Shapefile sidecar files, it contains all of the required ones. final Set shapefileExtensions = Sets.newHashSet("shp", "dbf", "prj"); if ( ! Sets.intersection(fileExtensions, shapefileExtensions).isEmpty()) { @@ -71,7 +76,24 @@ public static FileStorageFormat detectUploadFormatAndValidate (List fi throw new DataSourceException("Could not detect format of uploaded spatial data."); } - /** Given a list of FileItems, return a set of all unique file extensions present, normalized to lower case. */ + /** + * Check that all FileItems supplied are stored in disk files (not memory), that they are all readable and all + * have nonzero size. + */ + private static void checkFileCharacteristics (List fileItems) { + for (FileItem fileItem : fileItems) { + checkState(fileItem instanceof DiskFileItem); + File diskFile = ((DiskFileItem)fileItem).getStoreLocation(); + checkState(diskFile.exists()); + checkState(diskFile.canRead()); + checkState(diskFile.length() > 0); + } + } + + /** + * Given a list of FileItems, return a set of all unique file extensions present, normalized to lower case. + * Always returns a set instance which may be empty, but never null. + */ private static Set extractFileExtensions (List fileItems) { Set fileExtensions = new HashSet<>(); for (FileItem fileItem : fileItems) { @@ -89,16 +111,10 @@ private static Set extractFileExtensions (List fileItems) { private static void verifyBaseNamesSame (List fileItems) { String firstBaseName = null; for (FileItem fileItem : fileItems) { - String fileName = fileItem.getName(); - // Special case for .shp.xml files, which will otherwise fail this check - if ("xml".equalsIgnoreCase(FilenameUtils.getExtension(fileName))) { - fileName = FilenameUtils.getBaseName(fileName); - } - String baseName = FilenameUtils.getBaseName(fileName); + String baseName = FilenameUtils.getBaseName(fileItem.getName()); if (firstBaseName == null) { firstBaseName = baseName; - } - if (!firstBaseName.equals(baseName)) { + } else if (!firstBaseName.equals(baseName)) { throw new DataSourceException("In a shapefile upload, all files must have the same base name."); } } From a70d7e01a0ef6f25e5aced8edbf4b2c90acda92b Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 8 Sep 2021 18:19:26 +0800 Subject: [PATCH 127/187] comments and import cleanup --- .../analysis/datasource/GeoTiffDataSourceIngester.java | 1 + src/main/java/com/conveyal/file/FileStorageFormat.java | 4 ++-- src/main/java/com/conveyal/r5/util/ShapefileReader.java | 5 ----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java index c63468ac1..c1692460e 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoTiffDataSourceIngester.java @@ -63,6 +63,7 @@ public void ingest (File file, ProgressListener progressListener) { MathTransform wgsToCoverage, coverageToWgs; ReferencedEnvelope wgsEnvelope; try { + // These two transforms are not currently used - find them just to fail fast if GeoTools does not understand. wgsToCoverage = CRS.findMathTransform(DefaultGeographicCRS.WGS84, coverageCrs); coverageToWgs = wgsToCoverage.inverse(); // Envelope in coverage CRS is not necessarily aligned with axes when transformed to WGS84. diff --git a/src/main/java/com/conveyal/file/FileStorageFormat.java b/src/main/java/com/conveyal/file/FileStorageFormat.java index 53fcd9d93..7d9cbf7c1 100644 --- a/src/main/java/com/conveyal/file/FileStorageFormat.java +++ b/src/main/java/com/conveyal/file/FileStorageFormat.java @@ -27,8 +27,8 @@ public enum FileStorageFormat { // See requirement 3 http://www.geopackage.org/spec130/#_file_extension_name GEOPACKAGE("gpkg", "application/geopackage+sqlite3"); - // These should not be serialized into Mongo. Default Enum codec uses String name() and valueOf(String). - // TODO clarify whether the extension is used for backend storage, or for detecting type up uploaded files. + // These fields will not be serialized into Mongo. + // The default codec to serialize Enums into BSON for Mongo uses String name() and valueOf(String). // TODO array of file extensions, with the first one used canonically in FileStorage and the others for detection. public final String extension; public final String mimeType; diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 164e7ba26..271866489 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -1,6 +1,5 @@ package com.conveyal.r5.util; -import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.datasource.DataSourceException; import com.conveyal.analysis.datasource.SpatialAttribute; import org.geotools.data.DataStore; @@ -41,10 +40,6 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import static com.conveyal.r5.util.ShapefileReader.GeometryType.LINE; -import static com.conveyal.r5.util.ShapefileReader.GeometryType.POINT; -import static com.conveyal.r5.util.ShapefileReader.GeometryType.POLYGON; - /** * Encapsulate Shapefile reading logic */ From fa96ba8863d6cc7e39620faf0832d8361ed7c563 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 8 Sep 2021 22:14:04 +0800 Subject: [PATCH 128/187] remove region from /api/aggregationArea paths where needed, the region is supplied as a query parameter. also appended /gridUrl to path that does not return mongo documents. made regionId optional when fetching by groups. added javadoc to methods. --- .../AggregationAreaController.java | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index cf9aa6603..7fdb9ef1a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -19,9 +19,12 @@ import spark.Response; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import static com.conveyal.analysis.util.JsonUtil.toJson; +import static com.google.common.base.Preconditions.checkNotNull; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -61,7 +64,6 @@ public AggregationAreaController ( * @return the ID of the Task representing the enqueued background action that will create the aggregation areas. */ private String createAggregationAreas (Request req, Response res) throws Exception { - // Create and enqueue an asynchronous background action to derive aggreagation areas from spatial data source. // The constructor will extract query parameters and range check them (not ideal separation, but it works). DataDerivation derivation = AggregationAreaDerivation.fromRequest(req, fileStorage, analysisDb); @@ -74,31 +76,38 @@ private String createAggregationAreas (Request req, Response res) throws Excepti return backgroundTask.id.toString(); } + /** + * Get all aggregation area documents meeting the supplied criteria. + * The request must contain a query parameter for the regionId or the dataGroupId or both. + */ private Collection getAggregationAreas (Request req, Response res) { - Bson query = eq("regionId", req.queryParams("regionId")); + List filters = new ArrayList<>(); + String regionId = req.queryParams("regionId"); + if (regionId != null) { + filters.add(eq("regionId", regionId)); + } String dataGroupId = req.queryParams("dataGroupId"); if (dataGroupId != null) { - query = and(eq("dataGroupId", dataGroupId), query); + filters.add(eq("dataGroupId", dataGroupId)); + } + if (filters.isEmpty()) { + throw new IllegalArgumentException("You must supply either a regionId or a dataGroupId or both."); } - return aggregationAreaCollection.findPermitted(query, UserPermissions.from(req)); + return aggregationAreaCollection.findPermitted(and(filters), UserPermissions.from(req)); } - private ObjectNode getAggregationArea (Request req, Response res) { - AggregationArea aggregationArea = aggregationAreaCollection.findByIdIfPermitted( - req.params("maskId"), UserPermissions.from(req) - ); + /** Returns a JSON-wrapped URL for the mask grid of the aggregation area whose id matches the path parameter. */ + private ObjectNode getAggregationAreaGridUrl (Request req, Response res) { + AggregationArea aggregationArea = aggregationAreaCollection.findPermittedByRequestParamId(req); String url = fileStorage.getURL(aggregationArea.getStorageKey()); return JsonUtil.objectNode().put("url", url); - } @Override public void registerEndpoints (spark.Service sparkService) { - sparkService.path("/api/region/", () -> { - sparkService.get("/:regionId/aggregationArea", this::getAggregationAreas, toJson); - sparkService.get("/:regionId/aggregationArea/:maskId", this::getAggregationArea, toJson); - sparkService.post("/:regionId/aggregationArea", this::createAggregationAreas, toJson); - }); + sparkService.get("/api/aggregationArea", this::getAggregationAreas, toJson); + sparkService.get("/api/aggregationArea/:_id/gridUrl", this::getAggregationAreaGridUrl, toJson); + sparkService.post("/api/aggregationArea", this::createAggregationAreas, toJson); } } From 2bbb5345da86b75084a65a6483e1bcf5fa62ff80 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 8 Sep 2021 22:35:15 +0800 Subject: [PATCH 129/187] explicit mergePolygons parameter, rage check zoom --- .../controllers/AggregationAreaController.java | 9 +++++---- .../derivation/AggregationAreaDerivation.java | 17 ++++++++++------- .../r5/analyst/WebMercatorGridPointSet.java | 14 ++++++++------ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 7fdb9ef1a..17b0f27f7 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -57,10 +57,11 @@ public AggregationAreaController ( /** * Create binary .grid files for aggregation (aka mask) areas, save them to FileStorage, and persist their metadata * to Mongo. The supplied request (req) must include query parameters specifying the dataSourceId of a - * SpatialDataSoure containing the polygonal aggregation area geometries. If the nameProperty query parameter is - * non-null, it must be the name of a text attribute in that SpatialDataSource, and one aggregation area will be - * created for each polygon using those names. If the nameProperty is not supplied, all polygons will be merged into - * one large nameless (multi)polygon aggregation area. + * SpatialDataSoure containing the polygonal aggregation area geometries. If the mergePolygons query parameter is + * supplied and is true, all polygons will be merged into one large (multi)polygon aggregation area. + * If the mergePolygons query parameter is not supplied or is false, the nameProperty query parameter must be + * the name of a text attribute in that SpatialDataSource. One aggregation area will be created for each polygon + * drawing the names from that attribute. * @return the ID of the Task representing the enqueued background action that will create the aggregation areas. */ private String createAggregationAreas (Request req, Response res) throws Exception { diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index 9d24e2d4e..6a883fd09 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -58,6 +58,7 @@ public class AggregationAreaDerivation implements DataDerivation finalFeatures; @@ -79,11 +80,13 @@ private AggregationAreaDerivation (FileStorage fileStorage, AnalysisDB database, userPermissions = UserPermissions.from(req); dataSourceId = req.queryParams("dataSourceId"); nameProperty = req.queryParams("nameProperty"); //"dist_name"; // - zoom = parseZoom(req.queryParams("zoom")); + zoom = parseZoom(req.queryParams("zoom")); + mergePolygons = Boolean.parseBoolean(req.queryParams("mergePolygons")); checkNotNull(dataSourceId); - checkNotNull(nameProperty); - // TODO range check zoom - + if (!mergePolygons) { + checkNotNull(nameProperty, "You must supply a nameProperty if mergePolygons is not true."); + } + AnalysisCollection dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); DataSource dataSource = dataSourceCollection.findById(dataSourceId); @@ -125,7 +128,7 @@ handled in a streaming fashion (in constant memory). // GeoJSON, GeoPackage etc. throw new UnsupportedOperationException("To be implemented."); } - if (nameProperty != null && finalFeatures.size() > MAX_FEATURES) { + if (!mergePolygons && finalFeatures.size() > MAX_FEATURES) { String message = MessageFormat.format( "The uploaded shapefile has {0} features, exceeding the limit of {1}", finalFeatures.size(), MAX_FEATURES @@ -145,13 +148,13 @@ public void action (ProgressListener progressListener) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); String groupDescription = "Convert polygons to aggregation areas, " + - ((nameProperty == null) ? "merging all polygons." : "one area per polygon."); + (mergePolygons ? "merging all polygons." : "one area per polygon."); DataGroup dataGroup = new DataGroup(userPermissions, spatialDataSource._id.toString(), groupDescription); progressListener.beginTask("Reading data source", finalFeatures.size() + 1); Map areaGeometries = new HashMap<>(); - if (nameProperty == null) { + if (mergePolygons) { // Union (single combined aggregation area) requested List geometries = finalFeatures.stream().map(f -> (Geometry) f.getDefaultGeometry()).collect(Collectors.toList() diff --git a/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java b/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java index c3b585812..0fd779404 100644 --- a/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java +++ b/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java @@ -28,6 +28,10 @@ public class WebMercatorGridPointSet extends PointSet implements Serializable { */ public static final int DEFAULT_ZOOM = 9; + public static final int MIN_ZOOM = 9; + + public static final int MAX_ZOOM = 12; + /** web mercator zoom level */ public final int zoom; @@ -229,12 +233,10 @@ public WebMercatorExtents getWebMercatorExtents () { return new WebMercatorExtents(west, north, width, height, zoom); } - public static int parseZoom(String zoom) { - if (zoom != null) { - return Integer.parseInt(zoom); - } else { - return DEFAULT_ZOOM; - } + public static int parseZoom(String zoomString) { + int zoom = (zoomString == null) ? DEFAULT_ZOOM : Integer.parseInt(zoomString); + checkArgument(zoom >= MIN_ZOOM && zoom <= MAX_ZOOM); + return zoom; } } From f780f527ac5bc92abc32d6f91d45ef8d1d496e8a Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Thu, 9 Sep 2021 17:09:08 +0800 Subject: [PATCH 130/187] Fix merge conflicts --- .../conveyal/analysis/components/HttpApi.java | 4 +--- .../analysis/controllers/BrokerController.java | 9 ++------- .../controllers/RegionalAnalysisController.java | 11 ++--------- .../analysis/models/AnalysisRequest.java | 16 ++++++++-------- .../conveyal/analysis/persistence/MongoMap.java | 1 - 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 6b8bc568e..7c8cf3a3f 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -1,7 +1,6 @@ package com.conveyal.analysis.components; import com.conveyal.analysis.AnalysisServerException; -import com.conveyal.r5.SoftwareVersion; import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.eventbus.ErrorEvent; import com.conveyal.analysis.components.eventbus.EventBus; @@ -9,6 +8,7 @@ import com.conveyal.analysis.controllers.HttpController; import com.conveyal.analysis.util.JsonUtil; import com.conveyal.file.FileStorage; +import com.conveyal.r5.SoftwareVersion; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.fileupload.FileUploadException; import org.slf4j.Logger; @@ -19,9 +19,7 @@ import java.io.IOException; import java.time.Duration; import java.time.Instant; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static com.conveyal.analysis.AnalysisServerException.Type.BAD_REQUEST; import static com.conveyal.analysis.AnalysisServerException.Type.FORBIDDEN; diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 4ed241bd5..a80a6ad4f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -133,12 +133,8 @@ private Object singlePoint(Request request, Response response) { AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); // Transform the analysis UI/backend task format into a slightly different type for R5 workers. TravelTimeSurfaceTask task = new TravelTimeSurfaceTask(); - analysisRequest.populateTask(task, accessGroup); + analysisRequest.populateTask(task, userPermissions); - Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, userPermissions); - // Transform the analysis UI/backend task format into a slightly different type for R5 workers. - TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest - .populateTask(new TravelTimeSurfaceTask(), project, userPermissions); // If destination opportunities are supplied, prepare to calculate accessibility worker-side if (notNullOrEmpty(analysisRequest.destinationPointSetIds)){ // Look up all destination opportunity data sets from the database and derive their storage keys. @@ -173,8 +169,7 @@ private Object singlePoint(Request request, Response response) { String address = broker.getWorkerAddress(workerCategory); if (address == null) { // There are no workers that can handle this request. Request some. - WorkerTags workerTags = new WorkerTags(accessGroup, userEmail, analysisRequest.projectId, analysisRequest.regionId); - WorkerTags workerTags = new WorkerTags(userPermissions, project._id, project.regionId); + WorkerTags workerTags = new WorkerTags(userPermissions, analysisRequest.projectId, analysisRequest.regionId); broker.createOnDemandWorkerInCategory(workerCategory, workerTags); // No workers exist. Kick one off and return "service unavailable". response.header("Retry-After", "30"); diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 2363650ec..d5819f25b 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -376,11 +376,7 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro // TODO now this is setting cutoffs and percentiles in the regional (template) task. // why is some stuff set in this populate method, and other things set here in the caller? RegionalTask task = new RegionalTask(); - analysisRequest.populateTask(task, accessGroup); - Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, userPermissions); - // TODO now this is setting cutoffs and percentiles in the regional (template) task. - // why is some stuff set in this populate method, and other things set here in the caller? - RegionalTask task = (RegionalTask) analysisRequest.populateTask(new RegionalTask(), project, userPermissions); + analysisRequest.populateTask(task, userPermissions); // Set the destination PointSets, which are required for all non-Taui regional requests. if (!analysisRequest.makeTauiSite) { @@ -477,11 +473,8 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro regionalAnalysis.west = task.west; regionalAnalysis.width = task.width; - regionalAnalysis.accessGroup = accessGroup; - regionalAnalysis.bundleId = analysisRequest.bundleId; - regionalAnalysis.createdBy = email; regionalAnalysis.accessGroup = userPermissions.accessGroup; - regionalAnalysis.bundleId = project.bundleId; + regionalAnalysis.bundleId = analysisRequest.bundleId; regionalAnalysis.createdBy = userPermissions.email; regionalAnalysis.destinationPointSetIds = analysisRequest.destinationPointSetIds; regionalAnalysis.name = analysisRequest.name; diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index e36ad902c..3da43be64 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -11,13 +11,13 @@ import com.conveyal.r5.analyst.scenario.Scenario; import com.conveyal.r5.api.util.LegMode; import com.conveyal.r5.api.util.TransitModes; -import com.mongodb.DBObject; import com.mongodb.QueryBuilder; import org.apache.commons.codec.digest.DigestUtils; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.EnumSet; import java.util.List; import java.util.stream.Collectors; @@ -154,11 +154,11 @@ public class AnalysisRequest { /** * Create the R5 `Scenario` from this request. */ - public Scenario createScenario (String accessGroup) { - DBObject query = "all".equals(scenarioId) - ? QueryBuilder.start("projectId").is(projectId).get() - : QueryBuilder.start("_id").in(modificationIds).get(); - List modifications = Persistence.modifications.findPermitted(query, accessGroup); + public Scenario createScenario (UserPermissions userPermissions) { + QueryBuilder query = "all".equals(scenarioId) + ? QueryBuilder.start("projectId").is(projectId) + : QueryBuilder.start("_id").in(modificationIds); + Collection modifications = Persistence.modifications.findPermitted(query.get(), userPermissions); // `findPermitted` sorts by creation time by default. Nonces will be in the same order each time. String nonces = Arrays.toString(modifications.stream().map(m -> m.nonce).toArray()); String scenarioId = String.format("%s-%s", bundleId, DigestUtils.sha1Hex(nonces)); @@ -181,10 +181,10 @@ public Scenario createScenario (String accessGroup) { * TODO arguably this should be done by a method on the task classes themselves, with common parts factored out * to the same method on the superclass. */ - public void populateTask (AnalysisWorkerTask task, String accessGroup) { + public void populateTask (AnalysisWorkerTask task, UserPermissions userPermissions) { if (bounds == null) throw AnalysisServerException.badRequest("Analysis bounds must be set."); - task.scenario = createScenario(accessGroup); + task.scenario = createScenario(userPermissions); task.graphId = bundleId; task.workerVersion = workerVersion; task.maxFare = maxFare; diff --git a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java index 30eb25470..6001a82d2 100644 --- a/src/main/java/com/conveyal/analysis/persistence/MongoMap.java +++ b/src/main/java/com/conveyal/analysis/persistence/MongoMap.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.Collection; -import java.util.List; import static com.conveyal.analysis.persistence.AnalysisCollection.MONGO_PROP_ACCESS_GROUP; From b718b42c29ef1019cef27d48a0ac88d48c48afdb Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 9 Sep 2021 17:50:16 +0800 Subject: [PATCH 131/187] restore getMongoCollection method, add javadoc --- .../com/conveyal/analysis/persistence/AnalysisDB.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java index 571038788..ababb330a 100644 --- a/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java +++ b/src/main/java/com/conveyal/analysis/persistence/AnalysisDB.java @@ -79,6 +79,14 @@ public AnalysisCollection getAnalysisCollection ( return new AnalysisCollection(database.getCollection(name, clazz), clazz); } + /** + * Lower-level access to Mongo collections without the user-oriented functionality of BaseModel (accessGroup etc.) + * This is useful when storing server monitoring data (time series or event data) in a cloud environment. + */ + public MongoCollection getMongoCollection (String name, Class clazz) { + return database.getCollection(name, clazz); + } + /** Interface to supply configuration to this component. */ public interface Config { default String databaseUri() { return "mongodb://127.0.0.1:27017"; } From b5aae18d3674832499555c62fa217f0b68617917 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Thu, 9 Sep 2021 21:28:36 +0800 Subject: [PATCH 132/187] Add Project collection back in --- .../com/conveyal/analysis/models/Project.java | 25 +++++++++++++++++++ .../analysis/persistence/Persistence.java | 5 +++- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/analysis/models/Project.java diff --git a/src/main/java/com/conveyal/analysis/models/Project.java b/src/main/java/com/conveyal/analysis/models/Project.java new file mode 100644 index 000000000..1e3993b1b --- /dev/null +++ b/src/main/java/com/conveyal/analysis/models/Project.java @@ -0,0 +1,25 @@ +package com.conveyal.analysis.models; + +import com.conveyal.analysis.AnalysisServerException; + +/** + * Represents a TAUI project + */ +public class Project extends Model implements Cloneable { + /** Names of the variants of this project */ + public String[] variants; + + public String regionId; + + public String bundleId; + + public AnalysisRequest analysisRequestSettings; + + public Project clone () { + try { + return (Project) super.clone(); + } catch (CloneNotSupportedException e) { + throw AnalysisServerException.unknown(e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/analysis/persistence/Persistence.java b/src/main/java/com/conveyal/analysis/persistence/Persistence.java index c77e341da..63107f88e 100644 --- a/src/main/java/com/conveyal/analysis/persistence/Persistence.java +++ b/src/main/java/com/conveyal/analysis/persistence/Persistence.java @@ -1,11 +1,11 @@ package com.conveyal.analysis.persistence; -import com.conveyal.analysis.models.AggregationArea; import com.conveyal.analysis.models.Bundle; import com.conveyal.analysis.models.JsonViews; import com.conveyal.analysis.models.Model; import com.conveyal.analysis.models.Modification; import com.conveyal.analysis.models.OpportunityDataset; +import com.conveyal.analysis.models.Project; import com.conveyal.analysis.models.Region; import com.conveyal.analysis.models.RegionalAnalysis; import com.conveyal.analysis.util.JsonUtil; @@ -35,6 +35,7 @@ public class Persistence { private static DB db; public static MongoMap modifications; + public static MongoMap projects; public static MongoMap bundles; public static MongoMap regions; public static MongoMap regionalAnalyses; @@ -52,6 +53,7 @@ public static void initializeStatically (AnalysisDB.Config config) { } db = mongo.getDB(config.databaseName()); modifications = getTable("modifications", Modification.class); + projects = getTable("projects", Project.class); bundles = getTable("bundles", Bundle.class); regions = getTable("regions", Region.class); regionalAnalyses = getTable("regional-analyses", RegionalAnalysis.class); @@ -61,6 +63,7 @@ public static void initializeStatically (AnalysisDB.Config config) { /** Connect to a Mongo table using MongoJack, which persists Java objects into Mongo. */ private static MongoMap getTable (String name, Class clazz) { DBCollection collection = db.getCollection(name); + collection.find().next(); ObjectMapper om = JsonUtil.getObjectMapper(JsonViews.Db.class, true); JacksonDBCollection coll = JacksonDBCollection.wrap(collection, clazz, String.class, om); return new MongoMap<>(coll, clazz); From 3182e20b4e36231fc2bae881c74499d3e67491da Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Thu, 9 Sep 2021 21:54:56 +0800 Subject: [PATCH 133/187] Remove accidental test code --- src/main/java/com/conveyal/analysis/persistence/Persistence.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/persistence/Persistence.java b/src/main/java/com/conveyal/analysis/persistence/Persistence.java index 63107f88e..85e8be964 100644 --- a/src/main/java/com/conveyal/analysis/persistence/Persistence.java +++ b/src/main/java/com/conveyal/analysis/persistence/Persistence.java @@ -63,7 +63,6 @@ public static void initializeStatically (AnalysisDB.Config config) { /** Connect to a Mongo table using MongoJack, which persists Java objects into Mongo. */ private static MongoMap getTable (String name, Class clazz) { DBCollection collection = db.getCollection(name); - collection.find().next(); ObjectMapper om = JsonUtil.getObjectMapper(JsonViews.Db.class, true); JacksonDBCollection coll = JacksonDBCollection.wrap(collection, clazz, String.class, om); return new MongoMap<>(coll, clazz); From 74aaa9e77d96ca757971cd1584a5164c417fbe8d Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Fri, 10 Sep 2021 09:00:10 +0800 Subject: [PATCH 134/187] Update UI version in Cypress tests --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index 940822a0c..eacc46618 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: 8218cfeb98962c25841daf7db0791443a743b707 + ref: 68040a2e83039539d47b43fc204cfadcdb795cb5 path: ui - uses: actions/checkout@v2 with: From f4fa7c42324048ba6c43d153f6e7ca42c5a3cb79 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 14 Sep 2021 22:43:28 +0800 Subject: [PATCH 135/187] Simplify and shorten group description --- .../datasource/derivation/AggregationAreaDerivation.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index 6a883fd09..8dbd31c37 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -147,8 +147,7 @@ handled in a streaming fashion (in constant memory). public void action (ProgressListener progressListener) throws Exception { ArrayList aggregationAreas = new ArrayList<>(); - String groupDescription = "Convert polygons to aggregation areas, " + - (mergePolygons ? "merging all polygons." : "one area per polygon."); + String groupDescription = "Aggregation areas from polygons"; DataGroup dataGroup = new DataGroup(userPermissions, spatialDataSource._id.toString(), groupDescription); progressListener.beginTask("Reading data source", finalFeatures.size() + 1); From 86f1a2a412a3a737fc4290436bf9c0ed072d46ea Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 15 Sep 2021 17:43:00 +0800 Subject: [PATCH 136/187] perform validation in BackendConfig, update comments --- analysis.properties.template | 15 +++------------ .../com/conveyal/analysis/BackendConfig.java | 9 +++++++++ .../conveyal/analysis/components/HttpApi.java | 17 +++++------------ 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/analysis.properties.template b/analysis.properties.template index a232e08a9..2f9ac6367 100644 --- a/analysis.properties.template +++ b/analysis.properties.template @@ -9,11 +9,6 @@ database-uri=mongodb://localhost # The name of the database in the Mongo instance. database-name=analysis -# The URL where the frontend is hosted. -# In production this should point to a cached CDN for speed. e.g. https://d1uqjuy3laovxb.cloudfront.net -# In staging this should be the underlying S3 URL so files are not cached and you see the most recent deployment. -frontend-url=https://localhost - # The S3 bucket where we can find tiles of the entire US census, built with Conveyal seamless-census. seamless-census-bucket=lodes-data-2014 seamless-census-region=us-east-1 @@ -28,13 +23,9 @@ aws-region=eu-west-1 # The port on which the server will listen for connections from clients and workers. server-port=7070 -# The origin where the frontend is hosted. Due to the same-origin policy, cross-origin requests are generally blocked. -# However, setting this to an origin will add an Access-Control-Allow-Origin header to allow cross-origin requests from -# that origin. For instance, when running locally, this will generally be http://localhost:3000. It is possible to set -# this to *, but that allows requests from anywhere. While this should be relatively safe when authentication is enabled -# since authentication is not handled through cookies, it does increase the attack surface and better practice is to set -# this to the actual origin where the UI is hosted. This is different from frontend-url, since frontend-url is just where -# the frontend code can be retrieved from, and may not even be a valid origin (e.g. if it has a path name). +# The origin where the frontend is hosted. When running locally, this will generally be http://localhost:3000. +# It should be relatively safe to set this to * (allowing requests from anywhere) when authentication is enabled. +# This increases attack surface though, so it is preferable to set this to the specific origin where the UI is hosted. access-control-allow-origin=http://localhost:3000 # A temporary location to store scratch files. The path can be absolute or relative. diff --git a/src/main/java/com/conveyal/analysis/BackendConfig.java b/src/main/java/com/conveyal/analysis/BackendConfig.java index 68a747560..cf0b1d391 100644 --- a/src/main/java/com/conveyal/analysis/BackendConfig.java +++ b/src/main/java/com/conveyal/analysis/BackendConfig.java @@ -69,9 +69,18 @@ protected BackendConfig (Properties properties) { lightThreads = intProp("light-threads"); heavyThreads = intProp("heavy-threads"); maxWorkers = intProp("max-workers"); + validate(); exitIfErrors(); } + private final void validate () { + if (allowOrigin() == null || allowOrigin().equals("null")) { + // Access-Control-Allow-Origin: null opens unintended security holes: + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin + throw new IllegalArgumentException("Access-Control-Allow-Origin should not be null"); + } + } + // INTERFACE IMPLEMENTATIONS // Methods implementing Component and HttpController Config interfaces. // Note that one method can implement several Config interfaces at once. diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 53affdd34..2400391d6 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -83,27 +83,20 @@ private spark.Service configureSparkService () { // Record when the request started, so we can measure elapsed response time. req.attribute(REQUEST_START_TIME_ATTRIBUTE, Instant.now()); - // Don't require authentication to view the main page, or for internal API endpoints contacted by workers. - // FIXME those internal endpoints should be hidden from the outside world by the reverse proxy. - // Or now with non-static Spark we can run two HTTP servers on different ports. - - // Set CORS headers, to allow requests to this API server from a frontend hosted on a different domain + // Set CORS headers to allow requests to this API server from a frontend hosted on a different domain. // This used to be hardwired to Access-Control-Allow-Origin: * but that leaves the server open to XSRF // attacks when authentication is disabled (e.g. when running locally). - if (config.allowOrigin() == null || config.allowOrigin().equals("null")) { - // Access-Control-Allow-Origin: null opens unintended security holes: - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin - throw new IllegalArgumentException("Access-Control-Allow-Origin should not be null"); - } - res.header("Access-Control-Allow-Origin", config.allowOrigin()); - // for caching, signal to the browser that responses may be different based on origin + // For caching, signal to the browser that responses may be different based on origin. + // TODO clarify why this is important, considering that normally all requests come from the same origin. res.header("Vary", "Origin"); // The default MIME type is JSON. This will be overridden by the few controllers that do not return JSON. res.type("application/json"); // Do not require authentication for internal API endpoints contacted by workers or for OPTIONS requests. + // FIXME those internal endpoints should be hidden from the outside world by the reverse proxy. + // Or now with non-static Spark we can run two HTTP servers on different ports. String method = req.requestMethod(); String pathInfo = req.pathInfo(); boolean authorize = pathInfo.startsWith("/api") && !"OPTIONS".equalsIgnoreCase(method); From 0829723d12e73a34bc7f28f399ce9da1b8569729 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 17 Sep 2021 14:02:14 +0800 Subject: [PATCH 137/187] remove redundant validation All parameters are required so a missing (null) header value will already prevent startup. Throwing an exception here fails immediately, preventing reporting of any other missing parameters. --- src/main/java/com/conveyal/analysis/BackendConfig.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/BackendConfig.java b/src/main/java/com/conveyal/analysis/BackendConfig.java index cf0b1d391..68a747560 100644 --- a/src/main/java/com/conveyal/analysis/BackendConfig.java +++ b/src/main/java/com/conveyal/analysis/BackendConfig.java @@ -69,18 +69,9 @@ protected BackendConfig (Properties properties) { lightThreads = intProp("light-threads"); heavyThreads = intProp("heavy-threads"); maxWorkers = intProp("max-workers"); - validate(); exitIfErrors(); } - private final void validate () { - if (allowOrigin() == null || allowOrigin().equals("null")) { - // Access-Control-Allow-Origin: null opens unintended security holes: - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin - throw new IllegalArgumentException("Access-Control-Allow-Origin should not be null"); - } - } - // INTERFACE IMPLEMENTATIONS // Methods implementing Component and HttpController Config interfaces. // Note that one method can implement several Config interfaces at once. From 56d60ad2c83e6d2405a25b8bb007e209d70e8ce6 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 17 Sep 2021 16:55:37 +0800 Subject: [PATCH 138/187] ensure uploaded files are saved to disk Use standard HttpUtils.getRequestFiles in OpportunityDatasetController which causes all uploaded files to be disk files rather than in-memory byte buffers. Uploads were failing due to small Shapefile sidecars not being written to disk. Also added clearer messages to the checkState calls that were failing. --- .../controllers/OpportunityDatasetController.java | 14 ++++---------- .../analysis/datasource/DataSourceUtil.java | 8 ++++---- .../java/com/conveyal/analysis/util/HttpUtils.java | 14 +++++++++----- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 17882917f..1b7e64fd1 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -67,7 +67,7 @@ public class OpportunityDatasetController implements HttpController { private static final Logger LOG = LoggerFactory.getLogger(OpportunityDatasetController.class); - private static final FileItemFactory fileItemFactory = new DiskFileItemFactory(); + private static final FileItemFactory fileItemFactory = new DiskFileItemFactory(0, null); // Component Dependencies @@ -297,15 +297,9 @@ private List createFreeFormPointSetsFromCsv(FileItem csvFileIt * The request should be a multipart/form-data POST request, containing uploaded files and associated parameters. */ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Response res) { + // Extract user info, uploaded files and form fields from the incoming request. final UserPermissions userPermissions = UserPermissions.from(req); - final Map> formFields; - try { - ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); - formFields = sfu.parseParameterMap(req.raw()); - } catch (FileUploadException e) { - // We can't even get enough information to create a status tracking object. Re-throw an exception. - throw AnalysisServerException.fileUpload("Unable to parse opportunity dataset. " + ExceptionUtils.stackTraceString(e)); - } + final Map> formFields = HttpUtils.getRequestFiles(req.raw()); // Parse required fields. Will throw a ServerException on failure. final String sourceName = HttpUtils.getFormField(formFields, "Name", true); @@ -376,7 +370,7 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res if (pointsets.isEmpty()) { throw new RuntimeException("No opportunity dataset was created from the files uploaded."); } - LOG.info("Uploading opportunity datasets to S3 and storing metadata in database."); + LOG.info("Moving opportunity datasets into storage and adding metadata to database."); // Create a single unique ID string that will be referenced by all opportunity datasets produced by // this upload. This allows us to group together datasets from the same source and associate them with // the file(s) that produced them. diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java index 514c5443a..5ee6ee8b4 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java @@ -82,11 +82,11 @@ public static FileStorageFormat detectUploadFormatAndValidate (List fi */ private static void checkFileCharacteristics (List fileItems) { for (FileItem fileItem : fileItems) { - checkState(fileItem instanceof DiskFileItem); + checkState(fileItem instanceof DiskFileItem, "Uploaded file was not stored to disk."); File diskFile = ((DiskFileItem)fileItem).getStoreLocation(); - checkState(diskFile.exists()); - checkState(diskFile.canRead()); - checkState(diskFile.length() > 0); + checkState(diskFile.exists(), "Uploaded file does not exist on filesystem as expected."); + checkState(diskFile.canRead(), "Read permissions were not granted on uploaded file."); + checkState(diskFile.length() > 0, "Uploaded file was empty (contained no data)."); } } diff --git a/src/main/java/com/conveyal/analysis/util/HttpUtils.java b/src/main/java/com/conveyal/analysis/util/HttpUtils.java index 2e3737724..f3d3f26da 100644 --- a/src/main/java/com/conveyal/analysis/util/HttpUtils.java +++ b/src/main/java/com/conveyal/analysis/util/HttpUtils.java @@ -13,7 +13,10 @@ import java.util.Map; public abstract class HttpUtils { - /** Extract files from a Spark Request containing RFC 1867 multipart form-based file upload data. */ + + /** + * Extract files from a Spark Request containing RFC 1867 multipart form-based file upload data. + */ public static Map> getRequestFiles (HttpServletRequest req) { // The Javadoc on this factory class doesn't say anything about thread safety. Looking at the source code it // all looks threadsafe. But also very lightweight to instantiate, so in this code run by multiple threads @@ -22,13 +25,14 @@ public static Map> getRequestFiles (HttpServletRequest re // uniform way in other threads, after the request handler has returned. This does however cause some very // small form fields to be written to disk files. Ideally we'd identify the smallest actual file we'll ever // handle and set the threshold a little higher. The downside is that if a tiny file is actually uploaded even - // by accident, our code will not be able to get a file handle for it and fail. - FileItemFactory fileItemFactory = new DiskFileItemFactory(0, null); - ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); + // by accident, our code will not be able to get a file handle for it and fail. Some legitimate files like + // Shapefile .prj sidecars can be really small. try { + FileItemFactory fileItemFactory = new DiskFileItemFactory(0, null); + ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); return sfu.parseParameterMap(req); } catch (Exception e) { - throw AnalysisServerException.badRequest(ExceptionUtils.stackTraceString(e)); + throw AnalysisServerException.fileUpload(ExceptionUtils.stackTraceString(e)); } } From c7a306c38db815fcd2a35917b0a6996206e84631 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 17 Sep 2021 20:17:46 +0800 Subject: [PATCH 139/187] DataGroups and progress on seamless census import Add a dataGroupId to OpportunityDatasets. We were assigning an (unsaved) DataSource, but not putting them in DataGroups. Also stronger typing on parameters (pass DataGroup not String id) --- .../components/BackendComponents.java | 2 +- .../OpportunityDatasetController.java | 46 +++++++++++++++---- .../derivation/AggregationAreaDerivation.java | 4 +- .../grids/SeamlessCensusGridExtractor.java | 9 +++- .../analysis/models/OpportunityDataset.java | 3 ++ .../com/conveyal/analysis/util/HttpUtils.java | 2 + .../conveyal/data/census/CensusExtractor.java | 7 +-- .../conveyal/data/census/SeamlessSource.java | 23 ++++++---- .../r5/analyst/progress/WorkProduct.java | 5 +- 9 files changed, 71 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/BackendComponents.java b/src/main/java/com/conveyal/analysis/components/BackendComponents.java index 93282df8b..db0e3ce1a 100644 --- a/src/main/java/com/conveyal/analysis/components/BackendComponents.java +++ b/src/main/java/com/conveyal/analysis/components/BackendComponents.java @@ -86,7 +86,7 @@ public List standardHttpControllers () { // and therefore subject to authentication and authorization. new GtfsController(gtfsCache), new BundleController(this), - new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor), + new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor, database), new RegionalAnalysisController(broker, fileStorage), new FileStorageController(fileStorage, database), new AggregationAreaController(fileStorage, database, taskScheduler), diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index 1b7e64fd1..c22c08988 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -4,9 +4,12 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.analysis.grids.SeamlessCensusGridExtractor; +import com.conveyal.analysis.models.DataGroup; import com.conveyal.analysis.models.OpportunityDataset; import com.conveyal.analysis.models.Region; import com.conveyal.analysis.models.SpatialDataSource; +import com.conveyal.analysis.persistence.AnalysisCollection; +import com.conveyal.analysis.persistence.AnalysisDB; import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.FileItemInputStreamProvider; import com.conveyal.analysis.util.HttpUtils; @@ -18,7 +21,10 @@ import com.conveyal.r5.analyst.FreeFormPointSet; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.PointSet; +import com.conveyal.r5.analyst.progress.NoopProgressListener; import com.conveyal.r5.analyst.progress.Task; +import com.conveyal.r5.analyst.progress.WorkProduct; +import com.conveyal.r5.analyst.progress.WorkProductType; import com.conveyal.r5.util.ExceptionUtils; import com.conveyal.r5.util.InputStreamProvider; import com.conveyal.r5.util.ProgressListener; @@ -59,6 +65,7 @@ import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.GRIDS; import static com.conveyal.r5.analyst.WebMercatorGridPointSet.parseZoom; +import static com.conveyal.r5.analyst.progress.WorkProductType.OPPORTUNITY_DATASET; /** * Controller that handles fetching opportunity datasets (grids and other pointset formats). @@ -67,22 +74,26 @@ public class OpportunityDatasetController implements HttpController { private static final Logger LOG = LoggerFactory.getLogger(OpportunityDatasetController.class); - private static final FileItemFactory fileItemFactory = new DiskFileItemFactory(0, null); - // Component Dependencies private final FileStorage fileStorage; private final TaskScheduler taskScheduler; private final SeamlessCensusGridExtractor extractor; + // Database tables + + private final AnalysisCollection dataGroupCollection; + public OpportunityDatasetController ( FileStorage fileStorage, TaskScheduler taskScheduler, - SeamlessCensusGridExtractor extractor + SeamlessCensusGridExtractor extractor, + AnalysisDB database ) { this.fileStorage = fileStorage; this.taskScheduler = taskScheduler; this.extractor = extractor; + this.dataGroupCollection = database.getAnalysisCollection("dataGroups", DataGroup.class); } /** Store upload status objects FIXME trivial Javadoc */ @@ -134,29 +145,32 @@ private boolean clearStatus(Request req, Response res) { return uploadStatuses.removeIf(s -> s.id.equals(statusId)); } - private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) { + private OpportunityDatasetUploadStatus downloadLODES (Request req, Response res) { final String regionId = req.params("regionId"); final int zoom = parseZoom(req.queryParams("zoom")); final UserPermissions userPermissions = UserPermissions.from(req); final Region region = Persistence.regions.findByIdIfPermitted(regionId, userPermissions); // Common UUID for all LODES datasets created in this download (e.g. so they can be grouped together and - // deleted as a batch using deleteSourceSet) + // deleted as a batch using deleteSourceSet) TODO use DataGroup and DataSource (creating only one DataSource per region). // The bucket name contains the specific lodes data set and year so works as an appropriate name final OpportunityDatasetUploadStatus status = new OpportunityDatasetUploadStatus(regionId, extractor.sourceName); addStatusAndRemoveOldStatuses(status); + // TODO we should be reusing the same source from Mongo, not making new ephemeral ones on each extract operation SpatialDataSource source = new SpatialDataSource(userPermissions, extractor.sourceName); source.regionId = regionId; + // Make a new group that will containin the N OpportunityDatasets we're saving. + String description = String.format("Import %s to %s", extractor.sourceName, region.name); + DataGroup dataGroup = new DataGroup(userPermissions, source._id.toString(), description); taskScheduler.enqueue(Task.create("Extracting LODES data") .forUser(userPermissions) .setHeavy(true) - .withWorkProduct(source) .withAction((progressListener) -> { try { status.message = "Extracting census data for region"; - List grids = extractor.censusDataForBounds(region.bounds, zoom); - updateAndStoreDatasets(source, status, grids); + List grids = extractor.censusDataForBounds(region.bounds, zoom, progressListener); + updateAndStoreDatasets(source, dataGroup, status, grids, progressListener); } catch (IOException e) { status.completeWithError(e); LOG.error("Exception processing LODES data: " + ExceptionUtils.stackTraceString(e)); @@ -171,16 +185,21 @@ private OpportunityDatasetUploadStatus downloadLODES(Request req, Response res) * that PointSet and store it in Mongo. */ private void updateAndStoreDatasets (SpatialDataSource source, + DataGroup dataGroup, OpportunityDatasetUploadStatus status, - List pointSets) { + List pointSets, + com.conveyal.r5.analyst.progress.ProgressListener progressListener) { status.status = Status.UPLOADING; status.totalGrids = pointSets.size(); + progressListener.beginTask("Storing opportunity data", pointSets.size()); + // Create an OpportunityDataset holding some metadata about each PointSet (Grid or FreeForm). final List datasets = new ArrayList<>(); for (PointSet pointSet : pointSets) { OpportunityDataset dataset = new OpportunityDataset(); dataset.sourceName = source.name; dataset.sourceId = source._id.toString(); + dataset.dataGroupId = dataGroup._id.toString(); dataset.createdBy = source.createdBy; dataset.accessGroup = source.accessGroup; dataset.regionId = source.regionId; @@ -234,7 +253,11 @@ private void updateAndStoreDatasets (SpatialDataSource source, status.completeWithError(e); throw AnalysisServerException.unknown(e); } + progressListener.increment(); } + // Set the workProduct - TODO update UI so it can handle a link to a group of OPPORTUNITY_DATASET + dataGroupCollection.insert(dataGroup); + progressListener.setWorkProduct(WorkProduct.forDataGroup(OPPORTUNITY_DATASET, dataGroup, source.regionId)); } private static FileStorageFormat getFormatCode (PointSet pointSet) { @@ -374,9 +397,12 @@ private OpportunityDatasetUploadStatus createOpportunityDataset(Request req, Res // Create a single unique ID string that will be referenced by all opportunity datasets produced by // this upload. This allows us to group together datasets from the same source and associate them with // the file(s) that produced them. + // Currently we are creating the DataSource document in Mongo but not actually saving the source files. + // Some methods like createGridsFromShapefile above "consume" those files by moving them into a tempdir. SpatialDataSource source = new SpatialDataSource(userPermissions, sourceName); source.regionId = regionId; - updateAndStoreDatasets(source, status, pointsets); + DataGroup dataGroup = new DataGroup(userPermissions, source._id.toString(), "Import opportunity data"); + updateAndStoreDatasets(source, dataGroup, status, pointsets, new NoopProgressListener()); } catch (Exception e) { e.printStackTrace(); status.completeWithError(e); diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index 8dbd31c37..c8b518099 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -198,9 +198,7 @@ public void action (ProgressListener progressListener) throws Exception { }); aggregationAreaCollection.insertMany(aggregationAreas); dataGroupCollection.insert(dataGroup); - progressListener.setWorkProduct(WorkProduct.forDataGroup( - AGGREGATION_AREA, dataGroup._id.toString(), spatialDataSource.regionId) - ); + progressListener.setWorkProduct(WorkProduct.forDataGroup(AGGREGATION_AREA, dataGroup, spatialDataSource.regionId)); progressListener.increment(); } diff --git a/src/main/java/com/conveyal/analysis/grids/SeamlessCensusGridExtractor.java b/src/main/java/com/conveyal/analysis/grids/SeamlessCensusGridExtractor.java index 6bf0b3095..289b9987f 100644 --- a/src/main/java/com/conveyal/analysis/grids/SeamlessCensusGridExtractor.java +++ b/src/main/java/com/conveyal/analysis/grids/SeamlessCensusGridExtractor.java @@ -5,6 +5,7 @@ import com.conveyal.data.census.S3SeamlessSource; import com.conveyal.data.geobuf.GeobufFeature; import com.conveyal.r5.analyst.Grid; +import com.conveyal.r5.analyst.progress.ProgressListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,17 +49,20 @@ public SeamlessCensusGridExtractor (Config config) { /** * Retrieve data for bounds and save to a bucket under a given key */ - public List censusDataForBounds (Bounds bounds, int zoom) throws IOException { + public List censusDataForBounds (Bounds bounds, int zoom, ProgressListener progressListener) throws IOException { long startTime = System.currentTimeMillis(); // All the features are buffered in a Map in memory. This could be problematic on large areas. - Map features = source.extract(bounds.north, bounds.east, bounds.south, bounds.west, false); + Map features = + source.extract(bounds.north, bounds.east, bounds.south, bounds.west, false, progressListener); if (features.isEmpty()) { LOG.info("No seamless census data found here, not pre-populating grids"); return new ArrayList<>(); } + progressListener.beginTask("Processing census blocks", features.size()); + // One string naming each attribute (column) in the incoming census data. Map grids = new HashMap<>(); for (GeobufFeature feature : features.values()) { @@ -81,6 +85,7 @@ public List censusDataForBounds (Bounds bounds, int zoom) throws IOExcepti } grid.incrementFromPixelWeights(weights, value.doubleValue()); } + progressListener.increment(); } long endTime = System.currentTimeMillis(); diff --git a/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java b/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java index 77696cc04..773c4efd2 100644 --- a/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java +++ b/src/main/java/com/conveyal/analysis/models/OpportunityDataset.java @@ -23,6 +23,9 @@ public class OpportunityDataset extends Model { /** The unique id for the data source (CSV file, Shapefile etc.) from which this dataset was derived. */ public String sourceId; + /** The ID of the DataGroup that this OpportunityDataset belongs to (all created at once from a single source). */ + public String dataGroupId; + /** * Bucket name on S3 where the opportunity data itself is persisted. Deprecated: as of April 2021, the FileStorage * system encapsulates how local or remote storage coordinates are derived from the FileCategory. diff --git a/src/main/java/com/conveyal/analysis/util/HttpUtils.java b/src/main/java/com/conveyal/analysis/util/HttpUtils.java index f3d3f26da..de3ea3b49 100644 --- a/src/main/java/com/conveyal/analysis/util/HttpUtils.java +++ b/src/main/java/com/conveyal/analysis/util/HttpUtils.java @@ -27,6 +27,8 @@ public static Map> getRequestFiles (HttpServletRequest re // handle and set the threshold a little higher. The downside is that if a tiny file is actually uploaded even // by accident, our code will not be able to get a file handle for it and fail. Some legitimate files like // Shapefile .prj sidecars can be really small. + // If we always saved the FileItems via write() or read them with getInputStream() they would not all need to + // be on disk. try { FileItemFactory fileItemFactory = new DiskFileItemFactory(0, null); ServletFileUpload sfu = new ServletFileUpload(fileItemFactory); diff --git a/src/main/java/com/conveyal/data/census/CensusExtractor.java b/src/main/java/com/conveyal/data/census/CensusExtractor.java index 8a89b25b9..86918c2d7 100644 --- a/src/main/java/com/conveyal/data/census/CensusExtractor.java +++ b/src/main/java/com/conveyal/data/census/CensusExtractor.java @@ -3,6 +3,7 @@ import com.conveyal.data.geobuf.GeobufEncoder; import com.conveyal.data.geobuf.GeobufFeature; import com.conveyal.geojson.GeoJsonModule; +import com.conveyal.r5.analyst.progress.NoopProgressListener; import com.fasterxml.jackson.databind.ObjectMapper; import org.locationtech.jts.geom.Geometry; @@ -47,7 +48,8 @@ public static void main (String... args) throws IOException { Double.parseDouble(args[2]), Double.parseDouble(args[3]), Double.parseDouble(args[4]), - false + false, + new NoopProgressListener() ); } else { @@ -57,8 +59,7 @@ public static void main (String... args) throws IOException { FileInputStream fis = new FileInputStream(new File(args[1])); FeatureCollection fc = om.readValue(fis, FeatureCollection.class); fis.close(); - - features = source.extract(fc.features.get(0).geometry, false); + features = source.extract(fc.features.get(0).geometry, false, new NoopProgressListener()); } OutputStream out; diff --git a/src/main/java/com/conveyal/data/census/SeamlessSource.java b/src/main/java/com/conveyal/data/census/SeamlessSource.java index 4824a268e..7d912eccd 100644 --- a/src/main/java/com/conveyal/data/census/SeamlessSource.java +++ b/src/main/java/com/conveyal/data/census/SeamlessSource.java @@ -2,6 +2,7 @@ import com.conveyal.data.geobuf.GeobufDecoder; import com.conveyal.data.geobuf.GeobufFeature; +import com.conveyal.r5.analyst.progress.ProgressListener; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -34,18 +35,22 @@ public abstract class SeamlessSource { private static final GeometryFactory geometryFactory = new GeometryFactory(); /** Extract features by bounding box */ - public Map extract(double north, double east, double south, double west, boolean onDisk) throws - IOException { + public Map extract( + double north, double east, double south, double west, boolean onDisk, ProgressListener progressListener + ) throws IOException { GeometricShapeFactory factory = new GeometricShapeFactory(geometryFactory); factory.setCentre(new Coordinate((east + west) / 2, (north + south) / 2)); factory.setWidth(east - west); factory.setHeight(north - south); Polygon rect = factory.createRectangle(); - return extract(rect, onDisk); + return extract(rect, onDisk, progressListener); } /** Extract features by arbitrary polygons */ - public Map extract(Geometry bounds, boolean onDisk) throws IOException { + public Map extract ( + Geometry bounds, boolean onDisk, ProgressListener progressListener + ) throws IOException { + Map ret; if (onDisk) @@ -65,6 +70,7 @@ public Map extract(Geometry bounds, boolean onDisk) throws int tcount = (maxX - minX + 1) * (maxY - minY + 1); LOG.info("Requesting {} tiles", tcount); + progressListener.beginTask("Reading census tiles", tcount); int fcount = 0; @@ -72,14 +78,13 @@ public Map extract(Geometry bounds, boolean onDisk) throws for (int x = minX; x <= maxX; x++) { for (int y = minY; y <= maxY; y++) { InputStream is = getInputStream(x, y); - - if (is == null) + if (is == null) { // no data in this tile + progressListener.increment(); continue; - + } // decoder closes input stream as soon as it has read the tile GeobufDecoder decoder = new GeobufDecoder(new GZIPInputStream(new BufferedInputStream(is))); - while (decoder.hasNext()) { GeobufFeature f = decoder.next(); // blocks are duplicated at the edges of tiles, no need to import twice @@ -94,9 +99,9 @@ public Map extract(Geometry bounds, boolean onDisk) throws LOG.info("Read {} features", fcount); } } + progressListener.increment(); } } - return ret; } diff --git a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java index eb4da1d09..40dffa198 100644 --- a/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java +++ b/src/main/java/com/conveyal/r5/analyst/progress/WorkProduct.java @@ -1,6 +1,7 @@ package com.conveyal.r5.analyst.progress; import com.conveyal.analysis.models.BaseModel; +import com.conveyal.analysis.models.DataGroup; /** * A unique identifier for the final product of a single TaskAction. Currently this serves as both an internal data @@ -31,7 +32,7 @@ public static WorkProduct forModel (BaseModel model) { return new WorkProduct(WorkProductType.forModel(model), model._id.toString(), null); } - public static WorkProduct forDataGroup (WorkProductType type, String dataGroupId, String regionId) { - return new WorkProduct(type, dataGroupId, regionId, true); + public static WorkProduct forDataGroup (WorkProductType type, DataGroup dataGroup, String regionId) { + return new WorkProduct(type, dataGroup._id.toString(), regionId, true); } } From 333c3639108b286dc225bebade50bd90d72cf9b4 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 17 Sep 2021 23:03:22 +0800 Subject: [PATCH 140/187] fix capitalization in /api/dataSource endpoint also generalized one log message for local use --- .../com/conveyal/analysis/controllers/DataSourceController.java | 2 +- .../analysis/controllers/OpportunityDatasetController.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java index 171952fc3..b0bf7c626 100644 --- a/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java +++ b/src/main/java/com/conveyal/analysis/controllers/DataSourceController.java @@ -135,7 +135,7 @@ private String handleUpload (Request req, Response res) { @Override public void registerEndpoints (spark.Service sparkService) { - sparkService.path("/api/datasource", () -> { + sparkService.path("/api/dataSource", () -> { sparkService.get("/", this::getAllDataSourcesForRegion, toJson); sparkService.get("/:_id", this::getOneDataSourceById, toJson); sparkService.delete("/:_id", this::deleteOneDataSourceById, toJson); diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index c22c08988..ff097066d 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -246,7 +246,7 @@ private void updateAndStoreDatasets (SpatialDataSource source, if (status.uploadedGrids == status.totalGrids) { status.completeSuccessfully(); } - LOG.info("Completed {}/{} uploads for {}", status.uploadedGrids, status.totalGrids, status.name); + LOG.info("Moved {}/{} files into storage for {}", status.uploadedGrids, status.totalGrids, status.name); } catch (NumberFormatException e) { throw new AnalysisServerException("Error attempting to parse number in uploaded file: " + e.toString()); } catch (Exception e) { From f14ad3c8b36bd40a4e17507f7debe3d16e0007ed Mon Sep 17 00:00:00 2001 From: ansons Date: Fri, 17 Sep 2021 17:27:47 -0400 Subject: [PATCH 141/187] set regionId in analysisRequest --- .../com/conveyal/analysis/controllers/BrokerController.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index a80a6ad4f..7a6aad57a 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -124,13 +124,16 @@ private Object singlePoint(Request request, Response response) { // Deserialize the task in the request body so we can see what kind of worker it wants. // Perhaps we should allow both travel time surface and accessibility calculation tasks to be done as single points. // AnalysisRequest (backend) vs. AnalysisTask (R5) - // The accessgroup stuff is copypasta from the old single point controller. // We already know the user is authenticated, and we need not check if they have access to the graphs etc, // as they're all coded with UUIDs which contain significantly more entropy than any human's account password. UserPermissions userPermissions = UserPermissions.from(request); final long startTimeMsec = System.currentTimeMillis(); AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); + // TODO discuss whether the UI should set the regionId, or if we even need it. + analysisRequest.regionId = + Persistence.bundles.findByIdIfPermitted(analysisRequest.bundleId, userPermissions).regionId; + // Transform the analysis UI/backend task format into a slightly different type for R5 workers. TravelTimeSurfaceTask task = new TravelTimeSurfaceTask(); analysisRequest.populateTask(task, userPermissions); From 4c59ef88e32a593aa6c55031e22f661f28c233c8 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 20 Sep 2021 21:04:53 +0800 Subject: [PATCH 142/187] expect regionId in request from UI --- .../conveyal/analysis/controllers/BrokerController.java | 9 ++++++--- .../com/conveyal/analysis/models/AnalysisRequest.java | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 7a6aad57a..eddb3030d 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -130,9 +130,12 @@ private Object singlePoint(Request request, Response response) { final long startTimeMsec = System.currentTimeMillis(); AnalysisRequest analysisRequest = objectFromRequestBody(request, AnalysisRequest.class); - // TODO discuss whether the UI should set the regionId, or if we even need it. - analysisRequest.regionId = - Persistence.bundles.findByIdIfPermitted(analysisRequest.bundleId, userPermissions).regionId; + // Some parameters like regionId weren't sent by older frontends. Fail fast on missing parameters. + checkNotNull(analysisRequest.regionId); + checkNotNull(analysisRequest.projectId); + checkNotNull(analysisRequest.bundleId); + checkNotNull(analysisRequest.modificationIds); + checkNotNull(analysisRequest.workerVersion); // Transform the analysis UI/backend task format into a slightly different type for R5 workers. TravelTimeSurfaceTask task = new TravelTimeSurfaceTask(); diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 3da43be64..77049e049 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -31,6 +31,13 @@ public class AnalysisRequest { private static int MAX_ZOOM = 12; private static int MAX_GRID_CELLS = 5_000_000; + /** + * These three IDs are redundant, and just help reduce the number of database lookups necessary. + * The bundleId and modificationIds should be considered the definitive source of truth (regionId and projectId are + * implied by the bundleId and the modification Ids). Behavior is undefined if the API caller sends inconsistent + * information (a different regionId or projectId than the one the bundleId belongs to). + * TODO add null checks all over. + */ public String regionId; public String projectId; public String scenarioId; From 0dae9092155984b130be364ded8e636d1d78ab27 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 20 Sep 2021 21:07:00 +0800 Subject: [PATCH 143/187] remove projectId from worker tags workers do not stick to a single projectId, only to a single bundleId --- .../analysis/components/broker/WorkerTags.java | 15 +++++---------- .../analysis/controllers/BrokerController.java | 2 +- .../controllers/WorkerProxyController.java | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java b/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java index 274221877..8afb92ae6 100644 --- a/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java +++ b/src/main/java/com/conveyal/analysis/components/broker/WorkerTags.java @@ -5,8 +5,8 @@ /** * An immutable group of tags to be added to the worker instance to assist in usage analysis and cost breakdowns. - * These Strings are purely for categorization of workers and should not be used for other purposes, only passed through - * to the AWS SDK. + * These Strings are purely for categorization of workers and should not be used for other purposes, + * only passed through to the AWS SDK. */ public class WorkerTags { @@ -16,20 +16,16 @@ public class WorkerTags { /** A unique ID for the user (the user's email address). */ public final String user; - /** The UUID for the project. */ - public final String projectId; - /** The UUID for the region. */ public final String regionId; - public WorkerTags (UserPermissions userPermissions, String projectId, String regionId) { - this(userPermissions.accessGroup, userPermissions.email, projectId, regionId); + public WorkerTags (UserPermissions userPermissions, String regionId) { + this(userPermissions.accessGroup, userPermissions.email, regionId); } - public WorkerTags (String group, String user, String projectId, String regionId) { + public WorkerTags (String group, String user, String regionId) { this.group = group; this.user = user; - this.projectId = projectId; this.regionId = regionId; } @@ -37,7 +33,6 @@ public static WorkerTags fromRegionalAnalysis (RegionalAnalysis regionalAnalysis return new WorkerTags( regionalAnalysis.accessGroup, regionalAnalysis.createdBy, - regionalAnalysis.projectId, regionalAnalysis.regionId ); } diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index eddb3030d..39ecc1276 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -175,7 +175,7 @@ private Object singlePoint(Request request, Response response) { String address = broker.getWorkerAddress(workerCategory); if (address == null) { // There are no workers that can handle this request. Request some. - WorkerTags workerTags = new WorkerTags(userPermissions, analysisRequest.projectId, analysisRequest.regionId); + WorkerTags workerTags = new WorkerTags(userPermissions, analysisRequest.regionId); broker.createOnDemandWorkerInCategory(workerCategory, workerTags); // No workers exist. Kick one off and return "service unavailable". response.header("Retry-After", "30"); diff --git a/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java b/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java index 1a18cd1d2..2e9a36932 100644 --- a/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java +++ b/src/main/java/com/conveyal/analysis/controllers/WorkerProxyController.java @@ -61,7 +61,7 @@ private Object proxyGet (Request request, Response response) { if (address == null) { Bundle bundle = null; // There are no workers that can handle this request. Request one and ask the UI to retry later. - WorkerTags workerTags = new WorkerTags(UserPermissions.from(request), "anyProjectId", bundle.regionId); + WorkerTags workerTags = new WorkerTags(UserPermissions.from(request), bundle.regionId); broker.createOnDemandWorkerInCategory(workerCategory, workerTags); response.status(HttpStatus.ACCEPTED_202); response.header("Retry-After", "30"); From c3b472e79b41c30f1ac6ff47657937f3da8e9b27 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 20 Sep 2021 22:37:11 +0800 Subject: [PATCH 144/187] use normalized lowercase file format extension previously we were using the uppercase enum value names --- src/main/java/com/conveyal/analysis/models/AnalysisRequest.java | 1 - .../java/com/conveyal/analysis/models/SpatialDataSource.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 77049e049..1bfdabca2 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -36,7 +36,6 @@ public class AnalysisRequest { * The bundleId and modificationIds should be considered the definitive source of truth (regionId and projectId are * implied by the bundleId and the modification Ids). Behavior is undefined if the API caller sends inconsistent * information (a different regionId or projectId than the one the bundleId belongs to). - * TODO add null checks all over. */ public String regionId; public String projectId; diff --git a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java index ce94f0522..0be627908 100644 --- a/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java +++ b/src/main/java/com/conveyal/analysis/models/SpatialDataSource.java @@ -41,7 +41,7 @@ public SpatialDataSource (UserPermissions userPermissions, String name) { public SpatialDataSource () { } public FileStorageKey storageKey () { - return new FileStorageKey(DATASOURCES, this._id.toString(), fileFormat.toString()); + return new FileStorageKey(DATASOURCES, this._id.toString(), fileFormat.extension); } } From 117624b162690ce1e5ed64bffb3b139cfdfcbc9a Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 20 Sep 2021 23:33:29 +0800 Subject: [PATCH 145/187] prefetch shapefile sidecar files before conversion on newly started backends, we were fetching only the shp and not others --- .../derivation/AggregationAreaDerivation.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index c8b518099..a087ff716 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -9,7 +9,9 @@ import com.conveyal.analysis.models.SpatialDataSource; import com.conveyal.analysis.persistence.AnalysisCollection; import com.conveyal.analysis.persistence.AnalysisDB; +import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorage; +import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; import com.conveyal.r5.analyst.Grid; import com.conveyal.r5.analyst.progress.ProgressListener; @@ -98,6 +100,12 @@ private AggregationAreaDerivation (FileStorage fileStorage, AnalysisDB database, checkArgument(SHP.equals(spatialDataSource.fileFormat), "Currently, only shapefiles can be converted to aggregation areas."); + this.fileStorage = fileStorage; + // Do not retain AnalysisDB reference, but grab the collections we need. + // TODO cache AnalysisCollection instances and reuse? Are they threadsafe? + aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); + dataGroupCollection = database.getAnalysisCollection("dataGroups", DataGroup.class); + /* Implementation notes: Collecting all the Features to a List is a red flag for scalability, but the UnaryUnionOp used below (and the @@ -114,6 +122,12 @@ handled in a streaming fashion (in constant memory). */ File sourceFile; if (SHP.equals(spatialDataSource.fileFormat)) { + // On a newly started backend, we can't be sure any sidecar files are on the local filesystem. + // We may want to factor this out when we use shapefile DataSources in other derivations. + String baseName = spatialDataSource._id.toString(); + prefetchDataSource(baseName, "dbf"); + prefetchDataSource(baseName, "shx"); + prefetchDataSource(baseName, "prj"); sourceFile = fileStorage.getFile(spatialDataSource.storageKey()); ShapefileReader reader = null; try { @@ -135,12 +149,16 @@ handled in a streaming fashion (in constant memory). ); throw new DataSourceException(message); } - - this.fileStorage = fileStorage; - // Do not retain AnalysisDB reference, but grab the collections we need. - // TODO cache AnalysisCollection instances and reuse? Are they threadsafe? - aggregationAreaCollection = database.getAnalysisCollection("aggregationAreas", AggregationArea.class); - dataGroupCollection = database.getAnalysisCollection("dataGroups", DataGroup.class); + } + + /** Used primarily for shapefiles where we can't be sure whether all sidecar files have been synced locally. */ + private void prefetchDataSource (String baseName, String extension) { + FileStorageKey key = new FileStorageKey(FileCategory.DATASOURCES, baseName, extension); + // We need to clarify the FileStorage API on which calls cause the file to be synced locally, and whether these + // getFile tolerates getting files that do not exist. This may all become irrelevant if we use NFS. + if (fileStorage.exists(key)) { + fileStorage.getFile(key); + } } @Override From 6adbc993c02aaa026941b28935fab6b3d7ce9990 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 21 Sep 2021 00:02:07 +0800 Subject: [PATCH 146/187] try newest regional file name format and fall back This allows us to remove the old non-array fields in mongo records for older regional analysis results by copying their values into single-element arrays. We will still find older files by fallback. --- .../RegionalAnalysisController.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index d5819f25b..d52489e4c 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -277,23 +277,30 @@ private Object getRegionalResults (Request req, Response res) throws IOException if (!fileStorage.exists(singleCutoffFileStorageKey)) { // An accessibility grid for this particular cutoff has apparently never been extracted from the // regional results file before. Extract one and save it for future reuse. Older regional analyses - // may not have arrays allowing multiple cutoffs, percentiles, or destination pointsets. The + // did not have arrays allowing multiple cutoffs, percentiles, or destination pointsets. The // filenames of such regional accessibility results will not have a percentile or pointset ID. String multiCutoffKey; - if (analysis.travelTimePercentiles == null) { - // Oldest form of results, single-percentile, single grid. - multiCutoffKey = regionalAnalysisId + ".access"; - } else { - if (analysis.destinationPointSetIds == null) { - // Newer form of regional results: multi-percentile, single grid. - multiCutoffKey = String.format("%s_P%d.access", regionalAnalysisId, percentile); - } else { - // Newest form of regional results: multi-percentile, multi-grid. - multiCutoffKey = String.format("%s_%s_P%d.access", regionalAnalysisId, destinationPointSetId, percentile); + FileStorageKey multiCutoffFileStorageKey; + // Newest form of regional results: multi-percentile, multi-destination-grid. + multiCutoffKey = String.format("%s_%s_P%d.access", regionalAnalysisId, destinationPointSetId, percentile); + multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); + if (!fileStorage.exists(multiCutoffFileStorageKey)) { + LOG.warn("Falling back to older file name formats for regional results file: " + multiCutoffKey); + // Fall back to second-oldest form: multi-percentile, single destination grid. + checkArgument(analysis.destinationPointSetIds.length == 1); + multiCutoffKey = String.format("%s_P%d.access", regionalAnalysisId, percentile); + multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); + if (!fileStorage.exists(multiCutoffFileStorageKey)) { + // Fall back on oldest form of results, single-percentile, single-destination-grid. + checkArgument(analysis.travelTimePercentiles.length == 1); + multiCutoffKey = regionalAnalysisId + ".access"; + multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); + if (!fileStorage.exists(multiCutoffFileStorageKey)) { + throw new IllegalArgumentException("Cannot find original source regional analysis output."); + } } } LOG.debug("Single-cutoff grid {} not found on S3, deriving it from {}.", singleCutoffKey, multiCutoffKey); - FileStorageKey multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); InputStream multiCutoffInputStream = new FileInputStream(fileStorage.getFile(multiCutoffFileStorageKey)); Grid grid = new SelectingGridReducer(cutoffIndex).compute(multiCutoffInputStream); From 702ab3b01e79f76febd361edbf0ecb855d3f782b Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 21 Sep 2021 13:03:45 +0800 Subject: [PATCH 147/187] try-with-resources to close ShapefileReader --- .../datasource/derivation/AggregationAreaDerivation.java | 9 +++------ src/main/java/com/conveyal/r5/util/ShapefileReader.java | 4 +++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index a087ff716..172956c3c 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -129,14 +129,11 @@ handled in a streaming fashion (in constant memory). prefetchDataSource(baseName, "shx"); prefetchDataSource(baseName, "prj"); sourceFile = fileStorage.getFile(spatialDataSource.storageKey()); - ShapefileReader reader = null; - try { - reader = new ShapefileReader(sourceFile); + // Reading the shapefile into a list may actually take a moment, should this be done in the async part? + try (ShapefileReader reader = new ShapefileReader(sourceFile)) { finalFeatures = reader.wgs84Stream().collect(Collectors.toList()); - } catch (FactoryException | RuntimeException | IOException | TransformException e) { + } catch (Exception e) { throw new DataSourceException("Failed to load shapefile.", e); - } finally { - if (reader != null) reader.close(); } } else { // GeoJSON, GeoPackage etc. diff --git a/src/main/java/com/conveyal/r5/util/ShapefileReader.java b/src/main/java/com/conveyal/r5/util/ShapefileReader.java index 271866489..ade224ed2 100644 --- a/src/main/java/com/conveyal/r5/util/ShapefileReader.java +++ b/src/main/java/com/conveyal/r5/util/ShapefileReader.java @@ -26,6 +26,7 @@ import org.opengis.referencing.operation.MathTransform; import org.opengis.referencing.operation.TransformException; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -43,7 +44,7 @@ /** * Encapsulate Shapefile reading logic */ -public class ShapefileReader { +public class ShapefileReader implements Closeable { private final FeatureCollection features; private final DataStore store; private final FeatureSource source; @@ -134,6 +135,7 @@ public Envelope wgs84Bounds () throws IOException, TransformException { /** * Failure to call this will leave the shapefile locked, which may mess with future attempts to use it. */ + @Override public void close () { // Note that you also have to close the iterator, see iterator wrapper code above. store.dispose(); From 6063b283c4573fc40bfd1e4c8d685bdb804a855c Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 21 Sep 2021 13:04:14 +0800 Subject: [PATCH 148/187] validate that nameProperty exists and is not GEOM --- .../derivation/AggregationAreaDerivation.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java index 172956c3c..2bdd61369 100644 --- a/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java +++ b/src/main/java/com/conveyal/analysis/datasource/derivation/AggregationAreaDerivation.java @@ -3,6 +3,7 @@ import com.conveyal.analysis.AnalysisServerException; import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.datasource.DataSourceException; +import com.conveyal.analysis.datasource.SpatialAttribute; import com.conveyal.analysis.models.AggregationArea; import com.conveyal.analysis.models.DataGroup; import com.conveyal.analysis.models.DataSource; @@ -85,9 +86,6 @@ private AggregationAreaDerivation (FileStorage fileStorage, AnalysisDB database, zoom = parseZoom(req.queryParams("zoom")); mergePolygons = Boolean.parseBoolean(req.queryParams("mergePolygons")); checkNotNull(dataSourceId); - if (!mergePolygons) { - checkNotNull(nameProperty, "You must supply a nameProperty if mergePolygons is not true."); - } AnalysisCollection dataSourceCollection = database.getAnalysisCollection("dataSources", DataSource.class); @@ -100,6 +98,17 @@ private AggregationAreaDerivation (FileStorage fileStorage, AnalysisDB database, checkArgument(SHP.equals(spatialDataSource.fileFormat), "Currently, only shapefiles can be converted to aggregation areas."); + if (!mergePolygons) { + checkNotNull(nameProperty, "You must supply a nameProperty if mergePolygons is not true."); + SpatialAttribute sa = spatialDataSource.attributes.stream() + .filter(a -> a.name.equals(nameProperty)) + .findFirst().orElseThrow(() -> + new IllegalArgumentException("nameProperty does not exist: " + nameProperty)); + if (sa.type == SpatialAttribute.Type.GEOM) { + throw new IllegalArgumentException("nameProperty must be of type TEXT or NUMBER, not GEOM."); + } + } + this.fileStorage = fileStorage; // Do not retain AnalysisDB reference, but grab the collections we need. // TODO cache AnalysisCollection instances and reuse? Are they threadsafe? From 48558a42298d703950f482a559f56c1112fa75e3 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 21 Sep 2021 17:22:30 +0800 Subject: [PATCH 149/187] use same API path as UI for aggregation grid URLs Also return empty string instead of null in LocalFilesController and document why - when Spark framework sees a null response it thinks the path is not mapped to a controller. --- .../AggregationAreaController.java | 2 +- .../controllers/LocalFilesController.java | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java index 17b0f27f7..286fa5593 100644 --- a/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java +++ b/src/main/java/com/conveyal/analysis/controllers/AggregationAreaController.java @@ -107,7 +107,7 @@ private ObjectNode getAggregationAreaGridUrl (Request req, Response res) { @Override public void registerEndpoints (spark.Service sparkService) { sparkService.get("/api/aggregationArea", this::getAggregationAreas, toJson); - sparkService.get("/api/aggregationArea/:_id/gridUrl", this::getAggregationAreaGridUrl, toJson); + sparkService.get("/api/aggregationArea/:_id", this::getAggregationAreaGridUrl, toJson); sparkService.post("/api/aggregationArea", this::createAggregationAreas, toJson); } diff --git a/src/main/java/com/conveyal/analysis/controllers/LocalFilesController.java b/src/main/java/com/conveyal/analysis/controllers/LocalFilesController.java index ed7d5b38d..1541773ad 100644 --- a/src/main/java/com/conveyal/analysis/controllers/LocalFilesController.java +++ b/src/main/java/com/conveyal/analysis/controllers/LocalFilesController.java @@ -5,6 +5,7 @@ import com.conveyal.file.FileStorageFormat; import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; +import com.conveyal.file.LocalFileStorage; import spark.Request; import spark.Response; import spark.Service; @@ -20,14 +21,13 @@ */ public class LocalFilesController implements HttpController { - // Something feels whack here, this should more specifically be a LocalFileStorage - private final FileStorage fileStorage; + private final LocalFileStorage fileStorage; public LocalFilesController (FileStorage fileStorage) { - this.fileStorage = fileStorage; + this.fileStorage = (LocalFileStorage) fileStorage; } - private InputStream getFile (Request req, Response res) throws Exception { + private Object getFile (Request req, Response res) throws Exception { String filename = req.splat()[0]; FileCategory category = FileCategory.valueOf(req.params("category").toUpperCase(Locale.ROOT)); FileStorageKey key = new FileStorageKey(category, filename); @@ -35,15 +35,21 @@ private InputStream getFile (Request req, Response res) throws Exception { FileStorageFormat format = FileStorageFormat.fromFilename(filename); res.type(format.mimeType); - // If the content-encoding is set to gzip, Spark automatically gzips the response. This mangles data - // that was already gzipped. Therefore, check if it's gzipped and pipe directly to the raw OutputStream. + // If the content-encoding is set to gzip, Spark automatically gzips the response. This double-gzips anything + // that was already gzipped. Some of our files are already gzipped, and we rely on the the client browser to + // decompress them upon receiving them. Therefore, when serving a file that's already gzipped we bypass Spark, + // piping it directly to the raw Jetty OutputStream. As soon as transferFromFileTo completes it closes the + // output stream, which completes the HTTP response to the client. We must then return something to Spark. We + // can't return null because Spark will spew errors about the endpoint being "not mapped" and try to replace + // the response with a 404, so we return an empty String. res.header("Content-Encoding", "gzip"); if (FileUtils.isGzip(file)) { // TODO Trace in debug: how does this actually work? // Verify what this is transferring into - a buffer? In another reading thread? // Is Jetty ServletOutputStream implementation automatically threaded or buffered? + // It appears to be buffered because the response has a Content-Length header. FileUtils.transferFromFileTo(file, res.raw().getOutputStream()); - return null; + return ""; } else { return FileUtils.getInputStream(file); } From 6d74ad6a1bd237bcebc7ef73f4063073c67a8d16 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 21 Sep 2021 21:23:35 +0800 Subject: [PATCH 150/187] initialize variables immediately on declaration --- .../analysis/controllers/RegionalAnalysisController.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index d52489e4c..3bc3bae3f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -279,11 +279,9 @@ private Object getRegionalResults (Request req, Response res) throws IOException // regional results file before. Extract one and save it for future reuse. Older regional analyses // did not have arrays allowing multiple cutoffs, percentiles, or destination pointsets. The // filenames of such regional accessibility results will not have a percentile or pointset ID. - String multiCutoffKey; - FileStorageKey multiCutoffFileStorageKey; - // Newest form of regional results: multi-percentile, multi-destination-grid. - multiCutoffKey = String.format("%s_%s_P%d.access", regionalAnalysisId, destinationPointSetId, percentile); - multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); + // First try the newest form of regional results: multi-percentile, multi-destination-grid. + String multiCutoffKey = String.format("%s_%s_P%d.access", regionalAnalysisId, destinationPointSetId, percentile); + FileStorageKey multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); if (!fileStorage.exists(multiCutoffFileStorageKey)) { LOG.warn("Falling back to older file name formats for regional results file: " + multiCutoffKey); // Fall back to second-oldest form: multi-percentile, single destination grid. From dcdd7b960354c192794bdec0fa8620e16a284905 Mon Sep 17 00:00:00 2001 From: ansons Date: Thu, 23 Sep 2021 12:49:58 -0400 Subject: [PATCH 151/187] mark files read-only on non-POSIX systems --- src/main/java/com/conveyal/file/LocalFileStorage.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/file/LocalFileStorage.java b/src/main/java/com/conveyal/file/LocalFileStorage.java index 04bd8e8a9..1c7ff5dae 100644 --- a/src/main/java/com/conveyal/file/LocalFileStorage.java +++ b/src/main/java/com/conveyal/file/LocalFileStorage.java @@ -9,7 +9,6 @@ import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.PosixFilePermission; import java.util.Set; import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; @@ -62,7 +61,15 @@ public void moveIntoStorage(FileStorageKey key, File sourceFile) { sourceFile.getName()); } // Set the file to be read-only and accessible only by the current user. - Files.setPosixFilePermissions(storedFile.toPath(), Set.of(OWNER_READ)); + try { + Files.setPosixFilePermissions(storedFile.toPath(), Set.of(OWNER_READ)); + } catch (UnsupportedOperationException e) { + LOG.info("Could not restrict permissions on {} to POSIX OWNER_READ (probably because OS is not POSIX " + + "compatible)", sourceFile.getName()); + if (storedFile.setReadOnly()) LOG.info("Marked {} read-only via alternative setReadOnly() method", + sourceFile.getName()); + } + } catch (IOException e) { throw new RuntimeException(e); } From 5d4bf9b8bad79d461794030c84344cacd7df79d1 Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Thu, 23 Sep 2021 15:18:59 -0400 Subject: [PATCH 152/187] Update UI version in Cypress tests --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index eacc46618..c9aa49402 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: 68040a2e83039539d47b43fc204cfadcdb795cb5 + ref: f0080035c1562ebe940d5e6a444420f423f1c6a7 path: ui - uses: actions/checkout@v2 with: From edfd24e8670cf9e0cbf2b1798614c3e5fd14c052 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 28 Sep 2021 14:10:07 +0800 Subject: [PATCH 153/187] use cross-platform setWritable(bool,bool) --- .../java/com/conveyal/file/LocalFileStorage.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/file/LocalFileStorage.java b/src/main/java/com/conveyal/file/LocalFileStorage.java index 1c7ff5dae..44e8141bb 100644 --- a/src/main/java/com/conveyal/file/LocalFileStorage.java +++ b/src/main/java/com/conveyal/file/LocalFileStorage.java @@ -53,23 +53,17 @@ public void moveIntoStorage(FileStorageKey key, File sourceFile) { Files.move(sourceFile.toPath(), storedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (FileSystemException e) { // The default Windows filesystem (NTFS) does not unlock memory-mapped files, so certain files (e.g. - // mapdb) cannot be moved or deleted. This workaround may cause temporary files to accumulate, but it - // should not be triggered for default Linux filesystems (ext). + // mapdb Write Ahead Log) cannot be moved or deleted. This workaround may cause temporary files + // to accumulate, but it should not be triggered for default Linux filesystems (ext). // See https://github.com/jankotek/MapDB/issues/326 Files.copy(sourceFile.toPath(), storedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - LOG.info("Could not move {} because of FileSystem restrictions (probably NTFS). Copying instead.", + LOG.info("Could not move {} because of FileSystem restrictions (probably NTFS). Copied instead.", sourceFile.getName()); } // Set the file to be read-only and accessible only by the current user. - try { - Files.setPosixFilePermissions(storedFile.toPath(), Set.of(OWNER_READ)); - } catch (UnsupportedOperationException e) { - LOG.info("Could not restrict permissions on {} to POSIX OWNER_READ (probably because OS is not POSIX " + - "compatible)", sourceFile.getName()); - if (storedFile.setReadOnly()) LOG.info("Marked {} read-only via alternative setReadOnly() method", - sourceFile.getName()); + if (!storedFile.setWritable(false, false)) { + LOG.error("Could not restrict permissions on {} to read-only by owner: ", sourceFile.getName()); } - } catch (IOException e) { throw new RuntimeException(e); } From e2709d65e3cdc73a18363f05919d3bbcb0b9a0f5 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 28 Sep 2021 14:18:41 +0800 Subject: [PATCH 154/187] replace magic number with symbolic quasi-constant fixes #750 --- .../com/conveyal/r5/analyst/cluster/PathResult.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/PathResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/PathResult.java index edf8ad057..dd268ce95 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/PathResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/PathResult.java @@ -32,6 +32,14 @@ public class PathResult { + /** + * The maximum number of destinations for which we'll generate detailed path information in a single request. + * Detailed path information was added on to the original design, which returned a simple grid of travel times. + * These results are returned to the backend over an HTTP API so we don't want to risk making them too huge. + * This could be set to a higher number in cases where you know the result return channel can handle the size. + */ + public static int maxDestinations = 5000; + private final int nDestinations; /** * Array with one entry per destination. Each entry is a map from a "path template" to the associated iteration @@ -62,8 +70,8 @@ public PathResult(AnalysisWorkerTask task, TransitLayer transitLayer) { // In regional analyses, return paths to all destinations nDestinations = task.nTargetsPerOrigin(); // This limitation reflects the initial design, for use with freeform pointset destinations - if (nDestinations > 5000) { - throw new UnsupportedOperationException("Path results are limited to 5000 destinations"); + if (nDestinations > maxDestinations) { + throw new UnsupportedOperationException("Number of detailed path destinations exceeds limit of " + maxDestinations); } } iterationsForPathTemplates = new Multimap[nDestinations]; From c41399ebb8d23dd91a8f79e1023430de0d90111a Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 28 Sep 2021 17:10:41 +0800 Subject: [PATCH 155/187] make utility methods static, elevate visibility --- .../java/com/conveyal/r5/transit/TransportNetworkCache.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java index 424f47b71..916ea7837 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java @@ -140,15 +140,15 @@ public synchronized TransportNetwork getNetworkForScenario (String networkId, St return scenarioNetwork; } - private String getScenarioFilename(String networkId, String scenarioId) { + public static String getScenarioFilename (String networkId, String scenarioId) { return String.format("%s_%s.json", networkId, scenarioId); } - private String getR5NetworkFilename(String networkId) { + private static String getR5NetworkFilename (String networkId) { return String.format("%s_%s.dat", networkId, KryoNetworkSerializer.NETWORK_FORMAT_VERSION); } - private FileStorageKey getR5NetworkFileStorageKey (String networkId) { + private static FileStorageKey getR5NetworkFileStorageKey (String networkId) { return new FileStorageKey(BUNDLES, getR5NetworkFilename(networkId)); } From e29c86352fa5dfa906059d1395a7567ff8fe1f09 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 28 Sep 2021 17:11:47 +0800 Subject: [PATCH 156/187] allow fetching scenario from filestorage not mongo some of these scenarios are huge and we're considering removing them from the database --- .../RegionalAnalysisController.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 3bc3bae3f..210c62b1f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -20,6 +20,7 @@ import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.cluster.RegionalTask; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.primitives.Ints; import com.mongodb.QueryBuilder; import gnu.trove.list.array.TIntArrayList; @@ -42,7 +43,9 @@ import java.util.zip.GZIPOutputStream; import static com.conveyal.analysis.util.JsonUtil.toJson; +import static com.conveyal.file.FileCategory.BUNDLES; import static com.conveyal.file.FileCategory.RESULTS; +import static com.conveyal.r5.transit.TransportNetworkCache.getScenarioFilename; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; @@ -533,11 +536,28 @@ private RegionalAnalysis createRegionalAnalysis (Request req, Response res) thro return regionalAnalysis; } - private RegionalAnalysis updateRegionalAnalysis(Request request, Response response) throws IOException { + private RegionalAnalysis updateRegionalAnalysis (Request request, Response response) throws IOException { RegionalAnalysis regionalAnalysis = JsonUtil.objectMapper.readValue(request.body(), RegionalAnalysis.class); return Persistence.regionalAnalyses.updateByUserIfPermitted(regionalAnalysis, UserPermissions.from(request)); } + /** + * Return a JSON-wrapped URL for the file in FileStorage containing the JSON representation of the scenario for + * the given regional analysis. + */ + private JsonNode getScenarioJsonUrl (Request request, Response response) { + RegionalAnalysis regionalAnalysis = Persistence.regionalAnalyses + .findByIdIfPermitted(request.params("_id"), UserPermissions.from(request)); + // In the persisted objects, regionalAnalysis.scenarioId seems to be null. Get it from the embedded request. + final String networkId = regionalAnalysis.bundleId; + final String scenarioId = regionalAnalysis.request.scenarioId; + checkNotNull(networkId, "RegionalAnalysis did not contain a network ID."); + checkNotNull(scenarioId, "RegionalAnalysis did not contain an embedded request with scenario ID."); + String scenarioUrl = fileStorage.getURL( + new FileStorageKey(BUNDLES, getScenarioFilename(regionalAnalysis.bundleId, scenarioId))); + return JsonUtil.objectNode().put("url", scenarioUrl); + } + @Override public void registerEndpoints (spark.Service sparkService) { sparkService.path("/api/region", () -> { @@ -549,6 +569,7 @@ public void registerEndpoints (spark.Service sparkService) { sparkService.get("/:_id", this::getRegionalAnalysis); sparkService.get("/:_id/grid/:format", this::getRegionalResults); sparkService.get("/:_id/csv/:resultType", this::getCsvResults); + sparkService.get("/:_id/scenarioJsonUrl", this::getScenarioJsonUrl); sparkService.delete("/:_id", this::deleteRegionalAnalysis, toJson); sparkService.post("", this::createRegionalAnalysis, toJson); sparkService.put("/:_id", this::updateRegionalAnalysis, toJson); From 564b0ac328b384e75d2481bf6600f33992def38f Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Tue, 28 Sep 2021 17:33:13 -0400 Subject: [PATCH 157/187] Update README Including link to OTP docs, fixes #575 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fced34b93..9db4bd806 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## R5: Rapid Realistic Routing on Real-world and Reimagined networks R5 is the routing engine for [Conveyal](https://www.conveyal.com/learn), a web-based system that allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. See the [Conveyal user manual](https://docs.conveyal.com/) for more information. -We refer to the routing method as "realistic" because it works by planning many trips at different departure times in a time window, which better reflects how people use transportation system than planning a single trip at an exact departure time. R5 handles both scheduled public transit and headway-based lines, using novel methods to characterize variation and uncertainty in travel times. +We refer to the routing method as "realistic" because it works by planning door-to-door trips at many different departure times in a time window, which better reflects how people use transportation systems than planning a single trip at an exact departure time. R5 handles both scheduled public transit and headway-based lines, using novel methods to characterize variation and uncertainty in travel times. It is designed for one-to-many and many-to-many travel-time calculations used in access indicators, offering substantially better performance than repeated calls to older tools that provide one-to-one routing results. For a comparison with OpenTripPlanner, see [this background](http://docs.opentripplanner.org/en/latest/Version-Comparison/#commentary-on-otp1-features-removed-from-otp2). We say "Real-world and Reimagined" networks because R5's networks are built from widely available open OSM and GTFS data describing baseline transportation systems, but R5 includes a system for applying light-weight patches to those networks for immediate, interactive scenario comparison. From b9729a4a1cc4e0a2ccaead9350d2bb525b26642e Mon Sep 17 00:00:00 2001 From: ansons Date: Wed, 29 Sep 2021 18:33:00 -0400 Subject: [PATCH 158/187] Use new DATASOURCES category in pickup and congestion polygon modifications --- src/main/java/com/conveyal/file/FileCategory.java | 2 +- src/main/java/com/conveyal/file/FileStorageKey.java | 2 +- .../r5/analyst/scenario/IndexedPolygonCollection.java | 7 +++---- .../com/conveyal/r5/analyst/scenario/RoadCongestion.java | 9 ++------- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/file/FileCategory.java b/src/main/java/com/conveyal/file/FileCategory.java index 90fc41068..02ac8c856 100644 --- a/src/main/java/com/conveyal/file/FileCategory.java +++ b/src/main/java/com/conveyal/file/FileCategory.java @@ -8,7 +8,7 @@ */ public enum FileCategory { - BUNDLES, GRIDS, RESULTS, DATASOURCES, POLYGONS, TAUI; + BUNDLES, GRIDS, RESULTS, DATASOURCES, TAUI; /** @return a String for the directory or sub-bucket name containing all files in this category. */ public String directoryName () { diff --git a/src/main/java/com/conveyal/file/FileStorageKey.java b/src/main/java/com/conveyal/file/FileStorageKey.java index 22d504f3c..e9a1c5244 100644 --- a/src/main/java/com/conveyal/file/FileStorageKey.java +++ b/src/main/java/com/conveyal/file/FileStorageKey.java @@ -4,7 +4,7 @@ * A unique identifier for a file within a namespace drawn from an enum of known categories. * This maps to a subdirectory and filename in local storage, and a bucket and object key in S3-style cloud storage. * While keeping stored files in multiple distinct categories, this avoids passing around a lot of directory/bucket - * names as strings, and avoids mistakes where such strings are mismatched accross different function calls. + * names as strings, and avoids mistakes where such strings are mismatched across different function calls. */ public class FileStorageKey { diff --git a/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java b/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java index 2e488a366..d61f6066a 100644 --- a/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java +++ b/src/main/java/com/conveyal/r5/analyst/scenario/IndexedPolygonCollection.java @@ -19,13 +19,12 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.conveyal.file.FileCategory.POLYGONS; +import static com.conveyal.file.FileCategory.DATASOURCES; /** * This is an abstraction for the polygons used to configure the road congestion modification type and the ride hailing @@ -107,8 +106,8 @@ public IndexedPolygonCollection ( } public void loadFromS3GeoJson() throws Exception { - // FIXME this needs to be adapted to new SpatialDataSource. How will we handle .gz data? - File polygonInputFile = WorkerComponents.fileStorage.getFile(new FileStorageKey(POLYGONS, polygonLayer)); + // FIXME How will we handle .gz data? + File polygonInputFile = WorkerComponents.fileStorage.getFile(new FileStorageKey(DATASOURCES, polygonLayer)); GeoJSONDataStore dataStore = new GeoJSONDataStore(polygonInputFile); SimpleFeatureSource featureSource = dataStore.getFeatureSource(); FeatureCollection featureCollection = featureSource.getFeatures(); diff --git a/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java b/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java index d35559717..86d048c49 100644 --- a/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java +++ b/src/main/java/com/conveyal/r5/analyst/scenario/RoadCongestion.java @@ -1,9 +1,7 @@ package com.conveyal.r5.analyst.scenario; import com.conveyal.analysis.components.WorkerComponents; -import com.conveyal.file.FileCategory; import com.conveyal.file.FileStorageKey; -import com.conveyal.r5.analyst.cluster.AnalysisWorker; import com.conveyal.r5.streets.EdgeStore; import com.conveyal.r5.transit.TransportNetwork; import com.conveyal.r5.util.ExceptionUtils; @@ -28,11 +26,9 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.InputStream; import java.util.List; -import java.util.zip.GZIPInputStream; -import static com.conveyal.file.FileCategory.POLYGONS; +import static com.conveyal.file.FileCategory.DATASOURCES; /** * To simulate traffic congestion, apply a slow-down (or speed-up) factor to roads, according to attributes of polygon @@ -113,9 +109,8 @@ public boolean resolve (TransportNetwork network) { // and errors can all be easily recorded and bubbled back up to the UI. // Polygon should only need to be fetched once when the scenario is applied, then the resulting network is cached. // this.features = polygonLayerCache.getPolygonFeatureCollection(this.polygonLayer); - // TODO integrate this with new SpatialDataSource system try { - File polygonInputFile = WorkerComponents.fileStorage.getFile(new FileStorageKey(POLYGONS, polygonLayer)); + File polygonInputFile = WorkerComponents.fileStorage.getFile(new FileStorageKey(DATASOURCES, polygonLayer)); GeoJSONDataStore dataStore = new GeoJSONDataStore(polygonInputFile); SimpleFeatureSource featureSource = dataStore.getFeatureSource(); FeatureCollection featureCollection = featureSource.getFeatures(); From 30e49fc0222a7a4266f22a55a3aadcc6fe442677 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 15 Oct 2021 00:08:11 +0800 Subject: [PATCH 159/187] impose feedId when rebuilding GTFS MapDBs These are usually built in bundleController and cached. When they were rebuilt by gtfsCache, the feedId was falling back on the filename (which was a bundle-scoped feedId rather than just the feedId) or even on the feedId declared by the feed itself, not known to be unique. --- src/main/java/com/conveyal/gtfs/GTFSCache.java | 18 +++++++++++++----- src/main/java/com/conveyal/gtfs/GTFSFeed.java | 18 ++++++++++-------- .../conveyal/r5/transit/TransportNetwork.java | 10 +++++++++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/GTFSCache.java b/src/main/java/com/conveyal/gtfs/GTFSCache.java index cc945b9ef..0cb2168d2 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSCache.java +++ b/src/main/java/com/conveyal/gtfs/GTFSCache.java @@ -14,6 +14,7 @@ import java.util.concurrent.TimeUnit; import static com.conveyal.file.FileCategory.BUNDLES; +import static com.google.common.base.Preconditions.checkState; /** * Cache for GTFSFeed objects, a disk-backed (MapDB) representation of data from one GTFS feed. The source GTFS @@ -99,24 +100,31 @@ public FileStorageKey getFileKey (String id, String extension) { } /** This method should only ever be called by the cache loader. */ - private @Nonnull GTFSFeed retrieveAndProcessFeed(String id) throws GtfsLibException { - FileStorageKey dbKey = getFileKey(id, "db"); - FileStorageKey dbpKey = getFileKey(id, "db.p"); + private @Nonnull GTFSFeed retrieveAndProcessFeed(String bundeScopedFeedId) throws GtfsLibException { + FileStorageKey dbKey = getFileKey(bundeScopedFeedId, "db"); + FileStorageKey dbpKey = getFileKey(bundeScopedFeedId, "db.p"); if (fileStorage.exists(dbKey) && fileStorage.exists(dbpKey)) { // Ensure both MapDB files are local, pulling them down from remote storage as needed. fileStorage.getFile(dbKey); fileStorage.getFile(dbpKey); return GTFSFeed.reopenReadOnly(fileStorage.getFile(dbKey)); } - FileStorageKey zipKey = getFileKey(id, "zip"); + FileStorageKey zipKey = getFileKey(bundeScopedFeedId, "zip"); if (!fileStorage.exists(zipKey)) { throw new GtfsLibException("Original GTFS zip file could not be found: " + zipKey); } + // This code path is rarely run because we usually pre-build GTFS MapDBs in bundleController and cache them. + // This will only be run when the resultant MapDB has been deleted or is otherwise unavailable. LOG.debug("Building or rebuilding MapDB from original GTFS ZIP file at {}...", zipKey); try { File tempDbFile = FileUtils.createScratchFile("db"); File tempDbpFile = new File(tempDbFile.getAbsolutePath() + ".p"); - GTFSFeed.newFileFromGtfs(tempDbFile, fileStorage.getFile(zipKey)); + // An unpleasant hack since we do not have separate references to the GTFS ID and Bundle ID here, + // only a concatenation of the two joined with an underscore. We have to force-override feed ID because + // references to its contents (e.g. in scenarios) are scoped only by the feed ID not the bundle ID. + final String[] feedAndBundleId = bundeScopedFeedId.split("_"); + checkState(feedAndBundleId.length == 2, "Expected underscore-joined feedId and bundleId."); + GTFSFeed.newFileFromGtfs(tempDbFile, fileStorage.getFile(zipKey), feedAndBundleId[0]); // The DB file should already be closed and flushed to disk. // Put the DB and DB.p files in local cache, and mirror to remote storage if configured. fileStorage.moveIntoStorage(dbKey, tempDbFile); diff --git a/src/main/java/com/conveyal/gtfs/GTFSFeed.java b/src/main/java/com/conveyal/gtfs/GTFSFeed.java index 77c6c3bac..c8052c911 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSFeed.java +++ b/src/main/java/com/conveyal/gtfs/GTFSFeed.java @@ -187,6 +187,8 @@ public class GTFSFeed implements Cloneable, Closeable { * Interestingly, all references are resolvable when tables are loaded in alphabetical order. * * @param zip the source ZIP file to load, which will be closed when done loading. + * @param fid the feedId to be set on the feed. If null, any feedId declared in the feed will be used, falling back + * on the filename without any .zip extension. */ public void loadFromFile(ZipFile zip, String fid) throws Exception { if (this.loaded) throw new UnsupportedOperationException("Attempt to load GTFS into existing database"); @@ -259,10 +261,6 @@ else if (feedId == null || feedId.isEmpty()) { LOG.info("Detected {} errors in feed.", errors.size()); } - public void loadFromFile(ZipFile zip) throws Exception { - loadFromFile(zip, null); - } - public void toFile (String file) { try { File out = new File(file); @@ -838,14 +836,16 @@ public static GTFSFeed reopenReadOnly (File file) { * Create a new DB file and load the specified GTFS ZIP into it. The resulting writable feed object is not returned * and must be reopened for subsequent read-only access. * @param dbFile the new file in which to store the database, or null to use a temporary file + * @param feedId the feedId to be set on the feed. If null, any feedId declared in the feed will be used, falling + * back on the filename without its .zip extension. */ - public static void newFileFromGtfs (File dbFile, File gtfsFile) { + public static void newFileFromGtfs (File dbFile, File gtfsFile, String feedId) { if (gtfsFile == null || !gtfsFile.exists()) { throw new GtfsLibException("Cannot load from GTFS feed, file does not exist."); } try { GTFSFeed feed = newWritableFile(dbFile); - feed.loadFromFile(new ZipFile(gtfsFile)); + feed.loadFromFile(new ZipFile(gtfsFile), feedId); feed.close(); } catch (Exception e) { throw new GtfsLibException("Cannot load GTFS from feed ZIP.", e); @@ -874,7 +874,7 @@ public static GTFSFeed writableTempFileFromGtfs (String file) { GTFSFeed feed = new GTFSFeed(null, false); try { ZipFile zip = new ZipFile(file); - feed.loadFromFile(zip); + feed.loadFromFile(zip, null); zip.close(); return feed; } catch (Exception e) { @@ -883,11 +883,13 @@ public static GTFSFeed writableTempFileFromGtfs (String file) { } } + // NOTE the feedId within the MapDB created here will be the one declared by the feed or based on its filename. + // This method makes no effort to impose the more unique feed IDs created by the Analysis backend. public static GTFSFeed readOnlyTempFileFromGtfs (String fileName) { try { File tempFile = File.createTempFile("com.conveyal.gtfs.", ".db"); tempFile.deleteOnExit(); - GTFSFeed.newFileFromGtfs(tempFile, new File(fileName)); + GTFSFeed.newFileFromGtfs(tempFile, new File(fileName), null); return GTFSFeed.reopenReadOnly(tempFile); } catch (Exception e) { throw new GtfsLibException("Error loading GTFS.", e); diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetwork.java b/src/main/java/com/conveyal/r5/transit/TransportNetwork.java index 358c24dc0..5927a15e4 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetwork.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetwork.java @@ -113,6 +113,9 @@ public void rebuildTransientIndexes() { * doesn't really matter, particularly for analytics. Loading all he feeds into memory simulataneously shouldn't be * so bad with mapdb-based feeds, but it's still not great (due to instance caching, off heap allocations etc.) * Therefore we create the feeds within a stream which loads them one by one on demand. + * + * NOTE the feedId of the gtfs feeds loaded here will be the ones declared by the feeds or based on their filenames. + * This method makes no effort to impose the more unique feed IDs created by the Analysis backend. */ public static TransportNetwork fromFiles ( String osmSourceFile, @@ -194,7 +197,12 @@ public static TransportNetwork fromInputs (TNBuilderConfig tnBuilderConfig, OSM return transportNetwork; } - /** Scan a directory detecting all the files that are network inputs, then build a network from those files. */ + /** + * Scan a directory detecting all the files that are network inputs, then build a network from those files. + * + * NOTE the feedId of the gtfs feeds laoded here will be the ones declared by the feeds or based on their filenames. + * This method makes no effort to impose the more unique feed IDs created by the Analysis backend. + */ public static TransportNetwork fromDirectory (File directory) throws DuplicateFeedException { File osmFile = null; List gtfsFiles = new ArrayList<>(); From 54bbaf1e544ce8fcc44a418d740f33553d70ace6 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 15 Oct 2021 10:06:18 +0800 Subject: [PATCH 160/187] correct log messages when injecting feedId --- src/main/java/com/conveyal/gtfs/GTFSFeed.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/GTFSFeed.java b/src/main/java/com/conveyal/gtfs/GTFSFeed.java index c8052c911..6a742cc24 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSFeed.java +++ b/src/main/java/com/conveyal/gtfs/GTFSFeed.java @@ -209,13 +209,11 @@ public void loadFromFile(ZipFile zip, String fid) throws Exception { // maybe we should just point to the feed object itself instead of its ID, and null out its stoptimes map after loading if (fid != null) { feedId = fid; - LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID - } - else if (feedId == null || feedId.isEmpty()) { + LOG.info("Forcing feedId in MapDB to supplied string: {}.", feedId); + } else if (feedId == null || feedId.isEmpty()) { feedId = new File(zip.getName()).getName().replaceAll("\\.zip$", ""); - LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID - } - else { + LOG.info("No feedId supplied and feed does not declare one. Using file name {}.", feedId); + } else { LOG.info("Feed ID is '{}'.", feedId); } From 84bef0c4d5efff5f7cab0d62857ebd72157e3200 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Tue, 19 Oct 2021 12:53:00 +0800 Subject: [PATCH 161/187] Add max age header Add "Access-Control-Max-Age" header set to one day to all preflight requests. This allows browsers to cache the "Options" preflight request instead of sending it for each HTTP request. --- src/main/java/com/conveyal/analysis/components/HttpApi.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/components/HttpApi.java b/src/main/java/com/conveyal/analysis/components/HttpApi.java index 2400391d6..6b188c290 100644 --- a/src/main/java/com/conveyal/analysis/components/HttpApi.java +++ b/src/main/java/com/conveyal/analysis/components/HttpApi.java @@ -121,7 +121,10 @@ private spark.Service configureSparkService () { // Handle CORS preflight requests (which are OPTIONS requests). // See comment above about Access-Control-Allow-Origin sparkService.options("/*", (req, res) -> { + // Cache the preflight response for up to one day (the maximum allowed by browsers) + res.header("Access-Control-Max-Age", "86400"); res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"); + // Allowing credentials is necessary to send an Authorization header res.header("Access-Control-Allow-Credentials", "true"); res.header("Access-Control-Allow-Headers", "Accept,Authorization,Content-Type,Origin," + "X-Requested-With,Content-Length,X-Conveyal-Access-Group" From 31aa20657d84f39b806c65ccc128439b3afaa202 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 19 Oct 2021 10:25:46 -0400 Subject: [PATCH 162/187] fix: actually throw exception --- .../java/com/conveyal/analysis/datasource/DataSourceUtil.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java index 5ee6ee8b4..d85b883e9 100644 --- a/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java +++ b/src/main/java/com/conveyal/analysis/datasource/DataSourceUtil.java @@ -1,7 +1,5 @@ package com.conveyal.analysis.datasource; -import com.conveyal.analysis.AnalysisServerException; - import com.conveyal.file.FileStorageFormat; import com.google.common.collect.Sets; import org.apache.commons.fileupload.FileItem; @@ -100,7 +98,7 @@ private static Set extractFileExtensions (List fileItems) { String fileName = fileItem.getName(); String extension = FilenameUtils.getExtension(fileName); if (extension.isEmpty()) { - new DataSourceException("Filename has no extension: " + fileName); + throw new DataSourceException("Filename has no extension: " + fileName); } fileExtensions.add(extension.toLowerCase(Locale.ROOT)); } From 501e5b44c24fb497c8178f44ef6499ae15b17418 Mon Sep 17 00:00:00 2001 From: ansons Date: Tue, 19 Oct 2021 10:29:22 -0400 Subject: [PATCH 163/187] Filter GTFSController responses to only include stops Not generic nodes etc. which may lack lat/lon values --- .../analysis/controllers/GtfsController.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/GtfsController.java b/src/main/java/com/conveyal/analysis/controllers/GtfsController.java index 151345c25..1552e2314 100644 --- a/src/main/java/com/conveyal/analysis/controllers/GtfsController.java +++ b/src/main/java/com/conveyal/analysis/controllers/GtfsController.java @@ -150,20 +150,29 @@ static class StopApiResponse extends BaseApiResponse { lon = stop.stop_lon; } } - + /** + * Return StopApiResponse values for GTFS stops (location_type = 0) in a single feed + */ private List getAllStopsForOneFeed(Request req, Response res) { addImmutableResponseHeader(res); GTFSFeed feed = getFeedFromRequest(req); - return feed.stops.values().stream().map(StopApiResponse::new).collect(Collectors.toList()); + return feed.stops.values().stream().filter(s -> s.location_type == 0) + .map(StopApiResponse::new).collect(Collectors.toList()); } + /** + * Groups the feedId and stops (location_type = 0; not parent stations, entrances/exits, generic nodes, etc.) for a + * given GTFS feed + */ static class FeedGroupStopsApiResponse { public final String feedId; public final List stops; FeedGroupStopsApiResponse(GTFSFeed feed) { this.feedId = feed.feedId; - this.stops = feed.stops.values().stream().map(StopApiResponse::new).collect(Collectors.toList()); + this.stops = + feed.stops.values().stream().filter(s -> s.location_type == 0). + map(StopApiResponse::new).collect(Collectors.toList()); } } From ab05f16f3ab11fbc59146f83f2fcc724396d3d98 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 25 Oct 2021 16:21:41 +0800 Subject: [PATCH 164/187] Override access mode time limit with global limit Assertions elsewhere assume no travel time will exceed the request.maxTripDurationMinutes limit. This appears to only be an issue when cutoff times are lower than access mode-specifc travel time limits --- .../com/conveyal/r5/analyst/TravelTimeComputer.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/TravelTimeComputer.java b/src/main/java/com/conveyal/r5/analyst/TravelTimeComputer.java index f7ae15e08..b9f20a701 100644 --- a/src/main/java/com/conveyal/r5/analyst/TravelTimeComputer.java +++ b/src/main/java/com/conveyal/r5/analyst/TravelTimeComputer.java @@ -151,12 +151,17 @@ public OneOriginResult computeTravelTimes() { // egress end of the trip. We could probably reuse a method for both (getTravelTimesFromPoint). // Note: Access searches (which minimize travel time) are asymmetric with the egress cost tables (which // often minimize distance to allow reuse at different speeds). + // Preserve past behavior: only apply bike or walk time limits when those modes are used to access transit. - if (request.hasTransit()) { - sr.timeLimitSeconds = request.getMaxTimeSeconds(accessMode); - } else { - sr.timeLimitSeconds = request.maxTripDurationMinutes * FastRaptorWorker.SECONDS_PER_MINUTE; + // The overall time limit specified in the request may further decrease that mode-specific limit. + { + int limitSeconds = request.maxTripDurationMinutes * FastRaptorWorker.SECONDS_PER_MINUTE; + if (request.hasTransit()) { + limitSeconds = Math.min(limitSeconds, request.getMaxTimeSeconds(accessMode)); + } + sr.timeLimitSeconds = limitSeconds; } + // Even if generalized cost tags were present on the input data, we always minimize travel time. // The generalized cost calculations currently increment time and weight by the same amount. sr.quantityToMinimize = StreetRouter.State.RoutingVariable.DURATION_SECONDS; From 8d95c8f2537c1181c398e4d813196719791b514f Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 26 Oct 2021 15:42:12 +0800 Subject: [PATCH 165/187] add software product name to SoftwareVersion --- src/main/java/com/conveyal/r5/SoftwareVersion.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/r5/SoftwareVersion.java b/src/main/java/com/conveyal/r5/SoftwareVersion.java index f1f0acc01..4389afa74 100644 --- a/src/main/java/com/conveyal/r5/SoftwareVersion.java +++ b/src/main/java/com/conveyal/r5/SoftwareVersion.java @@ -20,7 +20,7 @@ public class SoftwareVersion { private static final String UNKNOWN = "UNKNOWN"; // This could potentially be made into a Component so it's non-static - public static SoftwareVersion instance = new SoftwareVersion(); + public static SoftwareVersion instance = new SoftwareVersion("r5"); private final Properties properties = new Properties(); @@ -29,7 +29,10 @@ public class SoftwareVersion { public final String commit; public final String branch; - protected SoftwareVersion () { + // Which software product is this a version of? Provides a scope or context for the version and commit strings. + public final String name; + + protected SoftwareVersion (String productName) { try (InputStream is = getClass().getResourceAsStream(VERSION_PROPERTIES_FILE)) { properties.load(is); } catch (IOException | NullPointerException e) { @@ -38,6 +41,7 @@ protected SoftwareVersion () { version = getPropertyOrUnknown("version"); commit = getPropertyOrUnknown("commit"); branch = getPropertyOrUnknown("branch"); + name = productName; } /** From 2fd1d3438b3510dc10788bbc669af7c52b91d0c3 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 26 Oct 2021 22:41:23 +0800 Subject: [PATCH 166/187] handle single point worker shutdown special case This was observed in testing when running many single point and regional analyses. Perhaps a regional worker started up and was assigned the same IP address as a single point worker that had recently shut down from inactivity. The resulting exception must be caught so this IP will be unregistered for this workerCategory and a new one started/found. --- .../analysis/controllers/BrokerController.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 39ecc1276..927151566 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -31,6 +31,7 @@ import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.HttpHostConnectException; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.util.EntityUtils; import org.mongojack.DBCursor; @@ -233,7 +234,15 @@ private Object singlePoint(Request request, Response response) { "complexity of this scenario, your request may have too many simulated schedules. If you are " + "using Routing Engine version < 4.5.1, your scenario may still be in preparation and you should " + "try again in a few minutes."); - } catch (NoRouteToHostException nrthe){ + } catch (NoRouteToHostException | HttpHostConnectException e) { + // NoRouteToHostException occurs when a single-point worker shuts down (normally due to inactivity) but is + // not yet removed from the worker catalog. + // HttpHostConnectException has also been observed, presumably after a worker shuts down and a new one + // starts up but claims the same IP address as the defunct single point worker. + // Yet another even rarer case is possible, where a single point worker starts for a different network and + // is assigned the same IP as the defunct worker. + // All these cases could be avoided by more rapidly removing workers from the catalog via frequent regular + // polling with backpressure, potentially including an "I'm shutting down" flag. LOG.warn("Worker in category {} was previously cataloged but is not reachable now. This is expected if a " + "user made a single-point request within WORKER_RECORD_DURATION_MSEC after shutdown.", workerCategory); httpPost.abort(); From 0fa990580428c43d7e1f2d8f0c708614c39e948f Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 26 Oct 2021 23:02:50 +0800 Subject: [PATCH 167/187] return 404 for results of incomplete analyses returning partial regional analysis results was only half-implemented, which was causing us to mishandle requests on incomplete jobs. --- .../analysis/components/broker/Broker.java | 9 - .../RegionalAnalysisController.java | 154 ++++++++---------- 2 files changed, 65 insertions(+), 98 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/components/broker/Broker.java b/src/main/java/com/conveyal/analysis/components/broker/Broker.java index 1358ced95..b701f1a47 100644 --- a/src/main/java/com/conveyal/analysis/components/broker/Broker.java +++ b/src/main/java/com/conveyal/analysis/components/broker/Broker.java @@ -492,15 +492,6 @@ private void requestExtraWorkersIfAppropriate(Job job) { } } - public File getPartialRegionalAnalysisResults (String jobId) { - MultiOriginAssembler resultAssembler = resultAssemblers.get(jobId); - if (resultAssembler == null) { - return null; - } else { - return null; // Was: resultAssembler.getGridBufferFile(); TODO implement fetching partially completed? - } - } - public synchronized boolean anyJobsActive () { for (Job job : jobs.values()) { if (job.isActive()) return true; diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 210c62b1f..609b6d0c6 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -4,6 +4,7 @@ import com.conveyal.analysis.SelectingGridReducer; import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.broker.Broker; +import com.conveyal.analysis.components.broker.Job; import com.conveyal.analysis.components.broker.JobStatus; import com.conveyal.analysis.models.AnalysisRequest; import com.conveyal.analysis.models.OpportunityDataset; @@ -230,103 +231,78 @@ private Object getRegionalResults (Request req, Response res) throws IOException String.join(",", analysis.destinationPointSetIds)); } - // It seems like you would check regionalAnalysis.complete to choose between redirecting to s3 and fetching - // the partially completed local file. But this field is never set to true - it's on a UI model object that - // isn't readily accessible to the internal Job-tracking mechanism of the back end. Instead, just try to fetch - // the partially completed results file, which includes an O(1) check whether the job is still being processed. - File partialRegionalAnalysisResultFile = broker.getPartialRegionalAnalysisResults(regionalAnalysisId); - - if (partialRegionalAnalysisResultFile != null) { - // FIXME we need to do the equivalent of the SelectingGridReducer here. - // The job is still being processed. There is a probably harmless race condition if the job happens to be - // completed at the very moment we're in this block, because the file will be deleted at that moment. - LOG.debug("Analysis {} is not complete, attempting to return the partial results grid.", regionalAnalysisId); - if (!"GRID".equalsIgnoreCase(fileFormatExtension)) { - throw AnalysisServerException.badRequest( - "For partially completed regional analyses, we can only return grid files, not images."); - } - if (partialRegionalAnalysisResultFile == null) { - throw AnalysisServerException.unknown( - "Could not find partial result grid for incomplete regional analysis on server."); - } - try { - res.header("content-type", "application/octet-stream"); - // This will cause Spark Framework to gzip the data automatically if requested by the client. - res.header("Content-Encoding", "gzip"); - // Spark has default serializers for InputStream and Bytes, and calls toString() on everything else. - return new FileInputStream(partialRegionalAnalysisResultFile); - } catch (FileNotFoundException e) { - // The job must have finished and the file was deleted upon upload to S3. This should be very rare. - throw AnalysisServerException.unknown( - "Could not find partial result grid for incomplete regional analysis on server."); - } - } else { - // The analysis has already completed, results should be stored and retrieved from S3 via redirects. - LOG.debug("Returning {} minute accessibility to pointset {} (percentile {}) for regional analysis {}.", - cutoffMinutes, destinationPointSetId, percentile, regionalAnalysisId); - FileStorageFormat format = FileStorageFormat.valueOf(fileFormatExtension.toUpperCase()); - if (!FileStorageFormat.GRID.equals(format) && !FileStorageFormat.PNG.equals(format) && !FileStorageFormat.GEOTIFF.equals(format)) { - throw AnalysisServerException.badRequest("Format \"" + format + "\" is invalid. Request format must be \"grid\", \"png\", or \"tiff\"."); - } + // We started implementing the ability to retrieve and display partially completed analyses. + // We eventually decided these should not be available here at the same endpoint as complete, immutable results. + + if (broker.findJob(regionalAnalysisId) != null) { + throw AnalysisServerException.notFound("Analysis is incomplete, no results file is available."); + } - // Analysis grids now have the percentile and cutoff in their S3 key, because there can be many of each. - // We do this even for results generated by older workers, so they will be re-extracted with the new name. - // These grids are reasonably small, we may be able to just send all cutoffs to the UI instead of selecting. - String singleCutoffKey = - String.format("%s_%s_P%d_C%d.%s", regionalAnalysisId, destinationPointSetId, percentile, cutoffMinutes, fileFormatExtension); - - // A lot of overhead here - UI contacts backend, backend calls S3, backend responds to UI, UI contacts S3. - FileStorageKey singleCutoffFileStorageKey = new FileStorageKey(RESULTS, singleCutoffKey); - if (!fileStorage.exists(singleCutoffFileStorageKey)) { - // An accessibility grid for this particular cutoff has apparently never been extracted from the - // regional results file before. Extract one and save it for future reuse. Older regional analyses - // did not have arrays allowing multiple cutoffs, percentiles, or destination pointsets. The - // filenames of such regional accessibility results will not have a percentile or pointset ID. - // First try the newest form of regional results: multi-percentile, multi-destination-grid. - String multiCutoffKey = String.format("%s_%s_P%d.access", regionalAnalysisId, destinationPointSetId, percentile); - FileStorageKey multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); + // The analysis has already completed, results should be stored and retrieved from S3 via redirects. + LOG.debug("Returning {} minute accessibility to pointset {} (percentile {}) for regional analysis {}.", + cutoffMinutes, destinationPointSetId, percentile, regionalAnalysisId); + FileStorageFormat format = FileStorageFormat.valueOf(fileFormatExtension.toUpperCase()); + if (!FileStorageFormat.GRID.equals(format) && !FileStorageFormat.PNG.equals(format) && !FileStorageFormat.GEOTIFF.equals(format)) { + throw AnalysisServerException.badRequest("Format \"" + format + "\" is invalid. Request format must be \"grid\", \"png\", or \"tiff\"."); + } + + // Analysis grids now have the percentile and cutoff in their S3 key, because there can be many of each. + // We do this even for results generated by older workers, so they will be re-extracted with the new name. + // These grids are reasonably small, we may be able to just send all cutoffs to the UI instead of selecting. + String singleCutoffKey = + String.format("%s_%s_P%d_C%d.%s", regionalAnalysisId, destinationPointSetId, percentile, cutoffMinutes, fileFormatExtension); + + // A lot of overhead here - UI contacts backend, backend calls S3, backend responds to UI, UI contacts S3. + FileStorageKey singleCutoffFileStorageKey = new FileStorageKey(RESULTS, singleCutoffKey); + if (!fileStorage.exists(singleCutoffFileStorageKey)) { + // An accessibility grid for this particular cutoff has apparently never been extracted from the + // regional results file before. Extract one and save it for future reuse. Older regional analyses + // did not have arrays allowing multiple cutoffs, percentiles, or destination pointsets. The + // filenames of such regional accessibility results will not have a percentile or pointset ID. + // First try the newest form of regional results: multi-percentile, multi-destination-grid. + String multiCutoffKey = String.format("%s_%s_P%d.access", regionalAnalysisId, destinationPointSetId, percentile); + FileStorageKey multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); + if (!fileStorage.exists(multiCutoffFileStorageKey)) { + LOG.warn("Falling back to older file name formats for regional results file: " + multiCutoffKey); + // Fall back to second-oldest form: multi-percentile, single destination grid. + checkArgument(analysis.destinationPointSetIds.length == 1); + multiCutoffKey = String.format("%s_P%d.access", regionalAnalysisId, percentile); + multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); if (!fileStorage.exists(multiCutoffFileStorageKey)) { - LOG.warn("Falling back to older file name formats for regional results file: " + multiCutoffKey); - // Fall back to second-oldest form: multi-percentile, single destination grid. - checkArgument(analysis.destinationPointSetIds.length == 1); - multiCutoffKey = String.format("%s_P%d.access", regionalAnalysisId, percentile); + // Fall back on oldest form of results, single-percentile, single-destination-grid. + checkArgument(analysis.travelTimePercentiles.length == 1); + multiCutoffKey = regionalAnalysisId + ".access"; multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); if (!fileStorage.exists(multiCutoffFileStorageKey)) { - // Fall back on oldest form of results, single-percentile, single-destination-grid. - checkArgument(analysis.travelTimePercentiles.length == 1); - multiCutoffKey = regionalAnalysisId + ".access"; - multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); - if (!fileStorage.exists(multiCutoffFileStorageKey)) { - throw new IllegalArgumentException("Cannot find original source regional analysis output."); - } + throw new IllegalArgumentException("Cannot find original source regional analysis output."); } } - LOG.debug("Single-cutoff grid {} not found on S3, deriving it from {}.", singleCutoffKey, multiCutoffKey); - - InputStream multiCutoffInputStream = new FileInputStream(fileStorage.getFile(multiCutoffFileStorageKey)); - Grid grid = new SelectingGridReducer(cutoffIndex).compute(multiCutoffInputStream); - - File localFile = FileUtils.createScratchFile(format.toString()); - FileOutputStream fos = new FileOutputStream(localFile); - - switch (format) { - case GRID: - grid.write(new GZIPOutputStream(fos)); - break; - case PNG: - grid.writePng(fos); - break; - case GEOTIFF: - grid.writeGeotiff(fos); - break; - } - - fileStorage.moveIntoStorage(singleCutoffFileStorageKey, localFile); } - return JsonUtil.toJsonString( - JsonUtil.objectNode().put("url", fileStorage.getURL(singleCutoffFileStorageKey)) - ); + LOG.debug("Single-cutoff grid {} not found on S3, deriving it from {}.", singleCutoffKey, multiCutoffKey); + + InputStream multiCutoffInputStream = new FileInputStream(fileStorage.getFile(multiCutoffFileStorageKey)); + Grid grid = new SelectingGridReducer(cutoffIndex).compute(multiCutoffInputStream); + + File localFile = FileUtils.createScratchFile(format.toString()); + FileOutputStream fos = new FileOutputStream(localFile); + + switch (format) { + case GRID: + grid.write(new GZIPOutputStream(fos)); + break; + case PNG: + grid.writePng(fos); + break; + case GEOTIFF: + grid.writeGeotiff(fos); + break; + } + + fileStorage.moveIntoStorage(singleCutoffFileStorageKey, localFile); } + return JsonUtil.toJsonString( + JsonUtil.objectNode().put("url", fileStorage.getURL(singleCutoffFileStorageKey)) + ); } private String getCsvResults (Request req, Response res) { From 894f97627dfe61defa614c3463c1022d5c28b028 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Wed, 27 Oct 2021 10:17:24 +0800 Subject: [PATCH 168/187] Fix possible NPE by using Objects.equals to compare graphIds --- .../com/conveyal/analysis/components/broker/WorkerCatalog.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/components/broker/WorkerCatalog.java b/src/main/java/com/conveyal/analysis/components/broker/WorkerCatalog.java index 0a21ac086..c91ca7ae8 100644 --- a/src/main/java/com/conveyal/analysis/components/broker/WorkerCatalog.java +++ b/src/main/java/com/conveyal/analysis/components/broker/WorkerCatalog.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * A catalog of all the workers this broker has been contacted by recently. @@ -164,7 +165,7 @@ public synchronized boolean noWorkersAvailable(WorkerCategory category, boolean purgeDeadWorkers(); if (ignoreWorkerVersion) { // Look for workers on the right network ID, independent of their worker software version. - return observationsByWorkerId.values().stream().noneMatch(obs -> obs.category.graphId.equals(category.graphId)); + return observationsByWorkerId.values().stream().noneMatch(obs -> Objects.equals(category.graphId, obs.category.graphId)); } return workerIdsByCategory.get(category).isEmpty(); } From 1269e1cfe11ddb86888dd042ba0f890caacbeee1 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 27 Oct 2021 17:34:43 +0800 Subject: [PATCH 169/187] comments, close OSM file after reading --- src/main/java/com/conveyal/analysis/BackendMain.java | 1 + .../com/conveyal/analysis/controllers/BundleController.java | 2 ++ src/main/java/com/conveyal/analysis/models/Reroute.java | 1 + 3 files changed, 4 insertions(+) diff --git a/src/main/java/com/conveyal/analysis/BackendMain.java b/src/main/java/com/conveyal/analysis/BackendMain.java index 431537095..67fab4159 100644 --- a/src/main/java/com/conveyal/analysis/BackendMain.java +++ b/src/main/java/com/conveyal/analysis/BackendMain.java @@ -51,6 +51,7 @@ private static void startServerInternal (BackendComponents components, TaskActio if (components.config.offline()) { LOG.info("Running in OFFLINE mode."); LOG.info("Pre-starting local cluster of Analysis workers..."); + // WorkerCategory(null, null) means a worker is not on any network, and is waiting to be assigned one. components.workerLauncher.launch(new WorkerCategory(null, null), null, 1, 0); } diff --git a/src/main/java/com/conveyal/analysis/controllers/BundleController.java b/src/main/java/com/conveyal/analysis/controllers/BundleController.java index d45a47d78..b282224fe 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BundleController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BundleController.java @@ -164,6 +164,8 @@ private Bundle create (Request req, Response res) { // Wrapping in buffered input stream should reduce number of progress updates. osm.readPbf(ProgressInputStream.forFileItem(fi, progressListener)); // osm.readPbf(new BufferedInputStream(fi.getInputStream())); + osm.close(); + // Store the source OSM file. Note that we're not storing the derived MapDB file here. fileStorage.moveIntoStorage(osmCache.getKey(bundle.osmId), fi.getStoreLocation()); } diff --git a/src/main/java/com/conveyal/analysis/models/Reroute.java b/src/main/java/com/conveyal/analysis/models/Reroute.java index 163b80fc1..4f7499ab9 100644 --- a/src/main/java/com/conveyal/analysis/models/Reroute.java +++ b/src/main/java/com/conveyal/analysis/models/Reroute.java @@ -10,6 +10,7 @@ public String getType() { return "reroute"; } + /** The _id of the gtfs feed, providing a scope for any unscoped identifiers in this Modification. */ public String feed; public String[] routes; public String[] trips; From 52c2ab1ae582156ce9f1d3ff9d8c014987149834 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 27 Oct 2021 20:09:52 +0800 Subject: [PATCH 170/187] Fix regional analyses with freeform destinations --- .../java/com/conveyal/r5/analyst/NetworkPreloader.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java b/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java index 52e915133..6b4ad2e77 100644 --- a/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java +++ b/src/main/java/com/conveyal/r5/analyst/NetworkPreloader.java @@ -111,6 +111,13 @@ protected TransportNetwork buildValue(Key key) { // Get the set of points to which we are measuring travel time. Any smaller sub-grids created here will // reference the scenarioNetwork's built-in full-extent pointset, so can reuse its linkage. // TODO handle multiple destination grids. + + if (key.destinationGridExtents == null) { + // Special (and ideally temporary) case for regional freeform destinations, where there is no grid to link. + // The null destinationGridExtents are created by the WebMercatorExtents#forPointsets else clause. + return scenarioNetwork; + } + setProgress(key, 0, "Fetching gridded point set..."); PointSet pointSet = AnalysisWorkerTask.gridPointSetCache.get(key.destinationGridExtents, scenarioNetwork.fullExtentGridPointSet); From 119a2aaa6be36a14c07f9ecd44fd0db388e000df Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 27 Oct 2021 20:16:45 +0800 Subject: [PATCH 171/187] cleanly handle non-existing regional result grids Ideally we'd infer from the database record that this analysis does not have any gridded results and immediately return a 404. As a short-term fix, we check assertions only when an older-format file is actually found, and return a 404 to cover a wider range of cases: files that are missing because they are not even supposed to exist, and files that should exist but have gone missing for other unknown reasons. --- .../analysis/controllers/BrokerController.java | 3 ++- .../controllers/RegionalAnalysisController.java | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java index 927151566..211a5ddb6 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BrokerController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BrokerController.java @@ -186,7 +186,8 @@ private Object singlePoint(Request request, Response response) { // FIXME the tracking of which workers are starting up should really be encapsulated using a "start up if needed" method. broker.recentlyRequestedWorkers.remove(workerCategory); } - String workerUrl = "http://" + address + ":7080/single"; // TODO remove hard-coded port number. + // Port number is hard-coded until we have a good reason to make it configurable. + String workerUrl = "http://" + address + ":7080/single"; LOG.debug("Re-issuing HTTP request from UI to worker at {}", workerUrl); HttpPost httpPost = new HttpPost(workerUrl); // httpPost.setHeader("Accept", "application/x-analysis-time-grid"); diff --git a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java index 609b6d0c6..60197114b 100644 --- a/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java +++ b/src/main/java/com/conveyal/analysis/controllers/RegionalAnalysisController.java @@ -238,6 +238,10 @@ private Object getRegionalResults (Request req, Response res) throws IOException throw AnalysisServerException.notFound("Analysis is incomplete, no results file is available."); } + // FIXME It is possible that regional analysis is complete, but UI is trying to fetch gridded results when there + // aren't any (only CSV, because origins are freeform). + // How can we determine whether this analysis is expected to have no gridded results and cleanly return a 404? + // The analysis has already completed, results should be stored and retrieved from S3 via redirects. LOG.debug("Returning {} minute accessibility to pointset {} (percentile {}) for regional analysis {}.", cutoffMinutes, destinationPointSetId, percentile, regionalAnalysisId); @@ -265,16 +269,19 @@ private Object getRegionalResults (Request req, Response res) throws IOException if (!fileStorage.exists(multiCutoffFileStorageKey)) { LOG.warn("Falling back to older file name formats for regional results file: " + multiCutoffKey); // Fall back to second-oldest form: multi-percentile, single destination grid. - checkArgument(analysis.destinationPointSetIds.length == 1); multiCutoffKey = String.format("%s_P%d.access", regionalAnalysisId, percentile); multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); - if (!fileStorage.exists(multiCutoffFileStorageKey)) { + if (fileStorage.exists(multiCutoffFileStorageKey)) { + checkArgument(analysis.destinationPointSetIds.length == 1); + } else { // Fall back on oldest form of results, single-percentile, single-destination-grid. - checkArgument(analysis.travelTimePercentiles.length == 1); multiCutoffKey = regionalAnalysisId + ".access"; multiCutoffFileStorageKey = new FileStorageKey(RESULTS, multiCutoffKey); - if (!fileStorage.exists(multiCutoffFileStorageKey)) { - throw new IllegalArgumentException("Cannot find original source regional analysis output."); + if (fileStorage.exists(multiCutoffFileStorageKey)) { + checkArgument(analysis.travelTimePercentiles.length == 1); + checkArgument(analysis.destinationPointSetIds.length == 1); + } else { + throw AnalysisServerException.notFound("Cannot find original source regional analysis output."); } } } From 4a9363c6b550b4c0c2e6584c852d0bc0a98a7230 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 29 Oct 2021 19:38:24 +0800 Subject: [PATCH 172/187] log a warning when file format is unrecognized rather than silently taking a legacy code path --- .../controllers/OpportunityDatasetController.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java index ff097066d..012309ee1 100644 --- a/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java +++ b/src/main/java/com/conveyal/analysis/controllers/OpportunityDatasetController.java @@ -571,13 +571,15 @@ private List createGridsFromShapefile(List fileItems, */ private Object downloadOpportunityDataset (Request req, Response res) throws IOException { FileStorageFormat downloadFormat; + String format = req.params("format"); try { - downloadFormat = FileStorageFormat.valueOf(req.params("format").toUpperCase()); + downloadFormat = FileStorageFormat.valueOf(format.toUpperCase()); } catch (IllegalArgumentException iae) { - // This code handles the deprecated endpoint for retrieving opportunity datasets - // get("/api/opportunities/:regionId/:gridKey") is the same signature as this endpoint. + LOG.warn("Unable to interpret format path parameter '{}', using legacy code path.", format); + // This code handles the deprecated endpoint for retrieving opportunity datasets. + // get("/api/opportunities/:regionId/:gridKey") has the same path pattern as this endpoint. String regionId = req.params("_id"); - String gridKey = req.params("format"); + String gridKey = format; FileStorageKey storageKey = new FileStorageKey(GRIDS, String.format("%s/%s.grid", regionId, gridKey)); return getJsonUrl(storageKey); } From 5e6b281b09023f86f21d388140c5b42dc47013bc Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 3 Nov 2021 15:46:15 +0800 Subject: [PATCH 173/187] Improve WGS84 bounding box range checking Add method to check for spanning 180 degree meridian. Move methods into GeometryUtils for reuse. Check envelope of uploaded OSM and GTFS data. Check envelope of WebMercatorGridPointSet when created. Check envelope of StreetLayer when it's loaded. Text describing range-checked object is now substituted into messages. --- .../controllers/BundleController.java | 13 ++- .../datasource/CsvDataSourceIngester.java | 1 - .../datasource/GeoJsonDataSourceIngester.java | 4 +- .../GeoPackageDataSourceIngester.java | 4 +- .../ShapefileDataSourceIngester.java | 4 +- src/main/java/com/conveyal/osmlib/OSM.java | 2 +- .../conveyal/r5/analyst/FreeFormPointSet.java | 3 +- .../java/com/conveyal/r5/analyst/Grid.java | 63 +------------- .../r5/analyst/WebMercatorGridPointSet.java | 2 + .../com/conveyal/r5/common/GeometryUtils.java | 84 ++++++++++++++++++- .../com/conveyal/r5/streets/StreetLayer.java | 2 + 11 files changed, 110 insertions(+), 72 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BundleController.java b/src/main/java/com/conveyal/analysis/controllers/BundleController.java index b282224fe..570d0138f 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BundleController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BundleController.java @@ -8,15 +8,16 @@ import com.conveyal.analysis.persistence.Persistence; import com.conveyal.analysis.util.HttpUtils; import com.conveyal.analysis.util.JsonUtil; -import com.conveyal.r5.analyst.progress.ProgressInputStream; import com.conveyal.file.FileStorage; import com.conveyal.file.FileStorageKey; import com.conveyal.file.FileUtils; import com.conveyal.gtfs.GTFSCache; import com.conveyal.gtfs.GTFSFeed; import com.conveyal.gtfs.model.Stop; +import com.conveyal.osmlib.Node; import com.conveyal.osmlib.OSM; import com.conveyal.r5.analyst.cluster.BundleManifest; +import com.conveyal.r5.analyst.progress.ProgressInputStream; import com.conveyal.r5.analyst.progress.Task; import com.conveyal.r5.streets.OSMCache; import com.conveyal.r5.util.ExceptionUtils; @@ -43,9 +44,10 @@ import java.util.stream.Collectors; import java.util.zip.ZipFile; -import static com.conveyal.r5.analyst.progress.WorkProductType.BUNDLE; import static com.conveyal.analysis.util.JsonUtil.toJson; import static com.conveyal.file.FileCategory.BUNDLES; +import static com.conveyal.r5.analyst.progress.WorkProductType.BUNDLE; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; /** * This Controller provides HTTP REST endpoints for manipulating Bundles. Bundles are sets of GTFS feeds and OSM @@ -164,7 +166,12 @@ private Bundle create (Request req, Response res) { // Wrapping in buffered input stream should reduce number of progress updates. osm.readPbf(ProgressInputStream.forFileItem(fi, progressListener)); // osm.readPbf(new BufferedInputStream(fi.getInputStream())); + Envelope osmBounds = new Envelope(); + for (Node n : osm.nodes.values()) { + osmBounds.expandToInclude(n.getLon(), n.getLat()); + } osm.close(); + checkWgsEnvelopeSize(osmBounds, "OSM data"); // Store the source OSM file. Note that we're not storing the derived MapDB file here. fileStorage.moveIntoStorage(osmCache.getKey(bundle.osmId), fi.getStoreLocation()); } @@ -198,6 +205,7 @@ private Bundle create (Request req, Response res) { for (Stop s : feed.stops.values()) { bundleBounds.expandToInclude(s.stop_lon, s.stop_lat); } + checkWgsEnvelopeSize(bundleBounds, "GTFS data"); if (bundle.serviceStart.isAfter(feedSummary.serviceStart)) { bundle.serviceStart = feedSummary.serviceStart; @@ -228,7 +236,6 @@ private Bundle create (Request req, Response res) { // Set legacy progress field to indicate that all feeds have been loaded. bundle.feedsComplete = bundle.totalFeeds; - // TODO Handle crossing the antimeridian bundle.north = bundleBounds.getMaxY(); bundle.south = bundleBounds.getMinY(); bundle.east = bundleBounds.getMaxX(); diff --git a/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java index b0dc8778c..1fd3c25a6 100644 --- a/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/CsvDataSourceIngester.java @@ -12,7 +12,6 @@ import java.io.File; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; /** * Logic to create SpatialDataSource metadata from a comma separated file. diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java index 5e6e49b0f..49f182860 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoJsonDataSourceIngester.java @@ -31,7 +31,7 @@ import java.util.stream.Collectors; import static com.conveyal.analysis.models.DataSourceValidationIssue.Level.ERROR; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; /** * Logic to create SpatialDataSource metadata from an uploaded GeoJSON file and perform validation. @@ -140,7 +140,7 @@ public void ingest (File file, ProgressListener progressListener) { } checkCrs(featureType); Envelope wgsEnvelope = wgsFeatureCollection.getBounds(); - checkWgsEnvelopeSize(wgsEnvelope); + checkWgsEnvelopeSize(wgsEnvelope, "GeoJSON"); // Set SpatialDataSource fields (Conveyal metadata) from GeoTools model dataSource.wgsBounds = Bounds.fromWgsEnvelope(wgsEnvelope); diff --git a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java index 215451223..64ba8d525 100644 --- a/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/GeoPackageDataSourceIngester.java @@ -28,7 +28,7 @@ import java.util.HashMap; import java.util.Map; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; import static com.conveyal.r5.util.ShapefileReader.attributes; import static com.conveyal.r5.util.ShapefileReader.geometryType; import static com.google.common.base.Preconditions.checkState; @@ -77,7 +77,7 @@ public void ingest (File file, ProgressListener progressListener) { query.setCoordinateSystemReproject(DefaultGeographicCRS.WGS84); FeatureCollection wgsFeatureCollection = featureSource.getFeatures(query); Envelope wgsEnvelope = wgsFeatureCollection.getBounds(); - checkWgsEnvelopeSize(wgsEnvelope); + checkWgsEnvelopeSize(wgsEnvelope, "GeoPackage"); progressListener.increment(); FeatureIterator wgsFeatureIterator = wgsFeatureCollection.features(); while (wgsFeatureIterator.hasNext()) { diff --git a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java index a513a6bee..03e61c6b7 100644 --- a/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java +++ b/src/main/java/com/conveyal/analysis/datasource/ShapefileDataSourceIngester.java @@ -15,7 +15,7 @@ import java.io.File; import java.io.IOException; -import static com.conveyal.r5.analyst.Grid.checkWgsEnvelopeSize; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; import static com.google.common.base.Preconditions.checkState; /** @@ -45,7 +45,7 @@ public void ingest (File file, ProgressListener progressListener) { ShapefileReader reader = new ShapefileReader(file); // Iterate over all features to ensure file is readable, geometries are valid, and can be reprojected. Envelope envelope = reader.wgs84Bounds(); - checkWgsEnvelopeSize(envelope); + checkWgsEnvelopeSize(envelope, "Shapefile"); reader.wgs84Stream().forEach(f -> { checkState(envelope.contains(((Geometry)f.getDefaultGeometry()).getEnvelopeInternal())); }); diff --git a/src/main/java/com/conveyal/osmlib/OSM.java b/src/main/java/com/conveyal/osmlib/OSM.java index f690bb18d..4d1972a20 100644 --- a/src/main/java/com/conveyal/osmlib/OSM.java +++ b/src/main/java/com/conveyal/osmlib/OSM.java @@ -69,7 +69,7 @@ public class OSM implements OSMEntitySource, OSMEntitySink { /* If true, track which nodes are referenced by more than one way. */ public boolean intersectionDetection = false; - /** If true we are reading already filled OSM mapdv **/ + /** If true we are reading already filled OSM mapdb **/ private boolean reading = false; /** diff --git a/src/main/java/com/conveyal/r5/analyst/FreeFormPointSet.java b/src/main/java/com/conveyal/r5/analyst/FreeFormPointSet.java index d580fae67..0cce92dd6 100644 --- a/src/main/java/com/conveyal/r5/analyst/FreeFormPointSet.java +++ b/src/main/java/com/conveyal/r5/analyst/FreeFormPointSet.java @@ -18,6 +18,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; import static com.conveyal.r5.streets.VertexStore.fixedDegreesToFloating; /** @@ -118,7 +119,7 @@ public static FreeFormPointSet fromCsv ( // If count column was specified and present, use it. Otherwise, one opportunity per point. ret.counts[rec] = countCol < 0 ? 1D : Double.parseDouble(reader.get(countCol)); } - Grid.checkWgsEnvelopeSize(ret.getWgsEnvelope()); + checkWgsEnvelopeSize(ret.getWgsEnvelope(), "freeform pointset"); return ret; } catch (NumberFormatException nfe) { throw new ParameterException( diff --git a/src/main/java/com/conveyal/r5/analyst/Grid.java b/src/main/java/com/conveyal/r5/analyst/Grid.java index 2b131ff2d..7a11f595e 100644 --- a/src/main/java/com/conveyal/r5/analyst/Grid.java +++ b/src/main/java/com/conveyal/r5/analyst/Grid.java @@ -63,6 +63,8 @@ import java.util.concurrent.atomic.AtomicInteger; import static com.conveyal.gtfs.util.Util.human; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.lang.Double.parseDouble; import static org.apache.commons.math3.util.FastMath.atan; @@ -93,9 +95,6 @@ public class Grid extends PointSet { */ public final double[][] grid; - /** Maximum area allowed for the bounding box of an uploaded shapefile -- large enough for New York State. */ - private static final double MAX_BOUNDING_BOX_AREA_SQ_KM = 250_000; - /** Maximum area allowed for features in a shapefile upload */ private static final double MAX_FEATURE_AREA_SQ_DEG = 2; @@ -599,7 +598,7 @@ public static List fromCsv(InputStreamProvider csvInputStreamProvider, // This will also close the InputStreams. reader.close(); - checkWgsEnvelopeSize(envelope); + checkWgsEnvelopeSize(envelope, "CSV points"); WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(envelope, zoom); checkPixelCount(extents, numericColumns.size()); @@ -671,7 +670,7 @@ public static List fromShapefile (File shapefile, int zoom, ProgressListen ShapefileReader reader = new ShapefileReader(shapefile); Envelope envelope = reader.wgs84Bounds(); - checkWgsEnvelopeSize(envelope); + checkWgsEnvelopeSize(envelope, "Shapefile"); WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(envelope, zoom); List numericAttributes = reader.numericAttributes(); Set uniqueNumericAttributes = new HashSet<>(numericAttributes); @@ -776,38 +775,6 @@ public WebMercatorExtents getWebMercatorExtents () { return extents; } - /** - * @return the approximate area of an Envelope in WGS84 lat/lon coordinates, in square kilometers. - */ - public static double roughWgsEnvelopeArea (Envelope wgsEnvelope) { - double lon0 = wgsEnvelope.getMinX(); - double lon1 = wgsEnvelope.getMaxX(); - double lat0 = wgsEnvelope.getMinY(); - double lat1 = wgsEnvelope.getMaxY(); - double height = lat1 - lat0; - double width = lon1 - lon0; - final double KM_PER_DEGREE_LAT = 111.133; - // Scale the x direction as if the Earth was a sphere. - // Error above the middle latitude should approximately cancel out error below that latitude. - double averageLat = (lat0 + lat1) / 2; - double xScale = FastMath.cos(FastMath.toRadians(averageLat)); - double area = (height * KM_PER_DEGREE_LAT) * (width * KM_PER_DEGREE_LAT * xScale); - return area; - } - - /** - * Throw an exception if the provided envelope is too big for a reasonable destination grid. - */ - public static void checkWgsEnvelopeSize (Envelope envelope) { - checkWgsEnvelopeRange(envelope); - if (roughWgsEnvelopeArea(envelope) > MAX_BOUNDING_BOX_AREA_SQ_KM) { - throw new DataSourceException(String.format( - "Geographic extent of spatial layer (%.0f km2) exceeds limit of %.0f km2.", - roughWgsEnvelopeArea(envelope), MAX_BOUNDING_BOX_AREA_SQ_KM - )); - } - } - public static void checkPixelCount (WebMercatorExtents extents, int layers) { int pixels = extents.width * extents.height * layers; if (pixels > MAX_PIXELS) { @@ -817,28 +784,6 @@ public static void checkPixelCount (WebMercatorExtents extents, int layers) { } } - /** - * We have to range-check the envelope before checking its size. Large unprojected y values interpreted as latitudes - * can yield negaive cosines, producing negative estimated areas, producing false negatives on size checks. - */ - private static void checkWgsEnvelopeRange (Envelope envelope) { - checkLon(envelope.getMinX()); - checkLon(envelope.getMaxX()); - checkLat(envelope.getMinY()); - checkLat(envelope.getMaxY()); - } - private static void checkLon (double longitude) { - if (!Double.isFinite(longitude) || Math.abs(longitude) > 180) { - throw new DataSourceException("Longitude is not a finite number with absolute value below 180."); - } - } - - private static void checkLat (double latitude) { - // Longyearbyen on the Svalbard archipelago is the world's northernmost permanent settlement (78 degrees N). - if (!Double.isFinite(latitude) || Math.abs(latitude) > 80) { - throw new DataSourceException("Longitude is not a finite number with absolute value below 80."); - } - } } diff --git a/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java b/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java index 0fd779404..b6b614df7 100644 --- a/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java +++ b/src/main/java/com/conveyal/r5/analyst/WebMercatorGridPointSet.java @@ -10,6 +10,7 @@ import java.io.Serializable; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; import static com.conveyal.r5.streets.VertexStore.fixedDegreesToFloating; import static com.google.common.base.Preconditions.checkArgument; @@ -79,6 +80,7 @@ public WebMercatorGridPointSet (TransportNetwork transportNetwork) { */ public WebMercatorGridPointSet (Envelope wgsEnvelope) { LOG.info("Creating WebMercatorGridPointSet with WGS84 extents {}", wgsEnvelope); + checkWgsEnvelopeSize(wgsEnvelope, "grid point set"); this.zoom = DEFAULT_ZOOM; int west = lonToPixel(wgsEnvelope.getMinX()); int east = lonToPixel(wgsEnvelope.getMaxX()); diff --git a/src/main/java/com/conveyal/r5/common/GeometryUtils.java b/src/main/java/com/conveyal/r5/common/GeometryUtils.java index a7c46ffd8..ff5e0adb6 100644 --- a/src/main/java/com/conveyal/r5/common/GeometryUtils.java +++ b/src/main/java/com/conveyal/r5/common/GeometryUtils.java @@ -1,5 +1,6 @@ package com.conveyal.r5.common; +import com.conveyal.analysis.datasource.DataSourceException; import com.conveyal.r5.streets.VertexStore; import org.apache.commons.math3.util.FastMath; import org.locationtech.jts.geom.Coordinate; @@ -10,16 +11,21 @@ import static com.conveyal.r5.streets.VertexStore.fixedDegreesToFloating; import static com.conveyal.r5.streets.VertexStore.floatingDegreesToFixed; +import static com.google.common.base.Preconditions.checkArgument; /** * Reimplementation of OTP GeometryUtils, using copied code where there are not licensing concerns. + * Also contains reusable methods for validating WGS84 envelopes and latitude and longitude values. */ public class GeometryUtils { public static final GeometryFactory geometryFactory = new GeometryFactory(); - // average of polar and equatorial, https://en.wikipedia.org/wiki/Earth + /** Average of polar and equatorial radii, https://en.wikipedia.org/wiki/Earth */ public static final double RADIUS_OF_EARTH_M = 6_367_450; + /** Maximum area allowed for the bounding box of an uploaded shapefile -- large enough for New York State. */ + private static final double MAX_BOUNDING_BOX_AREA_SQ_KM = 250_000; + /** * Haversine formula for distance on the sphere. We used to have a fastDistance function that would estimate this * quickly, but I'm not convinced we actually need it. @@ -88,4 +94,80 @@ public static Envelope floatingWgsEnvelopeToFixed (Envelope floatingWgsEnvelope) return new Envelope(fixedMinX, fixedMaxX, fixedMinY, fixedMaxY); } + //// Methods for range-checking points and envelopes in WGS84 + + /** + * We have to range-check the envelope before checking its size. Large unprojected y values interpreted as latitudes + * can yield negative cosines, producing negative estimated areas, producing false negatives on size checks. + */ + private static void checkWgsEnvelopeRange (Envelope envelope) { + checkLon(envelope.getMinX()); + checkLon(envelope.getMaxX()); + checkLat(envelope.getMinY()); + checkLat(envelope.getMaxY()); + } + + private static void checkLon (double longitude) { + if (!Double.isFinite(longitude) || Math.abs(longitude) > 180) { + throw new DataSourceException("Longitude is not a finite number with absolute value below 180."); + } + } + + private static void checkLat (double latitude) { + // Longyearbyen on the Svalbard archipelago is the world's northernmost permanent settlement (78 degrees N). + if (!Double.isFinite(latitude) || Math.abs(latitude) > 80) { + throw new DataSourceException("Longitude is not a finite number with absolute value below 80."); + } + } + + /** + * Throw an exception if the envelope appears to be constructed from points spanning the 180 degree meridian. + * We check whether the envelope becomes narrower when its left edge is shifted by 180 degrees (expressed as a + * longitude greater than 180). The envelope must already be validated with checkWgsEnvelopeRange to ensure + * meaningful results. + */ + private static void checkWgsEnvelopeAntimeridian (Envelope envelope) { + double widthAcrossAntimeridian = (envelope.getMinX() + 180) - envelope.getMaxX(); + checkArgument( + envelope.getWidth() < widthAcrossAntimeridian, + "Data sets may not span the antimeridian (180 degrees longitude)." + ); + } + + /** + * @return the approximate area of an Envelope in WGS84 lat/lon coordinates, in square kilometers. + */ + public static double roughWgsEnvelopeArea (Envelope wgsEnvelope) { + double lon0 = wgsEnvelope.getMinX(); + double lon1 = wgsEnvelope.getMaxX(); + double lat0 = wgsEnvelope.getMinY(); + double lat1 = wgsEnvelope.getMaxY(); + double height = lat1 - lat0; + double width = lon1 - lon0; + final double KM_PER_DEGREE_LAT = 111.133; + // Scale the x direction as if the Earth was a sphere. + // Error above the middle latitude should approximately cancel out error below that latitude. + double averageLat = (lat0 + lat1) / 2; + double xScale = FastMath.cos(FastMath.toRadians(averageLat)); + double area = (height * KM_PER_DEGREE_LAT) * (width * KM_PER_DEGREE_LAT * xScale); + return area; + } + + /** + * Throw an exception if the provided envelope is too big for a reasonable destination grid. + * Should also catch cases where data sets include points on both sides of the 180 degree meridian. + * This static utility method can be reused to test other automatically determined bounds such as those + * from OSM or GTFS uploads. + */ + public static void checkWgsEnvelopeSize (Envelope envelope, String thingBeingChecked) { + checkWgsEnvelopeRange(envelope); + checkWgsEnvelopeAntimeridian(envelope); + if (roughWgsEnvelopeArea(envelope) > MAX_BOUNDING_BOX_AREA_SQ_KM) { + throw new DataSourceException(String.format( + "Geographic extent of %s (%.0f km2) exceeds limit of %.0f km2.", + thingBeingChecked, roughWgsEnvelopeArea(envelope), MAX_BOUNDING_BOX_AREA_SQ_KM + )); + } + } + } diff --git a/src/main/java/com/conveyal/r5/streets/StreetLayer.java b/src/main/java/com/conveyal/r5/streets/StreetLayer.java index b7bfc8fe2..0372f5ddc 100644 --- a/src/main/java/com/conveyal/r5/streets/StreetLayer.java +++ b/src/main/java/com/conveyal/r5/streets/StreetLayer.java @@ -52,6 +52,7 @@ import java.util.stream.LongStream; import static com.conveyal.r5.analyst.scenario.PickupWaitTimes.NO_WAIT_ALL_STOPS; +import static com.conveyal.r5.common.GeometryUtils.checkWgsEnvelopeSize; import static com.conveyal.r5.streets.VertexStore.fixedDegreeGeometryToFloating; /** @@ -379,6 +380,7 @@ void loadFromOsm (OSM osm, boolean removeIslands, boolean saveVertexIndex) { if (!saveVertexIndex) vertexIndexForOsmNode = null; + checkWgsEnvelopeSize(envelope, "street layer"); osm = null; } From 8d2a6e0bf286b9d890954485183fde1a91a8e386 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 3 Nov 2021 16:08:57 +0800 Subject: [PATCH 174/187] Fix geographic extent tests Consistently throw more general exception type. Check that data actually contains points near the antimeridian. --- .../com/conveyal/r5/common/GeometryUtils.java | 18 ++++++++++-------- .../SpatialDataSourceIngesterTest.java | 5 ++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/r5/common/GeometryUtils.java b/src/main/java/com/conveyal/r5/common/GeometryUtils.java index ff5e0adb6..371798e1b 100644 --- a/src/main/java/com/conveyal/r5/common/GeometryUtils.java +++ b/src/main/java/com/conveyal/r5/common/GeometryUtils.java @@ -122,15 +122,17 @@ private static void checkLat (double latitude) { /** * Throw an exception if the envelope appears to be constructed from points spanning the 180 degree meridian. - * We check whether the envelope becomes narrower when its left edge is shifted by 180 degrees (expressed as a - * longitude greater than 180). The envelope must already be validated with checkWgsEnvelopeRange to ensure - * meaningful results. + * We check whether the envelope becomes narrower when its left edge is expressed as a longitude greater than 180 + * (shifted east by 180 degrees) and has points anywhere near the 180 degree line. + * The envelope must already be validated with checkWgsEnvelopeRange to ensure meaningful results. */ - private static void checkWgsEnvelopeAntimeridian (Envelope envelope) { + private static void checkWgsEnvelopeAntimeridian (Envelope envelope, String thingBeingChecked) { double widthAcrossAntimeridian = (envelope.getMinX() + 180) - envelope.getMaxX(); + boolean nearAntimeridian = + Math.abs(envelope.getMinX() - 180D) < 10 || Math.abs(envelope.getMaxX() - 180D) < 10; checkArgument( - envelope.getWidth() < widthAcrossAntimeridian, - "Data sets may not span the antimeridian (180 degrees longitude)." + !nearAntimeridian || envelope.getWidth() < widthAcrossAntimeridian, + thingBeingChecked + " may not span the antimeridian (180 degrees longitude)." ); } @@ -161,9 +163,9 @@ public static double roughWgsEnvelopeArea (Envelope wgsEnvelope) { */ public static void checkWgsEnvelopeSize (Envelope envelope, String thingBeingChecked) { checkWgsEnvelopeRange(envelope); - checkWgsEnvelopeAntimeridian(envelope); + checkWgsEnvelopeAntimeridian(envelope, thingBeingChecked); if (roughWgsEnvelopeArea(envelope) > MAX_BOUNDING_BOX_AREA_SQ_KM) { - throw new DataSourceException(String.format( + throw new IllegalArgumentException(String.format( "Geographic extent of %s (%.0f km2) exceeds limit of %.0f km2.", thingBeingChecked, roughWgsEnvelopeArea(envelope), MAX_BOUNDING_BOX_AREA_SQ_KM )); diff --git a/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java b/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java index e9c65e802..e32cdf9d9 100644 --- a/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java +++ b/src/test/java/com/conveyal/analysis/datasource/SpatialDataSourceIngesterTest.java @@ -63,7 +63,7 @@ void basicValid (FileStorageFormat format) { @ParameterizedTest @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) void continentalScale (FileStorageFormat format) { - assertIngestException(format, "continents", DataSourceException.class, "exceeds"); + assertIngestException(format, "continents", IllegalArgumentException.class, "exceeds"); } /** @@ -73,8 +73,7 @@ void continentalScale (FileStorageFormat format) { @ParameterizedTest @EnumSource(names = {"GEOPACKAGE", "GEOJSON", "SHP"}) void newZealandAntimeridian (FileStorageFormat format) { - // TODO generate message specifically about 180 degree meridian, not excessive bbox size - assertIngestException(format, "new-zealand-antimeridian", DataSourceException.class, "exceeds"); + assertIngestException(format, "new-zealand-antimeridian", IllegalArgumentException.class, "180"); } public static SpatialDataSource testIngest (FileStorageFormat format, String inputFile) { From 802b1429451a5f0eb96f62e41809a8f7f5196ff2 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 3 Nov 2021 17:15:37 +0800 Subject: [PATCH 175/187] add Fiji Ferry antimeridian GTFS and OSM test data these files should be valid except for the fact that they cross 180 deg --- .../com/conveyal/gtfs/fiji-ferry/agency.txt | 2 ++ .../com/conveyal/gtfs/fiji-ferry/calendar.txt | 2 ++ .../com/conveyal/gtfs/fiji-ferry/feed_info.txt | 2 ++ .../com/conveyal/gtfs/fiji-ferry/routes.txt | 2 ++ .../com/conveyal/gtfs/fiji-ferry/stop_times.txt | 5 +++++ .../com/conveyal/gtfs/fiji-ferry/stops.txt | 3 +++ .../com/conveyal/gtfs/fiji-ferry/trips.txt | 3 +++ .../com/conveyal/r5/streets/fiji-extract.sh | 12 ++++++++++++ .../com/conveyal/r5/streets/fiji-ferry.pbf | Bin 0 -> 84408 bytes 9 files changed, 31 insertions(+) create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/agency.txt create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/calendar.txt create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/feed_info.txt create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/routes.txt create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/stop_times.txt create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/stops.txt create mode 100644 src/test/resources/com/conveyal/gtfs/fiji-ferry/trips.txt create mode 100644 src/test/resources/com/conveyal/r5/streets/fiji-extract.sh create mode 100644 src/test/resources/com/conveyal/r5/streets/fiji-ferry.pbf diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/agency.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/agency.txt new file mode 100644 index 000000000..cf1f80ac7 --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_timezone,agency_lang +FijiFerry,Fiji Antimeridian Ferry Authority,"https://tidesandcurrents.noaa.gov/stationhome.html?id=1910000",Pacific/Fiji,en diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/calendar.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/calendar.txt new file mode 100644 index 000000000..0c07264f8 --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/calendar.txt @@ -0,0 +1,2 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +all,1,1,1,1,1,1,1,20200901,20201231 diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/feed_info.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/feed_info.txt new file mode 100644 index 000000000..998d039df --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_start_date,feed_end_date,feed_version,feed_contact_email +FIJI,Conveyal LLC,http://conveyal.com,en,20200901,20201231,1,contact@conveyal.com diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/routes.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/routes.txt new file mode 100644 index 000000000..afa999fd6 --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/routes.txt @@ -0,0 +1,2 @@ +route_id,agency_id,route_short_name,route_long_name,route_desc,route_type +FF,FijiFerry,FF1,Fiji Ferry,The Ferry operates across the 180 degree antimeridian,4 diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/stop_times.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/stop_times.txt new file mode 100644 index 000000000..64590d78b --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence +out,00:00:00,00:00:00,0,1 +out,11:59:00,12:00:00,1,2 +back,11:59:00,12:00:00,1,1 +back,23:59:00,24:00:00,0,2 diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/stops.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/stops.txt new file mode 100644 index 000000000..602b0987e --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/stops.txt @@ -0,0 +1,3 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,location_type +0,SUVA,Suva,Suva Ferry Terminal,-18.135791,178.4234745, +1,MATEI,Matei,Naselesele Point,-16.6862018,-179.8820305, diff --git a/src/test/resources/com/conveyal/gtfs/fiji-ferry/trips.txt b/src/test/resources/com/conveyal/gtfs/fiji-ferry/trips.txt new file mode 100644 index 000000000..ca40683df --- /dev/null +++ b/src/test/resources/com/conveyal/gtfs/fiji-ferry/trips.txt @@ -0,0 +1,3 @@ +route_id,service_id,trip_id,trip_headsign,direction_id +FF,all,out,Outbound,0 +FF,all,back,Inbound,1 diff --git a/src/test/resources/com/conveyal/r5/streets/fiji-extract.sh b/src/test/resources/com/conveyal/r5/streets/fiji-extract.sh new file mode 100644 index 000000000..6493d5b7f --- /dev/null +++ b/src/test/resources/com/conveyal/r5/streets/fiji-extract.sh @@ -0,0 +1,12 @@ +# Commands used to create OSM PBF data corresponding to the Fiji Ferry test GTFS +# Process is a bit roundabout to avoid using bounding boxes that span the 180 degree meridian. + +# First filter Geofabrik Fiji data to only roads and platforms +osmium tags-filter fiji-latest.osm.pbf w/highway w/public_transport=platform w/railway=platform w/park_ride=yes r/type=restriction -o fiji-filtered.pbf -f pbf,add_metadata=false,pbf_dense_nodes=true + +# Extract two small sections, one around each stop on either side of the antimeridian +osmium extract --strategy complete_ways --bbox 178.4032589647,-18.1706713885,178.4764627685,-18.1213456347 fiji-filtered.pbf -o fiji-suva.pbf +osmium extract --strategy complete_ways --bbox -179.9970112547,-16.8025646734,-179.8150897003,-16.6356004526 fiji-filtered.pbf -o fiji-matei.pbf + +# Combine the two pieces into a single OSM PBF file +osmium cat fiji-suva.pbf fiji-matei.pbf -o fiji-ferry.pbf diff --git a/src/test/resources/com/conveyal/r5/streets/fiji-ferry.pbf b/src/test/resources/com/conveyal/r5/streets/fiji-ferry.pbf new file mode 100644 index 0000000000000000000000000000000000000000..9324d8756305c50c3d8d711b404c9eb812195680 GIT binary patch literal 84408 zcmaHQLy#~GtmN3XZQHhO+qP}nwr$(?d}G`8{%_CwHiy2X(p{B9Qk?(*0Kh^)$}7uC zm>QaxI$>}i&|tB61 zuQ0Q$F@pdw6vL_m{0|ZMe?%gNE`}JfyMPEm`5;)KyMP`!*B!1Yx~rS~dA;$w&bp^_ zxa%@HkqHQclq43?mX!hl0G`-DvO<);TTHkAH}V|a-Q9okxS1JC?Ek)M_q5)e*Lj;> zHskrjp+qZvpW{EDZ)dUJ?EPLB$@i4cvF-mpWux97*UgAYy#H=v=lT0zF2vvL zd>zlmZoE;>e%syew!5B2#q)Q&zAkREQ9dqq>hE@nu_Cl#kil?YIBFue$XW zkJZ_K{qC}}+4+n2H%iBZ$rw-R*YD=f5F&laPL1ujcK1eqVC4FYSLXOVd~Ox;_8KYPV3l?xeG1 z*3|X=ovvPIJG(2zuP`8+?58zSv~&hosHj8+&+ zEO%4M0{yux$jfcTejvA0XAVB2ttJ4)HlUPnb_k9(;I{C|aJV|*v)F5%qgToEE}%=! z#TGpUc}T8dUG#T=F~YMQc||^VXt!3YF<_FKQt({x8phzD0TB zyytfgBp&Z6QXco4#(0s_LjSh)O+}!V`2Q;TCfE!ZA0u%CrQ-d?uM&nILIZ@pW)~H ze(s-Lh9l4Wwr6@{e{0wU(Zw*@nRu`pxW&?+1Hq%%5GKDaxc}iM63@e-^g_A|z@K)_ z!R6n2g6?fmyvd)24DNj-TgwgqG(Gp^8KdS>;hK_?Bw+6!9WaN(1!ULL+pPx~^3D#J zZ^zp$UzofU1bJi{X+R)>5)`Yn=q-*615&-}N>*pl^g z4+P)wp0Fk#S_b(rZg^vm07@K#0k9Y}5yn7U?g>*NzwJ4YC2%kC?>&aUw2wyxsgpf- z2jI-0pm+o5VP+?QRk*M_Kz(?S`g~3`v776fu2(JqyeQ_@cm^Z`5YZcNon4^Wp8MHy^*2}m%pf9 z&+_NTTx$w^@KFgBh3<kp6bMOqmu z)E)8DCDCg|T>4%RJ|0il+vp!KGJZ#iqaC_`sdpM%b2MAy;TeS=Oyu8`klb}-9h!}? zYXwU8P0pGO0@?eRxVBxHJwWc1-7QxM1;hbs-r9a(oEC@0stPHeH@C}qam6gelB4PF zr|7}B(TMs!GB5!&c%sP?Rm)!$MO+-UZF4QD3FG-?b_WMe|Kh)w1S_fjAmslYjnIb? zxeHvmop<)K+e`ci5p>mlk4%$N%nJFDg*t-c+Whp27#v5w(q5d{D}miKiG$StUDI`1 z5Cv?;>kV55cc?(pDrQYuFR8Bk;glTs%ar_L@8>=8?+JRN{(G*mw5?3RbG;}`<6Po1 zy=zP%_Tc&59}ZWte8bw`_%(WT%4tV0-~F`Q+spI10>YR6v3|XcJAqhH-@bbA6Dq+G zoTqmhM62euo}{ixL_vNPJBRbk|Jy-8Il~YBjVF5n)Ub_fl86#}aZ?pJexOTD8%3a=u^y8>)k;fR{mZXOS$ zi;uvMM}TzKadMQ%+KLwsW@{??4*0K;rVy#}F^MP*qI84;uMf@PH=m z#q8nqw#@pE6Vxk)wm1L%BY#wWc|$*-{$l=vZuiI&TYHSPTR3Uh(%(<(sWLE>-!;a& zGTKh$6?qMMJqEY$V{sr1V23-t3wE5fz;P!8mBGdyjDxaMPkHt8S{|}`lbV;3Lji9p zv~Bf@#TMFrfRS|L-nz?|yp!!@xa0jwJOD9u)(U&EsLfM8L zPKT15gtN!bI3~aEBpNTTb@{?Yp}{n=lo8b9<^#)1loC+Yg+Fo{rEIDYzBF@ck#nqr zO906t=Ss4$^O!Q$#i@tFpJjTE_amaGZwSjem4Ams{xbo}v#Xbscn~~~4_3se!F|0O z9`_NQaxwiNG; zdtq4W6F;6aaR{1u2apnD69}O6wC&w2OsC}bJ<=uz=F6G0zwv8cjG#Baw%gSdJr|Gg z2VGYFObIYP6^%&&g52Uim#OWe@o~=px8>u+a2j=;pnrw;P`Z4Qro8BV>?7RS-^}3h zw0VDvUh!2S(kEgp>OI|!u`s2DxlUPteL*E63TsuS7+GKCse{`; zOXOc7|5{{<^OgoX+41RrNNVJHc9bBc@Tt?@2rpXipth%LOTeC=UHn&GU&z@qeQayU zKP>+!YX6FDFWMTRm(;V&=EJ=-<*JQit)Fh+-v@hBhkfzV5@dtAcJcm!T*0|@oGq2c zKWOBTS~&lWZa)yRp=~AbD!$=v@;=3Hccq*QsmGTi6^V<%Apcl<)-~2XGUT7^_}v*+ zdsA}zkeyge3k?yY|4QU)Ir$}4h;yf_r~C{@I&vZR=oJVEUqUX@}apc;JWr z3Mzp=#U1z#-}idK;<{n4P3S5P2W-DZqJg6>L(jn6-d#^RQoMrn(GzQkJ0`O9+>L)dS7w= zEbm=y*AG- z{uP!D%;R5^`&8g={hDW1bT_s4xX*K!hWp}_jgAL&dA@IZmkd90?iVmT{Ay{F7A|Yw zL56PI$oTOvP!0-aF)95g>8rY~!g6r4J9@0(J(+5a(s$Nf@&YmynAX|0xv*{x0a=&l z`(9gW+@iGV=kiD&kIpLd0vqa3BJ_7opYR5<-S7L^+k;;3;&AE_#(SLMiCgY3gL4Sb2tD^4zmj18qmz!dWuWFhg7=~%$ z{|UqX;If%UN$L5x)@jsRhcp-Hk=rD>pP<2G>g*5*349q;4-SN~fc|5s0D zp#^Cj37`P-g(4hJV%7EMH;H7Sz(boyQiWLkWMY7R|hxEW}oTY_dy5LJL$hO;FP zdqyZI-ks{(VJam_gfBO1x<=ah^XHcJLJVCJ}4dRe1XW_`D zIjVpZ4Pq<-)Oo-3$ZPg*6i((0LSj3^2)w3sj=gp5uQXP6Dhza6Rsef)+{AT6DDh&EW^ooM_<6ohXg2{matG@7#ZueXDhX75A-F zN5~ci?lh&!OFGJc;L_ZL5|(8!*r-K{ABj923U%lu@z_(iXAPR=WqwNEQE7uu8AsO6 zGK7kT;R*SkR#n>|Y{Lx#REg)t(4AGA8s>RWYl62IjVkI@@YQrNiv2D;b@J$p1uT9h z7lv=s$@5^d<2TIyh0=_gbqtGoG>K+|!w4KF)US!)H}m;}|&)2&ri4Q0RH-WvlD`7?(;A!d|H+-J<`pB*}&Oe1?$f=!c-rH6y%K z!Rl+7yE<`o?9ouRD6lX}bq4B$9brVFD$XkhrW~IJX?(kw5*& zgWC*NGx66-R;1TFawvKMxq-5-oRkGEq$+cg26viIkDi)0sZy_O`7!sSH$F=T&tMp- zg3rp!vjF~0?;7Ey@i3o_Cx@zL^-{U=kk;a?EvUgajW2QlmgDG`CX{IfaR0lYSQ&j% zV{#7nvg`0pcpnh5yp5?4kEZr{p`3Yo@6yYuM7nJF)|Hb8RL@mNDRR zqBg>OXi-P28lGAJ*drSU`>968D2zcgECIq=YeQIR`jvpJ@Wwr@FePZIJ1oW>!%59f zG4hkQmtjH4z@miAPhJ}wzZOB%YmJme?>WG+DK^8gsM#{?_E76Kn%ApY1 z7rSt@3DF}hxQvRnNI3xi4g{qkh|#Du3YV6Y$}ZC2$>=7ML;qqLF2vTHpXKKCV3X@S z4~`I49iNp$?=>kuXyTWRc9Xpn?Md`(iAJ4SJsXJz+Vr)TzF`Fj!PO*be#e{9vsR#I zpH59e9@#NOtq(a9>r@bRNUR}eF#_TY{1o7ajN#=rfD588r|xtI`0^L|49b);nFDLA zGZ-l&jQy2By>;gWxbUiO0oxcUl+N)nMu=9g=vwe|iuo0l3w*|5*e1|gvRb6`DV&CQ z;KTlmz1cJKTRw$|wq6{1J9JKLgxhEV==C@C5PqwkG#tH`+J+Z7QpN{&euGVNETW0E zy!nSLh&Lc(sJ1}Ww*UHAO(%;N_31w2ZRW3o8uoewIDloHFm0k`Ah=zWKBYpfGOfZc zZ{K4ww1oUwsoHa1-T+1e_wP?Fe_7y}B90I=k6;n#vNu=msZ7W3yc<^3{d0e`Ax;A^ zeF!OIQ05`z$OpR)-57g*-=i0yY}qPOd#{l%XO2-NYhV6eeMGu`oP;0^-^CK6|48np zd5BAM7TQ2nDHVk0uVn(CZ;2A$u~2iG4^U0NrGJMBtVgL(pVBB34lgBm|7vk_;6b8E zCqKuW!Ud7s;_cJ9ZO?*$<)h8UBb_Z`#RN|kn(nujmygrQk} zD^fc*V0G}JST4(WB1=jej6zAae0Ue4d<#PCGr&=d7zY+$5|M!dD)-alyc90=8rW1K zhvHmC2Bl@#caA1nTo3QdJNNWed+e%6FZKHEyVfA|6Zpl9mqWOY7vM+uF^*hFb!Ii2 zFIy=a*~HljMXY&n|AEi3g0%(pm6A)3?qJi36*9Y3ZRDZo_l_#wOKJ_O@Tj9eWan;# zrX0)F+*KDZd0uVflK>O@+nT~u5uA$FFqA<;r9bT16X5Gf%^i71U<|ceOT&GQ2|%P= zxm0zeV7tNx$5>g?p6ICb1t9<935nkx^3>%7+3i$jrFXH(E zv&s#ouSt+?I`%fcTj?vEB2}&~Sm4g63>!kioF>6BMzjx>KXN%D_Pm6w@tjh)(jYcw zM7L~l2J*Zro-MvG5giFA@GrB6A>rrXy5IS4{CG3gsN~eB!{FoY>2_fMX-R#2el2wX zjz_Oft2Fqmc3<`d$9_$UYqMIz9Ds@gj6A4gSY?=KL^GKpZrar*IK0N5Wq{Yc8yJ_k{)m) z{o6F&8q&qPEC2+^C<9tO@iwJ&jr8tiF)35BWh`dz+SoY)s8Z)J11W^|pwtRbM|4U2 z5D61}v4AOFgxONa%RAApm_uBZ@?#oJ5cmcc&ahC>J>6`P=skP6zqXMknuBO*JpV8( zx{04y`LtgB%P^WtXemb&D*C`@vM=I7Q%;7-a}&|P`qxaR1g^^%7RKwgY+O2F&ZGe> zSp@Et!4vhgOJ3?jHAx{}X9omvwty<9V)&?kml+XSA2)PYg-8V^!%qk1OcIWX24%|WW<*dcpP>tfIN^yD zJoQrrdD=XaC~YGVSNOD&#oDFn%8|^yEo)3FF0bfjK`dfC=EGF{T=OzV?X_rAa_OJj z)rW3rCFwJVrzgQ-y;(SpTuILMdb!c8w(Z&!hWt8EP^ zWGHObsy^sgV4g8)TEXF4^)(O|!BQm}^8mKCq3eE)Bb5$kg2M?`{`oiE<)}%hV_$g5 z80H)_Z3D(O)U?4XK8u3gHBw3zMfyF~NtdQq4nZCC>g=VNB&)S%4~^VHjfb5w&zIBO zv?z|bE{0q8*DMAPR6AnZjxG^xqf$pbNS8u(d2mNd+V5^|Rq%yEK64m~Ahbja?VOe79aeQ19#eN zw5pbYI#Quj;kF2^@CTY2quJce$I`(tR44{lB?V#mB2Q5a{ZeWVL1@W$6tLIo{C zz0TaUiInrFFL4AbTO&}HyqCnG2q(CboYa#MeQs58^mvrY3rBB}_iD&RH8h=pWVg6y zC@UGHnudaX!h0-%lCo=YiFiFZkKg?hm=ax`wY*I5qgV|$%nh`V&`51MY>GJU;8K4R ztF^KKg8Gy*ykr`XuTWZwd7uG^CBZd+=Khs4Qzr7aFWLqRb3BeHm87v-_(ZnxbmPd< z!G&E%bag1MkM0Mis<1Eu)jA;rDs9AChIG%b zLyJIgT2+PS^u9b^(zaRAB3#Ea0CXwJD@$7Jzvgj{#ma)8&cJkWkUVfDerFZNkED=pBt)+T-6SW{i_0=J-970iiL5-1jK^?OM@)M* zXo=~4#*sd-w!T&xHl?@C5PWWH4 ztH{~q=n`mCVj4El(+a^FuIQGBQR8#{G@%+!8uc5Ef@uQxs%c<5X|mk?EN+|b>^tzO z8N)bQn08P`R0nFr1;K`FQda-!y=SVWYZW?Ac1MJidma+Fy^znrHIN3S>6H^7eNpAD zpQ#ech`%ACbj}Y)cIr5}Iqr+AsttioP3p8dY4bQqQ_9+~W|V6dtdn!DR5>1G)@Iam zmLzp?^^>nhI0;ZW@3l_j2O>*U+84h}nLi){*)AYungkyKjC-6wa!gsxr#6hDJLd|V zHdx|#+~-TA&0(j~Gzs_5{ET3}g6b|gH7)YY;zjStqbo(p(C-S8h^@S6F|tC*@uik? z6Mb6QR#i@+U7_7zO;YyAIwQAh&0Pk6?bXnWf|Uy8Y6LQT(yS9SI08hjD=Af~9Vo>a z95bNIA(;X5Ng$hCrU77)RwH6id-|0_)$;jh62~D^H5Bj6;&B?}bk$&#P}NQ~RQ|}= zrfjjhU!s>2;4saCE?APrX}+k!=r>>E2@vu%0L!0k0rHUa!Ueia{zGy)z+Dsgv;%p*YHj}gBFaD ztRPm~%8oA0zyVeCw52@zFiYAQH8>SEbxHL%b?XJTzwwZ?2fw=?=*Lbc=*K{*(z7Df zPnPD@@f8?&8M(W?8{kS4$zDQTmgYh0^qLlD~k)#~`r+qk};<_5JUmgx8?7O0C;B zW*4ur%|c`h&?NsIC)&S(ZN{fz+x*bK&OTkzLR{DHv#G1FDOsn_99_HTh8|a&P~{#h zA2inl=VXbU2@d}KQSfB~^+BUtxH_*KQ|^WzqcpcqP4C$>VPktzs4b0`d|QRZsYt2P zs81YcF~+U^a2am_^W!(7Vu9z-Z(wDsi#`|Gc z-USP@Q2T4hq8hZHxut60%rcAOxMD+WV;H2h{=%!{6+%8Sjy@P&1`qQt%~allL(VPd zupDGi#9z?{=lSH}f`=FkuA@D0E=^{koI^+-k~aw048Be3DP$1`>4`fva%q6&s(iw= z$FqsD2E_-ytZ3N+ebq)wm=&aIPpSm#S#mBVaOR0LN3_wrBy@+ip-uYW5@M*dEXKR6 zE#uQun3JzLoggiaIIm}wAr&4M9NRivDZf`G@zE~==1~=zEPBy(5y73vK9+7YsTO?h zx_nOshQV_I3rcp9ySn>y=g936HuPxK@Q_*?c|lxaOpL*!#JKwr@DM}5_e4bz7$o77 zL_`4ob}(1N*%Z=)252(oC(K(FpleX{x)>9)r|=KZC)@c?{5&KM$1V*&m2@lEoh*>2 zh`=-fdSfA2WZ;oFE_fe2D3*iR6yABQJ$#t4p2evWY(|i$0^kxNMvJf{Snj4Q49=Kg zIL>X1^r`Jbx`*0Om+y^Ed53flVGgmJA#a7D6FA@hz%dNm=wqzM3Qc_tLeB@w8z@&$ zAHc^9z}}gu9t6|~xAOSObdc*N-Y7ArE~rzYfjP^D*yN~I1GtSF_h&T?ggSk5h9ZRb z+CW1s2>Qlc%-$&WL}>Wzka_XgpMXfJ;!x^P+m3mafr$;LTEA73qyk>KJ*2$Ltap?u2WA;-8o`u61RyWUm%EJ$Ib zxmOC8-RXF~FoU%W3;<<0+%74hCD=9sY#UHAhr{k^nQ|{CB~M7Bw?QL)q<2BKaxahE z9LwKZ#0|r!YIJIRsxK_e*fLnfjWemCMjy2wKe0Nk4lYNjQn9+irLF>75-`uYV}sY! z$?_Th0_8qSXkT0mB92Yqt3~s!y09_>-E~|2`1B^#9Hk5|-bXea&BxB6B&4`)Gy#bn5NcrcOh=bmhJ7ElXjd5uo*r zh?AtWOl;JW`gjeP5V2WF4}TFt*O`{^7}7#f-z)a1vE?9f#%~&sEW&S~0c>kM+?pA1 z{M0A#PEii3W#C=WnP~**B&;7l$NE2)Legq>MeL&B56NH_z&YvPnA1_c%t3y;HhRi3 zoq*c)uFE;9GY4+^SJ!LR?4U233_0wz8OIT9tp%!*5Ug0ggF~Cu=PdINtjL5%2NCoUZJ@|tb zGojn_Gg}PbIh1FeqRXd`Wg}o{;xPyF#HS}Lr=~!~DBX%{jk?HM09xzcgLltjCICv~ zla6iWVu*^*zk(yMuYSZ@0?tMpO*9&f14`l1kfJ{Jn}@{0@IU>?Wr7JM;JmS!hiW(W zJ`xF4U?j3d%Hc0RiV+E78f96A#2(kTzHN=NIq5G)WxfS1&kAeg*?0wmMgR>z8D<3W z`?YJ9WHNxLH)K`_&_XydPE{Fox(2<-96}7QZAp8jxBNZ!=l0Z~ylPQr8O~seQU@gO z`(Hwu8T+WNbqC7Y28h~qQN|vQOc^g1{dnN;L8*JdOzq7o7dF9U&&;PYAFqy;IGr_c z-)Bh}rS1ILh|*)eeUsHgMgvxUXoD0wjf@~^NrvOa{zcIun!7<0f4yUMD=mn9-N9Ai zxton=Jz|9dhuhGRR%YQ?n`aV<^#q))i6Z8>8fmfatMlQ5mQ0WK^#+_M<%yghDA`1u zNP3w%nNtpX_#J@t{~d9HN{8MYU6~~*IQb)BfP+d9?^5ZPq0!RX^!@$!)*}R`@Oo$FeZVot@tr4Ek z7AyxSI!|fo*bji8kP4&GmH-T&^_7J%PU zs?|B3hqVfavw*~nZ$so>f>}erF5yRw%sQ+!$CAJ=Iy68@bu-L}fD^B^m&Z=hoT{Di z^NM*VoL+NQPEwrqOwDMcnZs{DBL2GZOCBN0O+0YD*^Iy zD}>jx%m)eeU?@kyAMBYMPf8PEJcwmC_^5#$!IaGrKi!-b0kIQ6|HRY);XWf{4w3uFWvM3d|;$P>f?ZH>VH`qW8`R zw$d{f33WNWN3Oi*E+bep^3p{IJWv#`coWvc^7&^013V|f)g?EA$oPIz$;Y7o4~26s zmMBQ&)LL)~d^d`B1!7I+obfy3rG_VtbyJkJVQ5mO1R4jvjl8DypT|#E1IRn^hXbT5 zxl*JJ4L)X2xvl`eQrL^})dras82I4Je0?{tItNIdc71RFquDe1k#)hz;o42CpUsHu zA{RJ^PL5AK1Nux_klJF94x=+d{yy!g)mEtU^MYatx=%kq6{5Qd5W=biZI-yXb zArYshkQleo?|d6C@}yN#^Kv98;kTgV0CRIc^aekLi>#0i=ZpAG3la$4{ADGp9-8l67`u@(a|7_r;LT>w9q8ggN5oddb3~VLKp-JEs-_h=Z3kOJ{Ik$(M;@xdavXzWNSK&~9vzE9`*xT= z6fW2=I(mu|X?4#Ur$IRmm4*QwB)CSWVET2;qSef;JX|k1F{f*|m5O)WT5e_{26ZSE zs?X~EyYa8z2WJ@%Fm}cRkfftd-E#1j73bDqc~5e?cPNr zQtV*5Eo68z9Y-%VA&&3x^sh)|QLRpk5P`P7^`St8Lo9j?T9~P0+I(#kIVF9|BK#yC z2w;?<4t1Ls{)xn9ZO{^jSWu2UZn*=7YZuZ;c2P}6zAa>q{3uzISR4jXv%MT%9?y0% z*pFc?CR<8uN6zNj%n5CeX0h5^&Y=EQ^9t}+hOSe#x$B-4r>b)S9|XucQqQ{}iLmho znDwmMhR^g-)clp&ROui(&WmF&iE8teVS4*)U?D2KcwI;?Nd>|i)ykB3M6U?yp{QVt zSb$OJhA2}P|7mK8wI+R?z7{{fSJ{_l`7$C*y2$FRszTV?2vM#DN~U9-8EO`xnE`OD z68=h!SbUZ<@JcZVYNCNA>AT79lb{u*$@qsX&Djt^BZ;HAjUV!~yFD0Y|E1cl&+^|f zl*{oalYD2s-55-ylpb<4-ExE)C>InV@HtU`7C*%&4M3KRY*0jUaEQ+;x8C~74LU;3v_>t`0lH7hcxxC}YiYyiB!Dv$G;M}PBUPt%p? z-cMf4rgbmdz{7gBeo9VM_qKN*%yUIbRw11bxP1aD$V3mLCENjVTS!JsgxMd+*YL>0&bPI2_0fwpISgIl35Mcb`)wL+rf`U zfzta^c0Us!XYSAl6p^$EA-nY;4ry%$@pYQe3 z>L+{cN*7Brrly5Bb>GBSf2<$hggTFDJ*?9ei%_R;J?RhOz0Jj+&QT4X^ zLhAt;%7yuJh!js?=txBGB&N_036v$s97uU4c~7p1PUY9$7*^#h~TLYUYKR~DFRo;p#qFy-an z0ZezIp^mux8XYH|v#&N@>UjOIS*f9SL>gq)X;?>YdYJ_}2*gdV;1gSpF?u6UNU6<2 zA91nMR#HEJtU%UcKzv$A`_u1-OwNSRcP3&^W9=S57VK6e(;~A7IffijzXtPD6>5+H z*%JI_6^F4U*5)J9TJhz~*Bs3gKa)()`kHQG2sRY7m3Cy9*wEfc6Ea?KBA^lt%K4mf{FRvb#^sIGsNFjFlXJ0G>D+ z=P5qcRp3Ci41Ve?)?l+kU<(?o_|)RLjID=_rwbL}ix|xzcz!sD0V&LEf;s56CmP^@ zgFrmDP}m60laYn)gi)7JR&efj*hYe!EwXj-7R^A&wwLg^d|E^5WA0i8Flj@M{bUJN z2=QPzw%rI=;KgIGJeup{YzCYP)uzOW0c4H7`Yk8(e0iP)!YDJDrXdf!@o>*j%7}Ei zKg(WXz=ML{*K0s-@E1l^|L1U?ngGQS27}K7P!U8^;f`!TWGB=&f^|&4Yfn9D&Ioqn zqYV7TIzDivMYe=C&_7E6N|SPa=bkyf16trOX=*NWIxA^cQO1x`fw-#{awoQ#w9 zIU+!)Q;^m!`bbv9b_~D|9bSZaUSo+Y?ttMp@Rvl9!4kX7(LrWD4KN`i!qkRQd=Q9X zs|lqxS$8gY_0VAf%tg=_>?~P|QR2$N*MzqTJ|?hSTCgyYJSFiM1Kk_9;}?fcD?zYV`~T_<4dJ7?-*GRW~HbOKF&%i45aVrXFFM{a|kY?x#|su ztCv5F<4}{gUJ1^$Wav8B)CZV`zIQPW#3$dLkg^eaOzRTQ{Fd};ZVuzW8zd8@#j=>X zg|qO{(+w*UdF19Ha$NoP(E><`qh~Q2ko6=;V7LapVuR^BG4Hu|6{x1c9DdwZ-BS+f z6j_ze+E8VMkRUszAxir+O1OEn6h*PwyFqp zEMj@C(PByckY*c;kgizNHv(hKV`4!U=sZ)4baI@13DKSuZpvyv=qtxr-XQFEhbqMU zaApRUF_LB?G!s-};&fNwfWbg7uy(Odu|lN`+DlFgZc>#`fd6s^G?G)k3@qU)t}xXV z8zj_iyoq+Qb06c<`K(6(N5mee>Noos_axPP+MNh)%Ccd16#Y_>A5t`mQqK&= zkrszCD9$?7haUKT%-2$cxp^c(t~HnOrr#u~w6i^a+ksI=bWQAwQs*D#UPam}!0_#hHIak?ijLg3nzi!V7IfjawIyx zN&g+z(<}9D3^&Q01mB#MXY5}4y(Z}5`n8Y`f>IK;h|ahW0FD%brIU}Th@llBLU?ly z&6y)*co3$~I&70sZODDFa{0HR+V)YHwW+b8#fKMwj}fHpjWVt_pm8vJQS7x}akSwW zuT{!oMAvfwgv+BV6>|R*|5UR4>xK;qpw^13>uDC$AW9p+_|j$TBfU0wXPQ}*%Cccc z2mlLIH9~U~GLHDxP`r}VKZQY~!5@qfuKwGeecauvx!;E-=1zO5E;LEj;l%jyh2wc>Twj?i~6z!;KPL-k80qs&SUc*Cn>ve?ZVq%BA}=hR_1!)h|s97Ko~d zK;O;>_4t=H)pr%nS)2*zFQ2J;?B>42o_s06VbdS!+c9 z%7S+RGE;ZqtKd{t0p>{Jwc(qV!KeGSG>NAf`99okBf#xK5WFC1g2=Y?FoF26f{eBA z`1s=i^Dn*=G8>V-p*{f}KKH?3nfT1GE%CY zpm#rj-aBSzpvnBB$;NZjg3g01t_H)Ly~>3F*%5t4Y4g>hmGV5*9zA`s({k0pdz3Qg zyMIjW!B-HNB1luQ$chu(t-oVElm-u7^b@)7Xd;;QU5uxTpe4W*5o=Q2G>kW6oiem$>PsZLaRb^e1RC|ayOjv{qiizLXwm;{+MD0-d75niq^2QwES zHaYn^20EJt|FuDfhkpekbJ)c(m_W#1f{`>{C%EqzEsco9RK|ky5r`-B2rmTRG&YCKjRk)W^&I80c@vd8>mhtJzfTcMHjqoNlFTF<CTsWK6L?_tm>^_y-&jX z>0KFmNJ8$Sxe@p9k(s(~U`o@rT2&zqKEKJMd=)jLskqG`qp{=hB}Qc-BmVP>Q3m{^ z?X-dNW$cRZW`@-7yK;#+wRt=C$aj&-ixcXh0SAtdE+6aT$tkoCdIqOmcs7fbT#%LmoZ-SLNe=(6v?Lqsr+7%>Xi2QFo*r#LE$X;UCWwElv{q zFt;Y6;2})Voalom(V2&I0Yz(GVxTDiHv%k_AlE-Y^#)puh*`p01X@8ThkS6DWgM}= zK%n18RNlhLk5=tG2~u3GEdy8&XN2QH*RG`1VW$CbQL-dQH|Qp?@PS0Pu3;ER!KQlx z3!pf*K&h6v_5V8o+|9l!C3ATt3 zsP-*ea|&|6c;_G=3Fbur?tTI`jL;e)cMG;XsMJ9L&I5`DAj{D@^eVl*I5CxD-?QVC^GbO#+^AkfKkkd-+}*DnR>qK!{kL=_m^b|E zSWHDr#1^^XDB*|q$Z_K-d)v5=Uj=V-+bkn4$lZ5&kVy+WmJNfr_-5eI<}@-=a>1!b zbdd$GLqF1|^|Y>*o@S*2F}>O6!?uTrmjqI}5pW=D;=BCz)=%zHA3R;~UFZ;8rI8Z~6TF>6=&5Dh|cl;&2?jJ*$#kZ0N2ZJoIZ@u;< z@D&e&Z{<-wZMtr?z|LkFG%x?ieBcm{#b0SFZc-x6=YqO9oLeA8P4eYw$vyZe-R0Od zSu3O@vcCj|JhwXYyTBF@)--C)l*id1WwiCrhybmcFH1y}!HSY&lw=jAPrCi3Z@}j7 zq_OHhQ7tzvH*;*GNFNj3uRtX@a;WwIF_po}kF9C?(g39Bjofq5gvGW%>rB&>T#Z`09U>I5mOUuPvucH(_4^(x?)@sl6rP z%7YdhRq2w{|8z?rN1&n~ z{$Z^dPN>>T#)P@r`5G;1na{Xu2cBF~&VDyA2H3wab~5xvx|Zon>F;D-t-R&>hlOB} zKMc%|E~$*&^F3qe3X(lx`p1cXZaZ&#qD{Qo%X0Xj(H_hnvr*sHp5~6KHET4!)O@PQ z9BlW4JdKbu!-{z-0Vi`*0^_h$s*&f)tduw_yQ+^U);Phs*JdR8@}`6hxh}R$MZ(|L z*UY5amrz@AF?xQKKZEu#R@gP7F~Ne^kekglm!FatD`qR?OUAScwLH~bHC;sua=EHA z;#8(FGX*+<5Q&4b&;APSVVz;&x=PBa3W347I1><#gkCi|lp&~K%)VwMaN#cAu+w%# zb0s-_xZMXiD*b{4b~ndb)Gt_W2YRRh+67yvDa|(Ak{8G584 zzn{fo$pW2GNrs#eU8nUP@?g4=5g-!8Gq#9i%#2lpnj)@LG9wZjgjHA>y002H<(u>E z$w8PxHZ$!cgW{t7%R(C>fplV6)sv3az%Ug{WJl5e2++%G%?2KCbT_~U;$hW(L6RhS zlAo$G@&DTU>iD?MD{a4*VeTD9fhAdBnITrAnX}7yycfUO|8qK})!a3(X@I0eGBMp+Gv9*wn9vYK}7kAA?)S z1Be5p2DKXH;%;&N)n3e&=WYxdwfl)c_+Afoxt&_%<>P^<)ssWhaKPy$ox1Y@r)v!B z&QGbvCSey>V0UXK3>dG0&7YRX0^?rDvQ$uXx5jW!<)Il-0Bqz{;=DInWC4;kahx>> zl1z!pgehueYGs;5sw|DEQyI!UnTL2qXr=C<*f7{LD3Toscl0;TCrA>68R8CuD8n@) zw->@L**MaLc9a{hP7!lvOgwdM>PnVjA7v-#!@8k(;DZIw31|;CK;=m1G&63^yzZf% zLapf4oiEauQKc~xvLQBz&6CbNGNp2o46$x5;OOd}EJ8V_S*G2=tKwDQJJ<-*gGZ@3 zNIKXls(@EO_lSek6Lb>pR4F|)3ch0_X%oarN;F19b;^u)GMJT9Zl_!~p#jPSInfPy z!A;*76OD+ngibHp@Jz@AD4IQY#O<3WO=AYA9xM)65NXVqXyyGOc1E@HOVfPq;BQ-h z>zRa8lkg!m?guPCHGXw>42pfq_I1aTP4RHSV=ks}op|0E&183Ut=6FOL#XNFhA_}{ zx#{xJqhq4MMNs=}UM#iDPbQbi`YYksI%E$y%JMCN<*zeOHrqj?t-w1Aw^%u2zLxJgy&F3T4Cj1E> z-3i&&1urcR^&Dy6%?iDw8#AgZL*FPk=Yo~EOl+dl_h`^#PVzl17sZ_BoHMA-pmbArQM6ohQMf4lbhCffzy>i33LA|U-Kp{2t-#jfD+W;)H=^;Zqa=W>CIn8kH zLfhFEWs4bTfZnp!US+(*3udq}*Ul|YMq^=`&W4p&J#QZxC4X#n08$^ff5BYOTs2|? znN)yuPvCKK!5gGMws(Kj2J*bNZm|EGXv0~Rq^%a-&IgI;m?-$Wc@L()J=Tnv9PrYI zu<}XZ;zKxSPrhpRTwpb*`O$I`oXHevjA%FNd}zZ@F$l5&hAIo&VI{2`XQ= zzt*SwYZuzSn@%bRsRj||pm`OFz3Vf8wVysb=kbdFA{M6GalJ}8r*?GR?xmJ%ZP$iw zcU%uT*LH31X2J~*%>aEL-5RwDofTEwi&mc>XZyw(w;C{`TGFw2rgetf=EQ| z1*`hYI;!SFpYF`-Ins{n0jKBgIJAH%wnB45-FyWuY5>cAV0aZc zzdR|kB$95Jo0EH2{NzqQwfT0H(UfdNz?JC>3p#1f=iTemsyj6&?hv-|hqyB;5t+fo zXby5jWC>4L67j%>H^u3)v5$p4#kJ1%Xt?P?+~akUx(|Dcftz8^6XW^NP8ygrh2}Jq zBwf(XKRJ4WIM8r8I*;oC;kK}+`u2rkc7K?t5SK`Icww?>coZ!p2QMW*ww5JmygYrg z_=#$R(39EX{gCs{fqjFK$3HIbuzC_RYjXcOxGwWOg*NSquCX;}kNJIyp zt9;pY8DFr32IkT7I*=zSpcfC2KD-ArL3*$ZCPz9Z>UtuD?4t&#<2mzn;JBwbNSq`= z>RlWF8DcXSJ3&GeWX6q8K&S8(qB{=?dU!%^C$JNwF?~0CFfJR(8A%-Gf-JwCh`RjH z0$4(;5Bu_}d8s!C>2mE{sk3BM6qo}~+|3Z$3Y-)pY#L)hm>|b{9@&6o3#gCk;%cN# zIrfRRJ+vJR-25QVc_>BN#SfrL&iaT7G~@WAPy6%JnRI4<2yoA^hjU)HBfY2=G*&$U zo-%>06`|E&QRwLk#ziR$AX?iS9YxcLpFrd(2JN3^F#l#UVN`}$(K%QElaLLC^h0rD za4%NyQ%f%p{*;flgQOu5SQ!o|gMOZ00=B9pGpZH+fYS#?jis<%Kb$Qz+W|fLPTB#6 zr4~^7Z|q+xd~K=Z)*6xT{y{m1_^PlR9aB zz>&;l8zr@d!RMTk=vYS(%8ZNbpKYjtnlG1XFE?=&Esx2*v8kRm1_Dpds+<+w5Qhjv zQ4f`9muYuFjT>-2>PWb0Zc^R4jz}vxE*@r!HJo1FxU+HQ_y`B6J}T>8ndXRKPR#_V zF${R^@H-XvBW_udDfiaZP484l!^A~THd~-tMP6vKFJz))_r1pRu`lN1*-rSkH z1tlG@IbuolV(HAW>65Z?Q94)Gx}g&T)ZW!Fri7Zu1F@1+_|V9T*9j$~n;#53mn%(Y zGMVE@n3VZHI3@#!KMlXM{$k;!#2c=PanZOaS30ZmKb^^JSwqQ)DLyORS2a`Xc&v8I zwPKCr8~a!KschjO75PvJcOpzbmWcjNhfF3{Ixgz}7fYh_&D?u-1zbrmN75NvUf}8B z+Z`cLY9?r209|*SIN*2ew*<5sIzGTSkoCCx7}zj{8wHNqsmgMt<-6ND!%z5Mv(zut zc0TKo0g~=*sI9P8f{=55{5olpo_Q)bVL|lCrwuHLX>{HX%eq}-7<8^eZ%N1Rft$LM zWc`T%qzgKc$>b|j$!;$FGB!Zst6a|vJ(vIDI&WUD_1nqMAEHZ`4dKSO6C80!aOOR8 zBoW=el1v`Ys66GR84qlE+ZSfl=PN`-0}8$%&n2kn4UNUrN{YGg8r=n1Z< zpu3G|o~Y)060bs#B+Zi#$gHT1v%Ma0)OXxe5&xR>!2~_-gN7kV$O`7i_PtQ2S`rR@ z8W@2r3Da;ZHifog2e2G|3>QN4AXDg?3f06R_JTn8W|?*ielHhk9fiI8@a*rmhk~eL z*hSGH@X}Zl_6WM@9&`(MfJsGO14zDMNeoiyjPGk3@7eI|1QW93{$Hfwr8Oa&kfkv_ zSn(oe^-vzyhbIiXp6a_0amMeViBhP)32x9ynxpH_SL!)xmEW0GP0EfrY4)92Rq30V zLQnPeGo!5QcSo-WpUZGZI1=ivP8RJ-&8>isyyUgrCt7!)eI&S$jx};j1ChTh)y}`S zeZj;6a{O>vXGK4SJfeI0e;P=427qKAh1VhFgu(#%UYwga0 zE};csK@`}5WBM{c<$8fAE|x}ESVo+ehg z`PtO{G$xH%(m8gOn|j~P9O=~k(k-Y;&rWbm?Q*1Xpc^{gK3AH2BeINMI~Ss?G8lRx zOjIUL5FRP=6h6W!NtV*)LW!zO(?u4eT6HDtUaEL!=yu=Dk~b}$negD}pC7*)`-pD| zltV`$)*ut_6zr-;>JGYsj`sqHEzAlk#2U$2I*?tKRI;U~I%V0)Lq)YDkmYI-><6dO zHUcYQf@n`z9Sa(;7IcKNSHI{uh6Fv?^2w1=?b9vOsJE8Az*S){ScpI@>B&Ph zTXSBd$~W>EQIcO)xjfF__sxujQhiA391B zA1{84q%2=LDmp&CLCR6Wk$81y zN$H!DvVJO+T-A@~szOZ?@IjPoW>AnR)K~&0vN^5uqz;mjTCk#D+5(If-K;CmQDDrD zv;8VZ-YnD1t>!Grl}4&%>f0L4mjL^-Tly>MVB_M^>8i(?6vy4tOI5|&V@7n%1Py{6 zNtxu>gs@A+!3c*ly+GXv$hftn<6Hr)E4KQ;^nG`wt*VmtCrTifXU*)uCXq?`3u6_F zn#V!7D2NwC>TNWEqf6s&a<|z^h5Os~0$biNoA}u=)^y|^qwKLsz1RsCVoy=_#-1ce z=%lDK>av9f6QI^KEN_BW6$++@j=xCR?E~Zdqu#WX)RTp@zG(U9>76RM;97tB(pI`Rn3szO$fT+$vgSi#ev#Dr*$l z@|`8g(pH(XWL0h}@Yb*tMJm_z20GAbskJzgA1_!Gxk`ow#o}=Bq%_=UP%mC!&)=04 zi(@6b5?isg*i~?RCtk2DN|qYSip9~Aa_$)GrJ1tUQ=xKWaV$TTb>%0E!-dn5Xi2}+ zQ86uCm9L8zMWJjvf1Yk+2bn|OI?tGira2f+yED_gTAn8zP1lq8jH}R|zbg052bDkQ z=jRM!i=u2%uBcWTE9vF*sTcYi?aO@C1|_M|MNz8IR_rY4=ST7jsYxQ03Z;ALP#%Yt z1kG%tF#F_U2$Zk8tL?36f2;ILrADpJs=QhwRPHFT=le3nLNf~}nP^A*`NiT|9q$*H zuQejm7tD!4bQwv7qk#hwMU-GQL>OXN``I>KOpH_3ykUvoM^(Hgri^yt$VvcO7X=53 zmPPvlTP=SN5H2p|#}|WXkwcKiqU=O#i6H1)2IQ>a`od}?XkfY|Tl^(K2!DpRLk!_z z)CPJx6t*Yq=~=<3s9VfAhdKT_$PCFdm{ZSDrX+yq0#PT^EMFGe^6L54ylQM8_ya%K zDOP;q7~+|afMr|C^P3LqT%g(s@}yI$3O%Qn_u3fmC)=;h&%&kJ&1$Y}KHp!Qu9JSz ztsbo_fHJEza>l@+3FP`J2dGRo$e08D@&v3DTOsy%4#IF5B-b7j-o|dlZmx(sNnJ6E` z!{J5L5@dk^$Odh}rr6$i+9sI4;SD%Xw0(DU6Kn@Ub8nmGs+l$7k=0mBA!@b?+MftM z#H-cKhWM%V<8$ z2Pl8*=B{HoGkxyr=%mfwot`1-TMiI8tO+@T%p0vaaCCH{!UjBLUOF+NrD zerXSI9))b&p@3E>yBf?{Le_Jj>!akG7O_Z&FX`_IY#W)C7=CXG<8Kcv@>z!{P@E|U z@DwyX6dD1QOo9DPU{wRf+OS;U2N`|AH3A(?LF=!!CW4t%UU3jz<)CZ70!-H>%bE$#n7Kjog5X8S!4ggb@sFEa z%#MQdGNE99;JW~qOWLl47E#0Gnlx6UZUoRQuJ9M2T*zHm&s${tfEmv$o^BPE5^pvUCXx!}f{w!(GtE3AI1IFLg`)xS z6fLVcQ_%+X_$X9|+sRU{G!x17r^A#fXBL&F>XVMXi~owO_h9Xw5*yJBT!1*iJ75TOph1WaF>}7>sl{+GMtYzYNC_N6b-|0bc7NU{ zUy?Z~3)nJQ-{t*}(>&-Li38MdKU0+#i0+;{c*5*r={8v5nUHpna}7L(A4p@DbHomrMvh=XsIVI8MG~=8zAxX|0X5ULlO%Jve4{A!`I%={k+^xZT{Ns-3a@9PT?-)16R`jr z*c6hd+o|Rng{9ze=o0RL`Otg39C`(}LZ;viD*L3=5vs=P(RN}UjYTRUN30swI(Z`k z*3LkC$(ClcjySOlwxNAIxe@it1U3~|F}6?Cr(t12A3T7yBUacjyou?-eYB?0N;;Yx zv;q^7F~JBsLKQJlj0w^TFEBoYJ-jN-0y&yHAy6MbLOh5aCSn{oEqNtSG>iA5o%kkR zE1s1*>xHVX8`g}%#)*iVbcd>wbhU$se$+;zrxwV0_!xgnb9V5Y-H}KRPn}xlHU^{7-d}anMVA|L@rhN?^f5Xp$T!kE| zV5}0Iy)%Lpff^76e82+Y0fj;4A;i2aok^twdF6N}et`NJy0A9E!1HwUd({JwAGY8^ zdfb^jXJ!V1`KCXrW3~k8S)>c}U28N|H^YkxSp!xt_468yCdfU5kU6lcELR)}eWSqU z(^kn)u~y-06~$olg{d}Rt>>g3EuGF4lwt8ja4lj9ZnlvrRPOyDd(zsd(%@(rn24hU zyJq+t5=_k`Qd^N|swuRl5g}?%MeOtf)W(!fk`>Bg4r1C>6^aS8@+bE4{TunWhYb!@ zhhP@80uyKkw*@W01ewG0a3|7*Fir;T7e}RDe()rod!d`17nllKdF!+{zDgx>P}7~) z%v9q3WH(jHL7GL8x6qi$Z_V_0^hh?WDlH_^7jh!*DAO;D9&yaMJtX zcA)<3*yGDjI!PAHKui{GaX@52{-7S2M#~X@ump0DJ$wnyL2sABqj({25GjYrI@k^? zAP(A@E@n>R!*io+n#>>PcV|dO9+Z26P4&n`EHHtrxiTkN;4*N6U+{KJv@&b~ zX(A8w-7XdA?)rc>X_=PKucN7Dl@IX91T}M@_SxbeyWepY)=C4Zdc9jU(!R&SLLpN< zM_Ij~UJ%N(t2Jvia&{jgk%T*MS~Sh<77Fc(@xfkVSHJP*?y3E&otGN*5(OoWZ~+q| z97F%bQ9h`3B8(S**WzfcR`m0wR5m)}2*=A$ERcC<1lsqAAC*e@kgX3^4SOZw%1S3-M#O5`>(;9K zVuS0blQEWaar+Qj=I5SFS2nWdOh7)h-VF9~pouhO&k~p^+alq4XJ~2*Bx?9;PoK#z zgv{SrqXlmIvw~r4KTEB{26iQ!%t3P`W_y1F)QU3b0JSJG2wiz5?Y&;{CRBQ@4C?>1 zrx{H8gN287seZM!Uf@p!in?k0Fe=7Dm>vZNU(0BdRW~ZL!}zY>jC@N?Ph*6z6K7kEc)Pa`+i_$i8`{;$Q(RyHiAr} zmpF3ulgdK4t%7QJ;HME`IP#69=ceeGfA0Idb6Y(UD<$n8vVPj}ovvIqJ`^eG0=;E; z6lE7OgbLg6vB1usFisu{V=3rWwwgWl`17fR9tw05M&H?yAc3=eRLF}*YA~>Qvq?iMyXd6~> zrXmVK#CqRr>Q*IiYe$-tDGaw`WcP)(|iv!ui%A`enj`hzvKHuTZ>bAh5}akjEv!}3ZO=_LuG;~%P};>F=` z6um`_5Jke3mxe@k?>DlIY@H+k-q&$q2Vcu${VqF^F^c#{Q`LN)sW5}`GoTzK9ynrV zSOzhM`(V9Lvv65iNWJ`W*$*q^MW=1P;dr(2YEBy10E2iDRS2bCnhFHD-&(r6C8{8X zPep6x7C`}z#L+AA8K&!e)6)&-jCtXbMNzzOvmMi;%@ixbx{$Nikvj;Sj!pPGLc5aAGFxDab-*OMrv4@J44 zfs#j^@PNnNnCVkiHwWl&{yznx>qIzJOgJJ9U>~GlDP)x>ng}4nFI1w&`cbJ`?l$^1 z@Q$D^LA_U0%mfM@ArB5&SfQ05YlM#ZMgN{w@;K-lMKU{)wQx}N=r+@ECE(l|_wg^b z+<+xJt#W^4<9;JBk!7 zEA_`SWa&s2Sv&&D#YyUcf*H-gEHKS;lI+uM)tWQZ&<}jnqP#g+;{1LVnR;sCQl7k< ziu48_Y(E(Nwg*t3xIWK6y5J$(z2DAsfOvTGVs0wPH*`4FKv&&wy55d9sG^}Q=!CC7 zmh)CZ1QhY!3}eBzD2|vKe!{~=TFuP51sK9D*f3)dv3KEO1?a-*LkUcJ`j3;!--kc38`A6M4Gs3>78j)npTDUfWiauS#4{e|a zZmjMTL7BMeOBu^>C@6)>%gE#h?$0$?2xpaDOrIHVkBA9|J!3$sm7pM(NR#m5=A>JApUal`90K)9eK_f-?)_|9b>JiCUOLgGX)tR%#o1^U|ZCQ zFQVy4C)@~G-t@t3h$ym*bYl~QpLjtt+$rthUgB_5D2d2=DFitEeCua3Oi?%p4uoto zWiwa-mH>XS3Z{`wEEX@PlXQYgAal~1*u?hHXu6Z{=17m+&eQVqLM=c*0r2kvonYkW zp>?2_Z#xOKsaqNCnu1o@`Zs+cORxe=0^l84%2NrE=rdNN8(xQHAm3bv1F3Gz70u@7 zpIvy+hs2>Kl0mu~sK$Q#r;;tgRXazm_Ng9KiDFN~ljsyApX{U}rckvi*q?O&Cr{m4 z$?7T{D~-q6@o=;i+Qeu1q3j`(iZ?>$_MnhBgo?lc;)b;lZRinvfKK5pbhu)VZX;%3 zH|U2qil72wk4*ijjLsMNWufK8z!g8FTA4|{v#^kkhhhyErQM?KU;FjQ2We-VGZ5Z_ zme4XXI|@%+0CwlBjON$`y2TZw+$``$bJg1M7%W!4XrZjZh__>SY4}pE0K-?bHGQpfLwrpSzRZ2JdeIu@5*a^ZqzI&2EE?myK^E)dze(S+TqT087hje9a4!UIDm zHPhQWsspsgm*ubhuq9pHy#K7S1pZ@`YS&@01mA{ zof&8yIK3k*hyX6U8SF^zsH5sy8?Il>jOU? zj{=E$`&54x$bwC5phMvOfn%+DvHBKb-vf?{w8AJ8o;lQCV^ATA;O`68D$bTLo2-eX zl8j`WPYk=wHROXu;^A-@zpw15qJbwkB6gAZC>f7rm=9W8V>wy z1zcU#3bKa_s4lHMYW0~2{b*2C-qQa0R zq)gvC4kXFC9oBe z3S8kxWEmMpjFEQut^*e_e#jm_T$xgUm;BTdi$^`FdZL@!#f!0I%K4>mEERh%4RN{H zfE6J*s2|>kOkrzJwcPNjff_GOQnQiJ7;D2nqC0y)7*;@~LB)tEnMqf(yOL5SQDy0+ z9U^R`{jmk!i2llbwy@3!PI{je#B!@_Dbvd~>c3sT&oG_}Ti z`)Ic;Y%&%Jq7z_cFvthFXj_Zgh%l;5nzs3Vhb=myWx1yESC|R^G#lmAqDw zZwMGo1q&v4w$e%BESY?$Owy(nq>9TX!^6nhr{W|F@a(%4zUYeFis~SCUl_t0XNj77 z%dNoIuv|A6o;;WcC;N(t#Usic*aEU`oA1*>!RPYtZ%YLP)#e- z`<^2mO*`Rk(v@%7L)Gvi6#7n+B^`PzHw;WlEAdf*dmrPX%sS`OM5Olcv##g=Yr+O* zqcJd@A(MB`=8He3qdVy|VB=BRTV-vV~2=H+cLjyFnw z7WmK2Y^|&O;H+&Nls2vwA~V0$m;}79Ph2w-jV6I=L4z<3omFjcev(iy(*pbMP)TKKdExLY(N`e w~?b%LP(DgTA!A3zZ7 zi{AccZ;JoPHSu4y^VgBz`QiAL`}q^6{9|eQZ}Jt<`CFa8c+B7b^nd5FuUzbp27KA* zf4l$XWB+}?@6YVZcP1&C$M6uA&l3nm;=iE$m#^{f#{Jjj%2&u2namyd@1DQCK)=XM zUvs5DAM!V7_y>{q1q*zQ()t&+{ffo^KJvA?$CujiS2vq4T>Wb&>etD>zg-K7^mi|R zmz{n;`;S-pOX>G#mw$Zk_qzSbxW8V{FP!nUs@5NG`-e7uofW=7*)Q%#zkSbH`vnI7 zp?t;1{~`HO$9`Fb`~x5S!H_R^z4jgqW@nm2#%lpf2_j)%KyrLKhe_u>*sup zvhcr+;D4BWae?LP{-1rvi{m=Kd--D#oYK|PH#lv0=Ipui7cTz(wyxgkOXn{CeRR~k z^7mQiZZeh4^QF%000dN2TxN?L}7Gc81~Kq5c|Ue8t%>kc$|cN30xFcw*IT? zg`%MW*TjkwR5Zqk5=>?iCz&@hG5aLWawd|1ngG_Y9 z=?9tEUU66{lh`Tjl}<9rkG2P#56X-V+J1lVN5xMvBPSQf?-cu$GDczN{4HTJhZTQ! zI-*n_kcoZ*iB<1A*gHAfI=eXAI@oV`?+2y5i&93tA)|NN$V?oSPPPY>_Rh8nJDKE& z;s+2~wb#ML`JmF#e#85Y4o*%RzI0F=kkP-if$ErjY^ONjUJGgv*Q1cdT??8kd zZIzn0jGdJG9qbP%9DkBo>{0B0&(Rr5)qH6FiPCYui=C3wY5Iku!r57K#yBe-p>ZH8 z|IFUO&h9782l6kJ_6L-l8rg28qvKDiR>?n5I{o0FbosmH)Yi#Sp|rDApYBmO+VU+k z{nFM>$!RtHKxuF51fLS>ntyUpD(yFXXlMH$ihWAwgBw0@wEaOTv-ncs?6P66%OMwg zg^FnWwaZWHL(|<*d!?o-Movn{A8es_*-qiS|KJ82>s<~rOYPh9Ms`A?OOE?m{+pE%eZ*6ObJ z6^_auHD?x|D_r&~>=YYzD>R+4$3&cjKNyS~3tz*4ReQ?U3TTBAa-UWxC#cX*%Tu!5`e>d&E*p{I5qw6#CL$;!dH_Re5mwT1qmu(fmfK|Qio?g7uR zw{>>dr{#=a!ub!}smW)y_M9@4&y|N1{4iJ8Iw>60s!aX>hmHJW?c4CVQn{SH z6=>{3d=#QJc*VO-O2YY^0f8s)VKod2&QKEiB{GEe?vrGcR*a^nWKHL32?YC3P%yzptYL8v| z4I2zl5KA1@6)^pSEx4zh=4`LC1H@@(wUxehvHiitRx@xN>^NWjT50RztTvy?XNm)g zA6zs&22s&YGuU;8*?yVnzkpZlS7?m<2ZfWZ9q0e1AA=jYXtmGLUhxCpUoU|h39v4SP1sBy#cDwL1iu@x7Cue@*Fn=G0I>rdi%qssF z286A>#=XtocU1iN9ZZIr&rLu4L237sy`~krp-n$<1)GAwQ5V-H#~)yFQN=9zhjw;2 zF=(g|rES0GX40>1oen~Gf1u=iY3-K^hrinHJBTKN0BEP>3ietJFaeQ68Y@~r_&H>2 zujMHwAHoF9HTFYj**+J??=@2y^(Ms6pB!z~umNOxP#+?pZ2R~6_dR#d_a)BcbNBrPi*-#MZOmrW6zo2?Dq}_b*%Q4 zgM*{ukXqv>E?Oh+J(urYoV0QvGvYw}glPVI2c^y!H~sJkCLdhJt2CTed~ zMWWAa4?`%@%14ZFSYzU4u~%ug&%wpfo^v(xuWk1`J7Cv+0TJl{l=G$1{zsg}REoc_ z<$c8LeMk`<6`Tt}s8KnIS`_AVN3AT-bnku#e!xH#wQ_^K&M*x5bJH&!4*aNu0HTho zS|P~|T^iNI)`MS&D6;cSk7h!6&Rl6RwQ)b2` zm)}A>JR~zdVC$&dPf`iyuQ2VABjf#c4lvW%LTr>dIzYal09!g^O8%45Q6}BzXnWv$ zB|6}bTIGT{wdO?GS9~Og(6wK|d#BlF5Z9e-zvm*`2QCM+`Vbo6;JBZm#K99G5D zcQ<^69qQ-=iMrBhKXefZ&RX%~)5Dl5tEW-fM@lYG%l9gOIt=ch9>%7B+`1bQZcWbd zdzjX>>?zk2qmJGXcpWs8s_aYU_uvDAdjNtIOwaGzI{&0;*GCFF9Djtq;!g)` zoi|_#^phq-eK`Y`yI9C2$L6&Cf?8@=5 zlQZ-`7i(o-Iyh-EEt5YT;C;#zazr~WGm(FAP&=)ZLr!X^xWKeUyx;W8p9oQ1F_`vYc3vhQ+=qb;QyjI^e(mA} z>5r4fY0dURV(Em*rN&hbDi1@B%ZCea9v4hw`GI8$4#>MQ7}OrJD6P{&S`KXbwJn-V zldQmM)0Q`b#h13<*}*)hu>W2?8?OJ_*52X32Jlx^lCGO?{|ww8I@-|3FJl)doemoM z+ zxdja0BZ>pIoU!-p|44yLKXq$OzjCot+HubHIiyX>@6;at{z0X^rb98GAj4TDwFffT z!5F8iY$ zH$9krpk8hAVaAC|pv^x~9>C$kRmb!b9E@DL{wG`yIjhMMnA7(k0_FC+d#H0X)nq99 z+QFI2limldNBGG_{yF&7KCN8LL1CwL1}@5~EUVOmPzA znrHkqzv)A)Pj*%0hruL{-~OOHXp74(lkZ^t@a^}G==w70_n2b6A!8iu4yg9`O!wo4 zo}-H@H?epR+6gWOYaz`-;dA8=N70axrv|GgcbYB$;>*Zn4P8_RxM8^}0}e1lFlC z`4O0ZV3>Rh3u|bz>?6oj@q-OJWiqG3aB|QdH+L9^@4mC$kK2s&IcWFgHa|YwcnvRHQMj+gVIrXK*nfF zhOTn3Q*!Bq$mx(RR0Wc_!w&mwv5G$)R4VQE!-C3DW&xdNYxnIBwobP28BCNiQ(K(z z4l5zLKoyOZf8P%?9_D05u-1Xmq?FMvN1ze7rwYxMF%aqwpc0t`GWa7-WRt}K(rj3s zLan6g?w9rSrGy3=~WRq2YjoqfGLNg9~J&cqy`958 zh-<2M-}zqYa9HUKtGM-h6n}@LRQQq zQ%pk)2+$ImrR6JiNQWMH4NX!ZLn~-H>h@CfHgPl}ISfY#MF>LhMesuKL~uj!Mxe=? z@uJ19)JdB;g!-HCMh!wWf>=z2g@@;xv;WStVLe0=TFQ##t5)mO1X=IzqXMynUMpqA@)wMCRjg^kN=hDsx+D&0q_yMNB|?iy--+3JSP~(2Z~#f&G-}Ly0x3-R74=bY*i&53@m{Hzz^3A%cta?#Z(Pbb$YUyjY z3|pKFc>NSwGVc<+*Zwr3aTog==CM}SZWeZ}WgqG&M;+7!#PgSl5$Q|#+An6mX1q^w zjoBiQ{#88=trWB}#nALbCAJ2E{?jS)l7L7K$q_9%L0&G>;BtPv-d6;A%~Q%j?~vau z533~v_Loc;&)#aO%YLy6Ymxpk)H>zZaFI_2MRHWuSybsZrAPfsCq zPX&)!s`Rm+F%k28U;f-!PuY$BNh&R7CWnnl_U>DICwnn^Cmj_4=ivx(#wU!`RlC1| zeTA8t*NlqSw(BS|>3RT)N=!(hzhW*eXm~qxN6*U*v%m|+!G3hqG8utC0#R$`&VptF zvb5F_U6p$7gxg#9&g2FDg4X$ED*QfSFDpG*l4@Fj!n*GGwS>!tL zmjXZfrWF0JdtzFWHVpCE3x~th8F1Sh0A4_$zx^fin+8tZ3x&xN-}FV0Qj`zAhLTYe zz#WRI$S04Zw>=v*Ps39QB|1T(mo+o8NT#N@AO?tEdV7~(s3aVPTDsp1Zrruh4!~|@ zJeP=*V6()(61IwoJE}4ybgb_ZA6wqhRl7b0X~80TEWSPB-4TRggh2#$8xw@05-j5_ zR`MKM1&}0N^MkBsqsJRagF0qZG`k))EkRz zO2SgQ&TD=?8aOilZp|84yb)#9Z}xF#{psI&~p7Xw-5jAZXI2AAz2^v6RC;MWE@ar*~-od&Q3| z7uusoEh8F-n>HOqU>%tAu4go^^IO4?{W7>6Mo>catx>DX_d7 z6gE~#u^%v9g#R84Q_*hu`#SusL#1FFG2MN!%y$cYhahHBp-_>E|GJ(hn*D3mRj3YY z!#q-HSu0^h9~kTQ$*MNAaOx(iy%~&r6MKLq}W_C$0ehk;$EeXacv z=<|p<hff1ukq=czJiT*55W zBR!dJJ&BqcTH|Ma;V}|<+x$wO|H&;)_~uoFzAzHb`|)Kq6OW%E&XdD4`&+MqhqWU> z4Q}FV7dg3&*sXg14!*E|U>5i)eIlsR&W=i`()jG7Ny`z0p^Q6~Tb|`UX_S!Dn^f?b z6kqFew~xh`(e#Ni)b}yt?gsi$9WBtdee4Rxf&>s*It?HZiAtlotF~N5peS-&p*o(U z*&R$gQG`MKQ{DMuWjzM*%4VD$MARc6^ww9IR5#9Be=q+`cOq&+ftXK74BpIS@XXHz zer+^x6_^`nb{|vA6~9Nf_>begfiv_8H-~h)c67ac8-f16{!Im*^R=EpfSP>G+~&&o zgKin`MpWT|4y0)JXc(5Q2w(l5+Tq`g8VlBnsrKet_(%B{x~mCa%vQdGg#U&&7f=34 zkT|{KiR(JLAt?hYdX*qKu0!CDAl6RuO1>0kU*kTYWGu(iLq6?>%PSWq(pS^oy}IrF zOFE8h?+|*pH*VB&5Mcn}AwsCKQ(F@i9q5J9)ys$ zLNLU~`Eus?O6VHp=b(})WL*a#FM+GzpL8p@st9=<6dimIN(?}tVc|B$YyMjP&$@Hh zcn(P9h*q=d6qDcwJc&faz$7Vo>CYDbVy46Or*bF#r>K2NV~->!(gKZix-{+B2@k8r zk>L4Fx8S3mYwe>}t;zNMR(|IKjk@5^r@9?dk#;H|3k4rM$d|-P<_7a6X{~`qXIk;S z+T2HXcTkrRiN9AvbR6r^d?Pz~6wA7M@g3zGsrjm7`<8=X?N#8p8l-r>-A z4+Q?GDg>Zj6oAK&%vSoHQRONB#)|*nOM<#?KYszV#K!{lQOpSsE3S(3$KmW5f;+w+ z@#Jar8n%#PqWsm`U@rMqs4bdGt(}>pJUec8?6}tg1{*@?M`(DMFR6pSxMnyxfj~zf z9wmoi1Xag^f1_E^K30zqW)Pkts6XnA&X?SPKR9&10&h<@0hBezQ&+CUQ?yttR`Yx) z_FJadUt`FB*LB?wYMa0WZQ4#Xe80-qdJqBnBdD70hraz@p%O5DV~ob&;bt`tF}J0 z9wsNEethXepza~|ki(~K{)0x72)FRuy9*%W)AUc>@StsRcE-{*qV=j#`lrX^0tStS zkhp}fXRHeG0*KYJd!EFd)y7#rg*Yf+0Ufwz%*fC)P0bm;w zhHX@L|2|=cCo8%AY~YRQn#bcdkRKmIDlzzeUC%Y`4u`?K;w{l)Y_u2_NHHAofG*L* zJpzbk#L(<7m{S~2soR6=swYlfe8Z<_IGFNBIC(d`#wH#C@}+!~(hSz`{w#D^^N9I@ zl=T6TqwS_4O=Gv@Fhd8P%h0T*zSkWr2{>3122VkzMtn=G{sV{@TpWhr8mW4rpO~;a zne*o|G|}&%U?N?ej;dOQPie;q_B*D5D_d@>Bk)-g3dD>Of%43TI_d&K>HN~L<^9II z#Fty~v>o9(LI(nQFa6{hcdL6R0IG)~5_eG)1&gv)zG{CJgnYAWXTj>BP);-5LK1>* zo_t6^HY!JQE)A`H%CqbQtkvB~z&d8EJ#uLFO=jVorbiF!_UMa5K(#cc+j^mm6xE0? zFnKTW<=D#|bSLllY;+j*T_zwsgIjYQ(H)B~7v%scY$njhIs}0ZMD!zvSgn6#g0+PI zrYpRm3CD40WCArl4FZo5=wU=E95qnj_@U`s1sw&3ko98OEGcI@b2Oa`7*4|Epa+R@ z6?d}Pk#Yc-rH}DyorSY>E=l~O6DaltsK6;=q)V~1uFG7-VQA_RLg=(cmg+SZGzRq0MqWvVKS@e;rrbp%4oC|}{isrqd-MatK zYn{nr323IMX6hmh>0T4ErTx||9tK0Pj7l5CXji1x!5R|K%6 z(W?GTID%nygR{KYl@gd@*1GW8+zI9=`iA~#Cc{kM%F)zvG3N=+F8-%mtL zKsbd!Cn26jfMw-VPK~=j&8x1(pe6;>LM8+UKw1oJt;v#Z4`szVHB%iE@WtG>LsY@hXs9* z&zBA)Q2r+(eXKovZ@5}}gqKU{vP;F%3-H%@u~_fvV2(t*&2pFo$ z_?mcA!ssyG1T}dDhEh4llK4<^U)}Z^Pa^I2=A>npd8VHpQ<~XK!p&x;nlIL0uh=qV zZHIAL&$9xAEaPKiU;*DA7{WHpEE3h}u=E=fi$F>@A2y5a1eH}2$<*1Z4%&?U6%&{V z5=JNyL`hl3(!?T|3ZZmk%BpMl9RNFmTh|gX74M$hWyJoLNk&oCS27ZLC?)t##JxD27lf6pGEI}SkfP}8^UFpwt}NV1wYHgZ97l={^w}OS$C`6-jfhF zZ})mxXB3TKN|8at$!4o0j1umg8|Mx}Q@v_(DaaIZGBY$=Zjx{cMzpS>YmtgujEyi|>>ZdY$r*X%nEeC&uHN)g1)dy{-7k&z4#N zI`EuPy?c#dGeJ6aEZxg0838&nQ%ilUo_c8$s<#5)$kpPVT*f!NE2k-+@Is3HB~z?L zjnkv92@Vdf-g5sqR;33q2y2A_gEz}5z0M%3&XrK#%UQhR#p^p>JMvbro0x_yjW3-R z^uILhDd~t!%y5>08PO=+D*z~%OO0rim!MbPir(1jX(}iB0!;ZAD7W$p6bxgx9Gt&s z(OOJ&nM5jyG8QgO+0m0wK&5+K_YNxh9<0#<5rPoF68X8DC6cLc1T7pb#Hzt8T%N0G zVG5<2t*=Te0nP=w45zoy^q{;ojS?_j6|eS;P%Tr2A}i`->=~MA%;kfgQ7ZLki?f2L z=Rv3D-fDY&A>>|%_9}7ecN4!;9g$Q#^uwSJ>F&$!ZSu**{$p0nl|#ce47K z2;{V{Xkgeza_Lmlu+7QiBqO_alx;l9j)%ngZ+?uR(*E@7kj(@FoXp_cPr1*?zs0G# zA)7e_nhuN}OWNg!pjE%l!XhfID`0MuTSV3TrqOT!vuB8-nMD*G!$dQ&58%_W@>3!? zWgN8TUdXw{O?2q&O!V792%y3nUp>ePn?W%>M>92xFlicwoyDLDs!G+moLle^{$f_St5aPiq->x=!_z24S+2=@@gU5K|3if$%T`S3@# z;n{VBYY2@94G1*|)d-gnG%b&rxQQMxjn!3oz&Sk+XfKYRqGF_&b{`GPRJ&HXz_m6e zv;Zqf+#TrVXMG!i9GzNl^|MYwz%Upa#ubyH=fBxK@#w!>nbha7jH}y$HDugm1Vb=W zMO-Uk#hH|`z|~U*f#^F6CCjMdId};aK{}nwi)0B!%#%vZZ?bh0 zi`T~e`YYaf5?6h7R~21BKd8#gwz-Eur_X>a9fBGjP`P*lE27IPMG#TG5qt|(H3Eeu zkd=E=SSI&%)jPt7B1^zM+pzf}Vxw!EzU{jDOxtjy*w?ifQ&?!hO|Hjs1mW(rF_OVH7qh^*=pDxl`=#m`*iK%8K(S;q@ouIET5GLlefY zj(b}_Mi@nSgg|octxO-U;k;0T@zilBsLJF5epxfY6Z0v-*GaFmcj0pOF(fkdT$+!M zbatl5wLrzaFz*)ZV;y}Q0E(MZ8J$c@1$vW`(PAPL=!a{@8NPQUg?a(sC?yGq z5lEcXA{GHrce*z!Is?SZQXs~L^Tc9(js7J-3||MtbS+|uJ~8b)5MRM-cQV5@y?sH@ z+b>>2(oCjzs}21u74R27g^1i{3$VgPTIvh*)lVjXWY-H2CQ7u3rTTNM=LI0X1*5Dg zob#27lwb-qPs3f3UY|m6P$~BK&zV}DUM9?rOeaf$Ug=v%0U0K@W{xYlq?hemq}M;E(8Qw{KtF)rdfj==l3$7(n4C&ug|4UJ!U0L#~f{%whr^hcvZ6R_Nzl zJ&oDZ)N0645|BqW5GpA_Kh$HAy$u%3iKIx}B@xRn3!1Z~Iu9?lDj%VPN=Kkc{&=O# zBkn4kQCxy-6+=`yqgA0Q%2=>%YMdUc-brWQV$P8|;bzfq1XmZ4z12zlg7^vfmROwa zJ&|mE4guPnu4xOsU9f8s+zicCZ$?38qg9n+N_f?W`BXggW>QM1yVdCo0QPO>8fSCy z6=CPEE5#JHs|RMy;@A#4DdL*1HGM0q%-0%R^14<@SM*Af5z&?YA&}O-%Cr!!YH_u_ zTDZ7kV%ney!6|EC(b&o}#g^Z$6I?G=c^$uOYRE~joIs#(XtrzBP5MgGdDSx)NGT!!MMk9=5G+1e|0u??0|Q* zmdHBr=F=#nSq>OAQYOa2-FlPGPAF?QZEb(rTGpCLQWb*9OB48lZs=>e4b;@?_(B^x z5fQ@h8PX7Pxs1f}?Pfs$kY{2;JSUFSjUHLG60ChLPU8^Qge&8zd!)_K&`geWT|inJ z_JB8xYx|;?^}-DR7);r5hC-EDDw@VOvKki#$uwR&F|BQs@@0%-I-3n~xUH^)y6A7g z+A&X1;yH2q_f3&~fp6Y?&>6L}$9K&=-!)X_6`u*4qeg=E&Zla6? zJ#;54ZN+FJh;ltBmF_0NjsI7f`!l@%wNgSdL&z$xo1fw0`!{WZX0N`6j^CNv7_&P% zv7u^3o@YU;4f`1r@Jv$~Seo3R^vt_0so})%>XLERbv)c&W z2!6$b(6g^GX+)X2v)U=+o0LEslhl_#P1OWQ*n`YfBr_e9y#0vCysd zdo7PZV05A@(K_ofK%uyI-lBiWqHoE9`b&QB+F{S$L~AO4>MDG+U@_*NRKM#90+D!( z!2TO^G%w%q#eu)^CKo1b0uvsK2phGGM4+P(!^vT(-%Ji$j6lz3E=KRX`5^kcit^E= zHOY8EA^RSa#H(K>;%tn1RxwSLU@%#jgYZJ9L_pXBGoHlPvlpsyTmKBIVCXLD4uxu% zdJ3#3)Fh2TO(OPNrbMg67Ff6F@_w7pfi|nDPqu!9Ko1k7NAP6~L2yLRp0=1wE`fs@ zjPOvO3xfKzkopM9k;Y>=Sp6OW{R*KA3@z<&uSEJbrdO-_`m?EVC-S0mZldOg0?j9h zW?>d|9(R=}2rYeDn)P>|WQ#aYQ$=AgPruC!oX=lw+wWzP2z6v8Pufs^Z5N){_##jq zh%jeDA!$|ln&93o4DP0Fy$b_xPZkPWKFwXY_IpNo~DK#kGNXpxsSS9Z3UB-smj4?%+?6#rlg#E z$Rs;ekE@7DMQBqrX*u=hc0I3^siG=INf_7jD8fbIM2Nzcf`M&B26iT10<@@2W*Tu@ zOBhskc6X|RR>Nqkr&S#S9h$l(D0K}T|76U+N!UwmVNk3E)pz%F7CVA4gkU71#bQ{V45;d&pVAxkqTC}2 zI70?G50LYif+XqD@};jGMF#C#-{@uSg+LEPW<9aNtt>-IO}fwsX;XLT7hVIC=G1T| z?bS1B748tjuMAi5(g zjQ}hn%pu@63WJ=EhXNg@*YG=$;NyvuYR&wFuYRB2nz;w}Sst|CAvJrAdCGJ9^}cKA zL~hy5JM_yhk+0LHdGdhXK2Jd^X)omP&v@FPUJf6NG~8fif5W)eYML}8Xws$_q-E>m z(&Lw&WNmwlK#tfC8Fx%Ct!)&bzBpN^(@hYUAmfv0P>(tQ^65|lUg?UL7%0z-KCR! zp)scsXuAFA@hr9#0S4D#-SX99$sHNRjkJI2U8vH_yoGkM+nEsqQY*^}pg@%fO`i`< z@->HHHc^*vXs4i|8+w3+s5D!r6$KM1y>P#x1EXNnqsC-win=leFJK_I`h4zYrf&tS zUb47wpE$z)nz>y6e?;YVWbTQ@m|D{?;AQQBSBBBzt3E5(cbM6$Sjbul!-|Zz>&pGC z+N)-oT1y%m-}D#X8E-G^iYxcLRJ+?y!G#8u>`s&wq|w*BLAGfH(#|Cm8(x-6v+H58 zNke4GK$>c5|5lj75Yp3`0uc<2Ocf88X*I&B+_=6Dh$k}&F{{hr6pE`T!F>GQWOvK6 zHJq`LY z)v@-{b>Qx1o*l0t{>QN5$CO-mY;7KIgaFhXOfKEj`PWla9k%lh05%MCnM;F_EgIoy--U{!~a` z^#E;{(xPoc_Lt0cUSvur@3JaioLpX*99P4Kdfl=&5M51JHf4NCRfT$dsZkx%?C+Sc zQ#D#EqIq1x$_3v5zcf>+YCqjuRGZG;Mflx6(sYJTfrNdFi8@uI`Tkh4Kz$<+UYmOv zgu^P1x>nd0!eo~TKf(`x(8oW$DN2b5!w-? z^lf~(A?uKJk?^x#z`F#YsBfbN#kD54*7L+hFE0$X<3DT5?MyP&| zI;kPofI~w5>@?>suC%~gq@}|kVUKQ@Knx?oDDOr?^K?Bo>@o2E=Rp8i(`+OV5O1Z_ z5^>iPvFyakc(&xaS>-ml-;z;F3+Fp`EF#PyKvv$FP^0{C14xPERt@AE?!E|b`mdo87Em0fz64tpw$1){u(S0mdH29OWy(=KBs3j zBiDDb1}yLea~DPQcRt=GTzh0r<{vBG^#nmBnVFV)YS%dgIvw#WIn?;s&zYe_&LrM? zCNU}X1HI~t!gnyok~pr99@nhrC|_ToA7U;do#fNLv~GR>SQ(4l_@3LPTTRa*rX!e~ zL?rJ@MW^tEjzXlOvacVrNDj@o@pdqt2O#((_#ouVz&EaNh4~2zv+Bi)bq)bjOfB7q z7taT%nx=j5H3k6^_}XNR0)K&m7ju(90XTO#eUT3K&U3R0Kp=g8gOdsnNU@VCAT=En zEuMuxI$ze-7v*7nq6+}h^Z`yTP$2ihwU+Z2-ar!;r-oP9sLct}{A=EknCHmBwC?2*(#3FcdDIprJB1smA@nzbDJHf6RDOj)^bSu86%J_2^HHE| z@^rLj(xxLsM@0o$Wnnw>5Mb2J@Zv{>;Kok{-3dP-ioei{T#WM}Y&{gi1LR9036MaN`TP|@(G zwp%>BcQT7R#liUyRjKR|DYS=a#KnAKx$KMz{JJ?bzfQF{6jxHh$VP6VD-G$Bup5}8 z$kT6%QB2geSt@~IEt!<%8B)*V!1w@Gt+&oLsa-2d;D;7P521DC8R7kwbPyt}qV6DD zeF-Klyb?F85BPY67kFbYPdp|h8dV|jH@LO(KurUq9M^NFz4=b14=rg6~) zFkei11%6TEmc|iJ=&4K_@d+@T|DySr#y|Be*)864XK3g&Yz46?2Sm$he5t9T&So9|I@;M$vVf zw7~(hnzGY97-ary{eQnO8D8OL{RDv?CrFM{=+ktrI71BM^CVaDX+ca`or_w(4nzSz zB1@AUt38C}lmR4g)uEJ2c5ZrMDG zI^HRV#uW19r|P*qkr@|>ao}FyiD!jGCT;?VFOy(!k!!SoWZsUXclfDy3)95wOw((8 z*c8ysOrVa}dD=O>I>sYy6Z6rxP{&rDn66jH5+sUAvfsuzNQS^Q*A5}eGHfbfe$$Hf zq~nj|3lVvxYeDx-UU#OR?%-*V`utNI_jh<+mXIgQM&AFHQS#TI_1!!(Td(z<$duw} z>v>M=rNh&t=YPwbJJ-bL;W@Nk9{#iJfTXxr0y|1GOtbn2n{BECkKVMYE3Up_l{~=T)pm%LfR+mxbEG`Wb^bqy?(2oSC1;(8|R7V^_ti{ zPnrnR%sEZ3cQrbH`dQe{ns6|$KY4E-~k@VmPyL!@B`ghfdo3jhWTJvn%6Ek$fSIGyo4D;;UBj7?*zUej zrTleR8yq*tU6UHbb%|DmHZutZMVPT*5U~K(S4>J~6SwFn)?0M^!o3KQjH_I;*ARSB|Xb^-Xh)wSwHfepeY(zP#NAuG)yX%JpC+?dIEVyVy*SH;wU ziuGE;#dr}`D=%_UZ`_ok$mcZK{p`}RQkkBWwzLlwo_UQv9*dK_Chv_}6Ce#0MX%Hj z!*}`eF6jrNCMYijZxEfz)QrD!VP4;L2S}O0U^@>qAiIt6%;I!b2z8ot_e_&eOp`CN z{IWE9D+PLA8MqHb_*Zi6=;)RU2xMIv`wWh*ze?5NJK<{}ItTH^UDk`veSIodWcM)P z$TOn{mOo&tG}lZ!|8_ z{Awkfvb&f_uIg6=RbO8}06LP8w-9%cviFC$ePKDpQ55}-IRd3DV}&uPhCvV zj4Ivo&Titr?-2~`MbwslA8)QWDvgkyg(LQtOmmJV`dz1W1=fv~%;*P9(jBib*EC|n zOwuGK8;Wra=#bE_0k4DO35T37^2(7%gWvk)HVLn{5abK1$zvO++P2#rJL(Yf(;$TY z8`GYvm1nf*=Na!tw1QmH#OSb1fB)TKd1pmKhpg@@D5yoaf}m>FdA|@>tCO?AuG}<- zD_EjlL@r&!@9tV@f#VrXHq4;ovDQqFR#lAf?j~UyOp@Zr71VvzX%wt@J@U9oyiGWF zzE*G_)M8Zh#~S&NY6VWZ-reYjfJ0;gUY{XAHIL;raoeRgXrT>cT@O`V8BMdFGsSsY z7Hg;Vo(=Nzu6|&*yp{JRNc{Ovdfk6MZ+zHBT8IcMyDQq-ZtB~Ab;={q<}Dv2H_Asb z;S4>9H-I;j8WUfY61xg~x?mK_Ce+}h;V#{gPH z?C+R?0#2k$D6(k)jaK~%Q6Jk1BF7O*+5mDWFFb}-!zi!ej!?tuhzFoy;Ps7szn$9( zapVav)2+A3JRklHe2)EwndVt{g{C9Kw4(naV2V=$`aGLbQY$zqXwBqX%)b1z}3WKgCuF! zWgz_>I8%l>>8J>o5I4fFYZkOUx|5kn4?a&V_bT2y&%Vvvf;i9g3g?+)c-ME1r~~O02;h0c9R0q4z9!DMYMv@eJ>O@Eo1*YC zQCBR5ximcDx@InUpf}Z527(Y50P83w;M8@zdhVe*ILrFze%i!X?~cAx)OgSI&DBeB zlt*-R=gJFB--|>diAa=MGdI8f90EBNNg~36I(B&7xZ1sAw6aX9dQ%1U*W=r58WAMw zx2jH`FWDN2JIxUYHF!!sf&Y=em52f(!b5h2AZR2^)UOgXs^4_W`XjrM@s7W)zCP?F zGeNZJs-34^XHzN02RTNni*%)|k?I;5PT6l5Zn={ zXHNlV5U3}JW8`oUaRA{yLN9sqHex42D}s0wG2~JId81&2GpG+Hdta=n`uqMI+E~ye zi}*WG$4vM7IBRMI5yq4sFCkXw-N+J*0(Y*JNg=cZAX7w$#y17`xUTN%uWx%j+TXon z2kt3L#TnN@#)b*hs?UIaeL7LfL`<2s&boE%STKiDf}Sg}(iJO|vLfXf@Tp`hk9~&; zMHwzr7ywrkJ}zf%I?73*YO`nf#8M2&_R;w3n&noG&;?%i#y)*ZTyhlLGan%V;Ut3i zc+nX;9{y&l!H=IH;I>}W^E;p$6%LaQXeAwB#ki(w2J4_;uo}(WtDmPBDfT5=H(pU_w%IQg_gH(_W;COP*i2azRMB& z$ns>eI4W_&Qv^!qUwjF?bsB+!6;hv1nm9k7K zWn$4s8|${efQq;T!MyJdBv?KT015OdI=dMTGMd#nR~05-6$r8FHD|o%9k4lq)|?Ve zcV+z1l#wxFsmkHR+=1$K_}Qqdbw2`}g#K3Ra{#GSyx0OUF8}PLI<6`5 zG~;_XjeH=;!DYV96$?~JgorHtdS=^H7qMN}ogdz;A9kPFe0J>kbtdLyUg_HDfg2BP zI<6+1WvQW43HfUuBBbKS$p|5?<7q5-W8NHAnq1>NMn9oRw)%ut=zbYtJUV`}WZV2` z^=s@wCLbl#56V&qFZ=y8^+sIZwuRb=v90q6UVUeVwodd8O9lpqC56V_#*dBTWv{;6 z9_Bt`b8}&I+@>92Bz9^{{&Y6xI+Ic~+{Q%4)qCM>Xjz>%G}rKq)9VCJ#%jY0Fx%&< zBVNAK(h~4;55k@P=3cx?qX-3YLjkhH3A$Z)p{H49Gsp&JPBJRb{2LSL*TuEzRbiXH zB#%Xg)Y&`T*ZLVkSyb|*P5#~B-Dcs1QMooLc!NVC0v(DNL=NealdidJA;Nit90ZyU zeCT~*7yA{{=6X--d+fgDCdb#w^$LOKIw)@s@NEwS{s^l+mI0$+B!di$Zu};_#Pgnd zH0nK;P+p!1%jTJIF4|AadXjI_O6adJMK|XEtctGFg6JwchPS*FolB%d>=tGadHPoa zn6?iu+%lu-iqxowrcioWyP77r@^2_$5ovTLJ{$Z(4#49#!N>1EURLYNV_9yUW)N&s zDxSt#otOW12a2+24-aE`A!ei8T)Tl1O)l~ENw3%z03`0cD{`;6}Vv9Y7?*x47}$d{a;5~SiK zNw)MKzx-F`xa5Tb$y0Gf&P?tanCx#GmN(752lAoP1rsX91eT0lhN(scw}BcXVSmBg zG0b`)6cvs(4z|cgY5;DVq)3}2@2v$d?KLcAa=ss=3Z=WNB(FekKQzoVjzVUll(K|| zg?d)xEoVw|&#}8H>uFqPP&n`7g+vCZFLXUdszfv=yXAd6i|s(5$n91U21G1M$iu;= z&iPtX6T^A!tAHD6i0g|rNxFY0_~p|G3~S9qaa=(VATi0+qqjj`5FYBfHU|MBezXDI zOfc%U6yf&xF$0_x$B-kn^f3lVYx>Y&BC5}CXlczo z1X?Pl*?(nj^lAijzxIBw{?cUcM(+aAP501JNhbn&_y``1sm|u`MI;@-7mB)tFCFCg zI-)824b#!Pyv=s|CQkKkEPWE@k5AF5IeogCWvGPypi;>N1WE7(|58a;NUwCFez;N+ zg=Z%a=JC|+Dge!HVD9%W-&SehM1RV98T9W10}P9XabSct`?;>A6A?k%Lqpnnk!yn4 zw*iKE0+u)0n%lnW5dsXzAp=xdCMrAO@dWb0jm)9}mQgCQB>3YkUiS@nT?9Uh-W}7N zvB7igtMZ2cK&AhE11f{Dj`Yu9b_y{-F-gNx>FL=qsN5zd+yIS9K^i5&eVup%9OE=B zO++&?_SZ}zA>ks7VI|M#b=!5H9RE@*a_iGPr_JsnbR!HRydZ(Nn?j_Qy}rNOV`|`y zGe?6f*TTr}R=whb#|>0U7C%9(x=Q{Murdv?%CLEiSWMx45(NfW^0EnH1S1M2Qk)*1 z4kOX<17gkE-6t<@yFEBqvo+~il$gz-m>D7>mfWInR=+J+!iQ3AC~=1h@`8!|SaV#+ z%RcqtGZx`^8il|g`Qiu1krIKYM4}V(fmQ^6JoP}hhNlI!Kz2iN6P`9Bv?1I?Xy=Y* zJpq;w+)=g%pGx;^*uWTZ$H(8|m|QT#7{(-pf%#;KFqWTBQcw+5SKFP4jBjY||!2P?ic=EkSUtL_;aum}z^610>U;H5%80~`wc;XLRL#@#FtfoOdVnpbH+KpKkz z_)9nK?J1Y2Aq7ER(s`5;{z^-!mBoID2xDQ;C|WV#KuUM;nrwD>wbr#lkXBn@a#6TKUw9xd((hd&x0fOvtU3g)NWQ$VH3)kNW! zkh5$Ak(I?x_NwdpbH=@5`Eo7BFKJlh-@6b%7AuKT^v^|se;S|nRnd*1{lS;jI_V`r zvK{pZSS2lYrN?{kN@?1ZJkeSzg7_6;NJd(F8#Tiq4>d$FE=5Yp+Q_w_=%_(OMaWtU z9o}z?+dr`kp5$i(Fp}ES0doVw$R#?>5UWBxb4~ ziCDf^k+zA-K}6Hgt()+0^Njol^o)=7uT+8 z+9=rLAYL3=uX0YGhUG0GFGbdHGuaZSAi9n>80Z0;s*wD`t=K%B zUlJ<@7!ow9B7){HZ4O*pzT(n>45Z+F?1r?+x{HA_FfyMhVLxV4uM@|Wc#FVXKDzy| z%{SJ6;q}<^8`U&!iX=Q7jT4#WAW}CH2YW2Eri87`Wuj=gT-t=xsc2oUF_{Z$uN&+i z#`n6)GC&LNG)${+oL15PiD|oyG=_$OyInd$v&O!nb;jHML}t2M^{>XK%%B6dR7Ay? zH6wudXu*jXGingA0pE&!3PWNeGkSY@+Zsh>q7H2AV?q)6i|~$Dp)0j<;QrWp`p zx}QTV&jTVw#md-UFt>f?ZfSO!B>K0AuWFlEB2A8+YU`noBVfK6;yXu-#IUqrC4`D6}pFXXwMuA)6NwYizy2mSQ)bb7jO?TtVZG=vQo2qyFZD{)1v2V+ zp7vgO6;E*&FFnM4`*GaQoh?uW4@iv}71>3JE1Qh0rSNg=nb4H0nj~aDT9~ zzZos~2PtnrX=dkgQEP4AC9t+d8CMq#YW-y)gnGl+7Q)f7^$WVlsWR8GDndm#ajj=hVCvq0lYLBSXRAB z*m&}94NM8Hkp|?<s{evYB(7W;JyjI~A(@m@vuM7KS`&EFnW)k?Hny zMd{p4WV<<0TM-ZxE7BT!Ttc{z&+o%gUW{)qvztco9SpV{1X)8o>bTL?G{MiQvMi5>Qc zmjVxXY`6eV}}e@&~z-rV`j6d@l1bj1pey7NyAj58KjccV*ac_mdQ+u7qIf` z@!EjyA!WBWWq|38i^1oA9YynNwG$sX%;QS_%FLcp(C=@{`p<%ROYxq6{ChHi6G% z8%3blzcO9-iZ!*73u?2+dogQnb7}gWY>6lzF&%*sOS14a6TyNi#+Nh%Gbw!$U&|51 zgtLMilXvlo%R7e4TJ1SJJx9fa6)kNlL;4j27&hI!h~BW7$IHiK!^j2a?CZ>3!`fH6 zqW0dh$m~}|^d(e{-=Fjt)-6rNrlURY878f$#|zGT{YV8)VLgD~`y75QQThl4Mu`;u zX5^&dN>qC0VkNAYlJ-)Aq^h(w73sorNVO`GHRD=;(O~uBChpE@Vrr|;AT;N9OiV_Z zA-+|S`=hzdRh^~v@(P56@VQX4QwT&uVQ;+~F6|=5FBy#DeG6{$SmM--N3j$f z!6_xF&XaJ*$I{$P9wU%me|*ZI$jXwtK13v~v!`8t1_9=v?D+EKWA=*9JVGDw@)VE3 zABnPArwyt%X{zxB{=3F4HDgt`5XKnl!NC!usRJ+WE0^U+besb72h?+}zkESxTJr#ObcLA#6&sC#Uxi zVAHUU>}je;L@qo7=ebYFVkkz+O5-RfNx(0~r`F0u6{l}U?PPz+JW%ru{CLXh5`2(u zRbtX3`3+akA$cGEfC?1zkgtra9_W|qw|O}g=kJ4GI+2d!|^>w+Pkbn!&R{Vf>RD+on|H&TgAguelh9g8g0 z1aLg@!K}ss^q#-Nkca8eWThCZMbvvxkdbo~b91i5N7IfD|D?EAUR^e27pHcm z6c|(DGx(B*0GUjX*DZJ2z)D)$aWBe$E8dqJ{2Z217qUb+tA{*~)=QM3w$SI%`j95U zC&afrhh|bX2DFE{ac5xK`Xl4(gVg2r{ob6p9|^^VJzE%O!V#hn=;Mg72zgEOWdGQ`E8eE7K6>x|%~gXy`B8`w%bDf^Cl#C1)m%ZjG&L+45+KqR-V`+XAl8O+=R2 z88WFvcGc#KtKBTX;-{_|;2GBz0dZk&A2rue%7X^z# zM4=*9zRuVfw;zv+))|NCMmi?&+0)*Q# zJqFlPNw`P~MKDWrgve7Q&~D+`HQBpythc#M(r}|H-yAP3aU*h~|Hk;X4r?XIUB@mL z;=@vXN$gZs!GPn)2`-Ar1JVashUAB9Sq8pm6-x&b>)95@2i z%!N|2;19}Avp9oe=p50p+!$&?1w%RAB6brqmurA!)Vl~QF<#CypKG`_^1elt$(G!) z>Q>BxA%)B^VG=J|G(<8lxP@2HB!ey=4^+5?Su(_xja>t-XC~6#WTMX_O+p-jm7)_I z6L@|DHha9{!4-x&>k9}TdhGmjJTxrzipoP@XO0t|*k43855GCC%QP>ozm~gYv973O z#i`-T`Q|X0Un=BXMt?}LC0G^r+-}jO;ybV!l~NBndPOJWayziW<%NbC?A{MGx_G6T z*rmcC&6Ci2khYnrG(NbMy*?!DnnXL$rk@` z0niw{&gDv>+U>DdP_>{}%o@f(c}eUNbqW6PLW*ibs^v{cZFuBL*TEk!Yo1eSSqqZs zAeet3yw@7iDQzb-m`Upj{}2C>cBrVYGxdefi6Iu9V^`EMq|TKpamAm8ynfFs@ZRn>o`9A&e|G~B(>|>v*Vsge3 zK7=B>D-L^rbo~jDFu9H@Iz2^E?tZt%ZJtF8XPe+}V?Iy7s22tL@p%J18D|Z;lxn~f z6@o`?W$qE;as$9S=t}Wc{AKpmnl(V|HAwVV5#yImw5-JK9z#S~EoL+E>De*3mGHn2 z&1|4CW#N_7OL4!Jk#9UUF%Jz+I)mSq$L3sXm%tRz&)G#Job=6N zRPCTUVQKONf3ke0z<^GZa?Mn!dhDPfo^g6Yb@A5Z_7=Qvbk-oN7rm~b+XFXcB8m;L z*2Z*!F8X3rch#111lp^yxg9+|(vXNW4n^=$>QRO$MhQrf;%02Lh9mqwG#x2;i}3J8 zgT^zj4Xi2N>eJHFF7Zcz4j#Fa%b?mqqHzcF*br@H zEq>8H6F+%{d4eiw^3cFB!*rasg5%uj2jN}R>542eT|F^aiFT~S#qOqdq;-^OnkD!d z`}hQ4(UVtdph>SWx5^A?vk1oA>^$bIlIZb@JagPGzg}Q)v>Mevf)Kd)OX%n)gtlBq zPnN`8*osSp@#j;}t0xRmOlFA|`cc>|yvJkG5Kq>CBdURnsO`*|DFd_>i+vF9av?E* z?l^rOjl7vjHN-Y1sl*~Ca~qknhFH>col|fqL70V;+}O5l+qQFK+qP}nwr$(CZQIG- zt*w37-P*UV>gvB|Ub<%fKIeQ1Z^BmON1g=1pxhm|ij*xt+;%y3DQuPFaS<+5ue`jF zmi=*@iCWb4SIEQ{qInLiO8~Rf25tqeh620quo5%LpX|ER7zGm-nt5kmoyQ2I?Bhnx zz|^*U4$1ZzsJ%a`uO!QmU|nOE=*?QVLo*@O#q}v#Ix=sn+pmlQh4n2gFF*|NbU5T zw#31;4OvS}Gx>DDq?ETPDkX@^{|X+gQ2_62Ef?9TUyhs%6_@}glfn}yN9Erv;yjaGDs1m zX=GBS9-^+w79?ig`0(DS659gw7PPs><K6!PLV+%s zK{@$@tk@k8@G*>0`ocI}_nC2JUysVi0`hb0*0+`MbxOWE-~sF`5Z(hkFp zB{Yu{Y`j}W&+c+jDAj`%(d*lEugdSwsYjJWP&0dkp3~BZULjLiHVF8nC;xvtzkQp zDwOAIMYrI+TgIB=DkYN1@#Ql{*%5}eYm?lGP1gX@V^{T;39!_CcO^$^$r^EFkap!R z%je;{Y9_VPEG42bDE?B3u)Ht8V-2@(ky2mYeqEAMd6CVJf#9dw5zP?Wb9@^bU@H~` zN2G2Ga~o7Fv+OXAB3JKrl#z1Z03(?|biKA=w+UdRw3ZXG29c4WN7m{iSt!vhbk z$aIBUpPl2wuZ_`kIOoJkpM+ERtI;0+5|v@YO=mb!yrMskl)*~CusC*Y4>>!w2*W#( zR!n3a5@;7>iyR?g0;>d?S?G$06YHsK957b^}#!~2XBC9 zyvQsmwv5-!bYp{LPW^@fkXs(g8A8yCq!}*^{SA5MjVzOk4dL?}6tMX*H!;-g&NzIpMHs2V80yYm zr>IDKGj z4|`B+q)mJ=H-O?sA!H3#POXZ-+zTxCg!q_h3IV{kg)Ezd*#GzS!7A zc5b@WnZj-G%ft`}kw7&bUQOFbEu507j?o30I3O@`I&Ok8)WW=HXjCTY1a$gyi=d=E z3+eGb6P$AKE_}vYk9*(oH713+PT1h8tJ3Z_ud1rdSe+7yW7g7_^pLBha|EA~cgYrn zJ&MA5wk~%7#f=WbGK)+(a~_{n{QoSBOgZ@D{P^}l(-ycLAyL4H<_E+yCg*Z$^1aDR z7yNNRsXqWBangT)&m9Aihg2qss{YWv5kMN9O8q7ET}^VGqYwV{D`G3Wov_mh*z+dA z3yr$Z-2)aqYp<7SBnR4v(HM7zv181-WNXq|z27(p;Sv>Sb0F7WYj)Qnk|n=TJgtu# zco+s&t&$go>E?9~8hTEbh!55P7wB?tB#-LK!qJ9t>C#7#Z=ml#R|05{Cj)eb`Hn$LFjNg|`94jgfNH9u6oh#_2 z2N1wq+44n7R)PXX^ETKWq&vrzS>2R%2U1psH8v-0eu@)R2TiXkFM%d98?XUl#_MKO zgz(->TuoILC5n7MBvzIsIL2b9A4+O!wf<*z14Vl&D*KYS0Ns?#lx$=ku+|0TfrMbZt=cA>j(4kGr7C_4J+7|T zxFv++3czJtkzg)OwR%4Sq)#2)n$`ej<-{z#ar@8Qx<7e`jk3b7hxCAFJ4^+$EC@7n zj|?TzBUu!%YHL4T5D-DMNirUp4H1!CxQZ(_wPWGN?o^{)QRJ0s?lyqwm+-D!!Pv`o zf{}r&1nlr$(omC{SL~h|DxxLdr{H2Q;T$=!GaT7o2xx3RQ1wywor+I2oV+Tt@N|*% zs41e*;1jolHNSrm0+l=izfn&ekTZ}Igph3&zlYz>;9MF?S>WoqA+v1gn?V}k@Et=2 z{@ysQSuWi2zsPZ_7&hj#CY2`Hq5-A6f%V5C&-Ox!4#=2xd@ZPfcmFHOjFK~gOEMDh zX+;#;n6XPeO&*k(LcdQj2xG?_Hf|)BAQtw1AQQn|JB??@{vS*EFx5Cb5`h^s@G@mz z!7#aw3UBg0$fI635XFUtZGhnfWXsd!6*_E0NMjM61uYI;^%rX-1-)}4}AqX z3(cn@{z&R%I1R5LXJ7j#;|Vbn8!9o^Gj2J@Xn9ne5gwCN(U((s<>m~b`Y-qHJf;0G!?7`6lU(k!;6>~qPG#~|-MCpQ< z1N6y53CC0n5r9R0R-`4SSFzbIDfaa!EDPwh@agN4yv2wHUc*H&5g>*_^;q}`kR^mi zehR-I07>2QGa%PMW&lC2P$H;D4g6RY9TSfU;OqfX=hM!&X>iQyq|}Rr8mRq>^ccHD zC6rU7j`|p+E}aA%5;Bl*b2+vGQd#no?&WK4>yyCBdIgV-vim5o?-J&v^AaPp(*TM( zK~sS@!S?t6$m)l?A=eno@LZk;DA5uE8%Kk}GuM`qFh94!=DdTWEP{+T0e`^P21=GC z%$)oQ8cmBllA%7_6UUS+NBMs8+OQ$NPIqI)N#(k^X6rDeqUjwLy2;ECawppn9Z1nP zU7g{cMBbqEjl?}OpXf-VEHwG2@me2^b)3dkhu1p#mcv0tWs5l8k)KDk;R9UV+PsDa zt#yk#uNxI#21l_LeheH&L1q>&pI8XC>v3#>Q3^X~CikH>$16cA!NgC0uv`)drA0%K z_}T?ajlHhj1%5pqm%s2^!9$#6OB$B2XE{a+s%zRJfgE-QJO+rZKpyC?qcdGWZ#eLm zc$7QInP&ZUj@~=kr5cQTn!!I_CBNIe- z;YO&3L1WOfGg0rmo)7me>HD=Lrgz2{l5aO>-?JVw;xy9gSLi`n(9MO*+s#;6c z+MjNdRd+Ds6why)pvu!aH-sSnHkW#!+Vk0Nff$&wzkwry#)oZC?N&FgY{Kob1pmvT ziY#Pyzq18gV?)n8f*8gYH;f4K@S+kk7THKBH#+$VKDpZa&7bLdO^Tq&4Eu;q=2`6u#55~E_RK7! zSM0!43OYX@fT13D5(whNCT{N~o6HI#%J)B1b(xGu7W}!|!cWr+Y0M?Jm||W-8Wpmg zYu%pf0-#eOS9fa4Of|LYto=np?L*XVpA_>PDPA*H($kJ`4znFYYrky^dak`$)cT?K z^w!WO8+`>kQHJ*sxwQ^#DBcfWM6W#la2I>sckvJgh;A@7jg*dgu17%Y3{>a3q6)-VgpJ>Vi~btwDOj8z;_ zW-LrW`-|Bt4_4h;U61!dx4=TpQsUTC1~GvXOB`h=W9o@0!eSpAX9M});m93f2-SK6 z`O~3rclHsdyN9)mNr7p6t-q+S9wPk}en5=B`G~#Zd&5YCNB(|!T+2xPgG;ZSU5!3sw|)?dzR}4cK~G6H0a}#ncPAwQw4*TY4OPdn8SE5WGnjwr8+f*C zwp`^tj~z&Ty-eRUD}#^S7e}KC3_aYK=JRZS8dYAuXPB=WS`v4x6PY1{{l4RAPT!{R zB+;QS2^&cDAD~)I5&nfw*-b&e(17Xw33+( ziM2!lrq*1~Ry}KpV`K4V=(Hp|f!mbZqkZi@p8{C51}|B8Mb$$8mj+-uQMKyv&h3?W zjFO|X;1){pM5EEKb^UnR@;kbb8Y%jjw#PP! zt9-Y@^aApe{*1q4QdqBEITSoUWg6;1dPia)|84fKS=hX|&Ff`}6!7zb2|KkTG#QgR ztP89%f)q&gyi@(rz8{&kf?bLty9+r|w3OcLar<`sPR4c*$v`LQT9$Uk;>P%FG|q`M zO8yIVoD2Dj)UNdp)`hvdvahKOk8NcK^O{29#S8$*1)h%(FlcMV7vz0(`E&Qn;$a|> z&}*D`dPMW*6m6s4W9+MrVA>l3NJR?v?HBFu3Y+U~-BhnKxS_|iQ&5!dvw)S(CQ)J} zpG~ufIucQ<&TBL51lW@uA;!<(VAZ%W#J}Pbn`6E_QqiAYq*WFT!~u!snK{rw?v0&3 zQuaEOzSwm}@$h;i5mk}G{>RE5qJbaug9fXVMNbDDBn|ECfHlb;UYItu-t-vz*=s@f zd&+HK6w+V})ZYs-X4uEfjr2(S--?HcSHmuX;Ho0$Z_Hyd_+*9bGXhD|(%wm5{f(vl zxfH)R#~&<$&=l>ELnq#)axrZEw%?yJ2s<1<0^>R%PSxv$U;%p>Gn};m;8p_!*lt$x zzG|Dcp!EonEws(TjN&Oyd8E_}2o=Mk(6UGFmf4P`0{=2}{5 z4BJ>?X%RW%3I>E7T|=2_<4WM~^jk+ll<1S;h2!am$66xy$bu$7f{?AT#ENl^`F4K< ziMN&jCSyLE6&sdO5ZNgsHa}KDQrr_kCQ8jHpz-Hfl7;Lv=x4WIr54@&Ib_S*gt=t2 z*yHNq3@f|t?65m6OCX%JwwRRR^QGt1^a@iSV$YTf3qBDq1nz43ZBH+|kw)J;W0wzE z^}kei?K5>p?^>%Fuw>solPY-#hjhq?jA?vy*|4{`QO8D3vR0>4{Pa~zl$Jl&}z5tu2)hpXB&*@KZUMh6m zl3POK8BHw*h?Vu<-WkxqhCY)9ManL7@eLYC)Nov*h35=tjhLs>)5-b1Eo;Anoz=;I z{7X|uUgAd!W?>?cX8-nbxiQPGuUn>u>4v+r*3c$RZHHv7<;{BF>m~2zAN|!P)3N-y z-7WqNRBmc+US>#jAQrs+3f9v)qqxL~?J-Q00QY4TcjyOwePiqF_AcP@fv!O}{efEm zS(*Na;emz(lT(rr)_2lJZvcc(1j4NP$KW}1I5F5WpeUynKR%$9q@kslkft4XU1`VmAv+_EX@OlmEQsY@Ib@=^glsDZ&iTs{`G$_ z@BZ^8uG(C+3|E_3ue@jG=N0J62!u`1L-f-Ip6;V~Zb)uuYU+_%_W6!-y*zIV zZ*RQ63h{nJqPSldEa~FnYxzDdaMM4&*xrU#No~Df=38^)!r$grbA4a27{3!ecYQuH z@w{DcH*0x+KMHdb!gIZU&P{tiz;oYj&~|;k-&b>g&d73oKNE9z48@H5ukNVRu&vv# zQ&>Upq4iZjum6CH5P&lZz=s9$Td@KY#E}5)Yab=M4ZnjG^q~h8AP-~q%+X;7)n?W7~ zNuLz8uZ_EdvhQm&6a+1hfuV4io+D18=$~9qB@5+c6M=Z z?`DXOjbuR7R#hYP_Y16}*H6|IPZwc0$4{Pgh?x*i3g$q*99xiZ=jx01xAoQJl1gMUoEuMyJDw{`<=`;@b5G+Eeq){ z?2CJdBSX-*&#c|koTx;he^BGA9(;MN(^23a;|Tf&mQ&trLk_*aVbnvVb_uV=LHGlQ z$FJ@kwN0NurH5hyuboq$8hks_JSyDJZ?8Y@Z$QXM3$d|6<)cQi;q$k2qi|wnR$6&* z$R4@y7-&rx5e9EO^jB)dN1q{L-g=;;6N3~4?B(a@T3oVC_BV> zyI8GQ;EDc*IQnlRU>75bEVG-*~)oXaO`b9R_Uz@4?WS8bsNy}(uZ<*Pls|}%{Z^Oo0yMoeqVYIZXrLV!*jX9&b^VAKWjm^w!U^t zxjnyKAG~~P_1k*C?in9=(_mLyzb}ArHkhaD*Kz`QWzZgXPy_*=S^J$|EgfBF8lQab z++`TpvtYyDZ))f2?fFu0Njuq^#AlPmltXyiah1ll5B=Ut+C2cAv2u2|rT4 zH=)Uay)Bozk^RxPuHtHYpbuz&zQ=ae-sK2{BfMjkD|Wx1?+vNGR2>rM;;_HJRWI9q z!L7IXCZ>Mn>Uw7jdXsbxf2|PtzAguM4NrXsVt>h7e3jT?@AqO4-LBSZ?~Hof1Ue%-O|D0d}|`Vpjeo5LV!xT zU4Jtzj^cOjeII23vEYif(jMTrN6*M>xkh)BU3-3WewFilTR(T@HRiovYh|Ht_5hcc z4{N8UKAI4IKhI3wcD=wCdU{!kB}Yxa4)CXEf5BLPO_NF#eqR3ROMH*joZniLxOD)2 zPi@jxdq4cCN{&@3$o24?K0R&ZQ)9ls;C%AoHo8xL9OfmLb-5wme~aFJ#qy&+a1VC4 z{eCCPZU%ih*nGj3CwG666=J;xUe`9O-z~p>q4K1@S1~Q$OMTaGf21A)MSs!T_xEoH zb(~kfYpZUF#(N`1Jb$lde0e~9m3v&WTwhtgYEK}13Vd;B&bRMZe_px2)`Po=elS_&~xJ_U=pX=$4cD0tG{E@+)ze|*(L~?zR(OIP0m?o7N zMF;TaL$sywN??vMfl1b_O$`2IapDE3h#QK;Vd| zC~T*)YSfk)_-tm5LgBIBDmZg4Z+m#aSdD6?j-iVj>Q;X0V8wKy9Mgqa^-}HbHi?k<*`>MXF*NDrwkf>$h9f2`6LFTuwy)N&=S zk#=xbX+DFrqv-SlxLJM@r<*E_uRIObod#Mwth65HZ0^d zy+&%)O0cOd8q0a&Aif%ii(T5%Mr~z;fjk?NE-1kWErqlE=*>=V6=o*z)Oc4K?^%Jb zJ-s(|^QCQ}jyNxN%v2A0WW#s&q?DK(I>K6UbKfM_oZJ z56OKr&??^)fFV`w!*@#HRgpn7#-7H~V5o{FqKz2DanLJZX02TzhnK zLLpn!IE{E~iTytViI`KW@%*x_ni@p82KtSb4n4uwpdf>CVz_YiFzY_qr)KkEF| zU~A&1%Hy<0C_0w*T)8(-2XAY5o-w?%=xti==-?kM)ntJ%a1QzAG@ERniTc;g(jd$X zKQZ2cWMQdYy}G~pRznoCZ?A+Yx^Z&Qp3ezi{3wIJWinTG*?y_O&j2& zBD&V=zS@dbIlR!FGV{fdHBQ%mr{*n5Ad2?==`vT^e?Dt1mvXP7H;RMQZB`doI3)}x z(UN&gvoqwV82djyuB*dpO9SJ^gVk(XCL)1RjNaPXyJe5RT zBJd-CadSS6#FG&@AHC;ik1lleuGt{sxTk}*tK&YaqrmX+Y<`P)rWn_*9(p3=rdaOU zYkt*nAA0#+c<47! zMe|aRz-ffk&f2O^9LiSD1|(Rl&F?bWVWiK9bgI5>Zk!5Ss;%c7 z3&Ydw`DN?A=*tdA8V9C6JlueUV%!MN7t5ZWroRB*LD`Hb?nH0i_gqdmz`r( zOdj|?-UG#?k1%=LE{XPZ8veG^-rwgO&n^rc@|%HrD_dos7&`ool5T=hM2u_S34NsU zuQ{A&#No(WV(!%b%V*~!{vxBYZkkBTuPx;Q(NWJ1)e-0y)uZ$x8q!E-7U9-fm&?p3 zszT!hZ=`^1!CW6mVm3S6vSs$jmizH!cVr;LBEECn^ufJSEMj6YONOSKX3cPq-o8!J zOg+)Vifz}@f$=D9BK2kKdGbR}ms*_Cb+svFR>|9)YoMw2aNqRUJTN=bI2&Bj8~2y& z;<;mq*>1-Yy@^;b*;o}hJ*b6O$B?y}ejY1oGt_?wc5qPJMVt6)Nx*K_ZCS|Kqwz-T z?_@g!S;IH*vSlb1gAkWTdH-sO%0L!0(-{O;X?%W6Xev)DUrj=`W=11~0{0~)tCgnE z3gPVas^-+KqNC)~OG3`xA&y(Avco!4ocz*ne0Qz(KC$ecv?d@!^O(tI zrUv=D7K$hY)VH<(7T0e2Uky`s=1+^Q8=g=0hXPglyLETV!Ski3CYivCOYl!S`r5X7 zT)SzKhwJU@#n5xt3rL&kqeP2{sO4{JiNNQbEv1VAxh+&*jb4kMD=BhqAKIm5gtw_~ z^d^^b>t)t5WOH0pP|BMN(A>1$d8!H%lD8bknm0y%AF*?XO6mtE{*v*mqlcv*L(Xg0 zcqLodPL1Yi_|{R0?ikh{qNgNggQ)I(7n)Yv2O^P0JV;4f6(gSJ&4&ObR;|>~rQ&t# zmyM>*-?7REGMxv_pF>qR4f5{}teK9?7zWp^R~`B5ZM7--X){2v=eY&;q(l4i{9DPE z>61B;R(vO3ZOD;OA2U4tb9Dnam8@<7sao_PWZ$;=D3d51$mh_M)~lmNQ&5(n<$fyX zn4o(P$(+md40bU2o(B=#b0TM@QzlXx5=*=hQvA&6rLlIZz(hNpH4zy!R{X-LqV*&R zitHOTi>t^(8(;D%?q1^VxZ;o&pz2bO`W2VNGvxz$B7JM=q%$%3*UZ^e-*Q<8Aj1VW zc*T3)B8(86`+szMB4YFL&Q046C#|?o?OlihILZBDC$3jz>XHGAfoC3TLQm$L_!Q@2 z!-wsI;aUv?44;?ORg;?L4!`eRBUZJ^K|b_30p&{9hY2&N`8cNoFKv43%GD+f)1FPC z*xMqNPJSZD$+q&#@?p49cf!Qv(~bqcJ4>Hi@}Y{doFfDPj!_{=rCqIoxrleS)M!V% z!@S$iJFoQXmoceAWmBYXtBDip z2FG>?p{pPQ$5Vj996v{lMqN{v$IEMf$~)3Dojf2r{ zAv#%LUY9a&QuoX0V`>xm2exCZpa#OpJL3g?`epkGE0!CY+-@Z?Z?olr&uYiqF)F?7g>fDK#5c!D(mi*{E~Y|?Z~4Q7>cuT-rGfCs|t?fMR%WOTq^S>mGNB; z^gK_eGq8zx-t=q1NB7o!&qifil$h*SJJK+WM%dL11M~Ow5*aGcbHIVYqSjVM-<+eD z&f`5*=3=@dv9zF%-4xsORa>pBi(l5tPt!>h-C$up+~tQ3hfDKX+ZN;5VNV_hqXccs zRrb%r_v^@H$Rd)*rvNrrr~3i^K>EplRK}qs)&6K%(V2P|46fK%;Kx*{szZ%jbcn*M zAa7nRW3CH=G)S+7%0!H2llD$CILUvXwdh<(*MH1mYoKX1P%QEc0>@#HRw=BGGn$g{ zq5goysIIbP?G=r(k>x79E@XySF?|VFXG@<#KE{a)Hb5<9nr5!u=dfpy$}(luu;88P zg^okRX@P6Q5{@&=wGN+|b)mKfx0LoT{z=8k;!S1Ik~IT};ahX|WMB5*QhqwjyNLwR z=Zk{s*zEm~-0ZJ*SQaUb+f{|_X>tw{AEo1v={{05$>M?tmzudv8;-dYlM65S`s3fV zj5LJpRUQ3BYm(km<*xltvpN`|ZCoM~GBxGeM_mhSggnP~_+~97i*Ng+%0=~Ij9M&P zS8;!(a&%tkwIVvW+*DmVu)Vh-58nbOX`++6=)SzL2FOS@h&qRR_i%F?D^ zZ<+L7(a6}! z`Uv&FQZeLf{Dco$Cr*t{%_Vc4&%urL{1sO&q|4?v?Bp;ml*mWzQ~Enn-?c(l-2Jx> zXf5;gN5G~0;~hG%PEX!*__@+UcH+?ImRUC`H?plI9oijZe0rcJ$UZf!bI6(s!Zd4l z;%`qmLAgXtty3c-?}8V0Uwn7-?+gJGCn9j0U)HONPn8eFadloSOKYL+@b-CY&6oBJ zmp+=%+j7Y=-RYWH^ao68+9fH05k>Xm%a!AXoXQlAua4EuhZULa@67}gU`uf=xeRM= z3!9=QT*eHfi@SNQ@YehTRxZ^vsQPRRCLK?Pi#iX?%dU7EK|7Zx?_ts?7KcY6p39G- zC8Jlxt{MUa>Ko(@d+!=+AL(`b2GwyD_M*9EI{j={X1zH@ePgdG-B}Np7su?)9a!Kg z5U%ptwShzc^FWa{sArhJ?X~ybkV76A@4SA)NHte8^8MrA@2$1mQeN$m97v$b7B1Fc z&A!h@^lv+-s2(@V5$3Id$LKO$S>nJNJTN%_Bcr*0?$B70RyQp@42*N`X7Qiv zugsW&8;+NJ1!odWH(rt~=HaTI?p~>7ZP_HEc9K-lH=n>;t`C56v^5`YAyCW@{?8Dk zIFs|^+qlIUxEu{VN_mX(IE3M>(iR{%1>7y5Y5nUa<@ijq$%^ChzRU1ZFj<@=Alpi( z&H-L}MQ|46K>yicVFn=y#rnc@OwsFi5))jHc|Zq#Q+yhgFO|OK8*`&QD6lth*%m9) zX@t`p>ZQv)+^q~h#S!MWSSb?YIjKOQKqr+AJXOCl4*(1ZRWz1ci&)^ptB zY7{SvM&wepR)mXnoL(fkA`s zXN;vyu+$DKfBcxX({o)g6 z_^9|YlVr#xpu-#eQpcgUWL<(Z^x?c$PgynjnU@jl(bfXF*@=*=AU)q+Y^w#@?dbSO z(681x^&DwhodnD2k2RlsqhbMB3o#y)M0{}(gMu+3uW#&lwiQuyG;7d$x(8dhG4SfHTb#G+96vJXZ`!kITV@)vu~m8LeWy--cLqwq2nzew=ywIoZm3b0TEM z)*BSw%Yo+5#hI!u(n7O%@(o6^M@e>uw-4x}k;{Vl zs<`dKTUtmmQN--B@DZL5EL2NUF-UpibGz2_6T;GgDVaU%ebO)kVfaN}DUR-ebP)#v zG+DM)A!Iv&npVO%d2J4#T-ODsf*;ocjAK!Ec?LPb=tI08!sDlou|cCJ=@IwXR#!Uc z3|je2TgfswbB1j4mTMCzaI$HB>mjY7;6rH`5@Ezbj7`?gmf^0v%Xo$&WrgkHtI=58 zYbK|YCk)(v#)#1H??iWlJ!e~85FlFV9UV4|-{Pa>6E4h+o*xGmY*T#w$Z^Q6Q4vKA z@W&^+HWcdp9M>0nj@C>(r^Vz(&;C)-cfsFKN{JwHWZV$;=)5RT8jOSToX2fHa#Pdx7MmIN%=@Qh^#E+JH2HC#2*RHwD0& zfKg#P3zOCcBmA9W&p&zJ&XqDVxS+1Ww;BKhSJT7Uxnk2{)1JRI3Li(bC@0Tg zG1D<`C%l87iRkxWK6-QN`@L_}#;>Fx8Q^#j(m;k|kt@ljLvz0DY|!RFJUMcvgMuT% zP#|mQEp!H)*U8IV9j7r7X+$nk{C2BfZ>fIz23F$BbY9$^{8Tlfli0mvv+=Ila0x#* zJl6k99c9uSZwJ+c!@K%c`Jp@HoyWAxbm_~wC)lpeK-bScllNW{E3@I=o)b|`Jd2NL zGNBI9xNFA-=9V@_j=9kt?})SIZUL6wBK4NH+2=TZTg~<_AV>* z2v1wj?HDIxL1mb606yDLp)N2*cDX050Y(*n$xml6tFKYDQMJN(*%ey0YxnISL3lv2 z;k7v{8O1>uvxcagAQIQe7M+Al@p|{w2bPK-^Gw``(k&a(YYovlH5c7!$RgyXFZuBp1V=?9DKgB>y+Pk}x06QiFO3@;j165NkwA8BGi?tT`5Tf-zuCiQKV)sxV`KOXqcBGmp6MI|N^hQBtb zL%Pi^R;87s`|or&VkU78$FL(P{B0yQR4oam&p`ENaz>|JiF7l1m2MU(wl5ZWO1uo= z)ETQ}(igAfIp`6Zi$jD)Zp&=Y%zL@;3@_?hTWMXSJKnj+avPOu8v2{4p$}7>bZCk* zdV4%NqK)%oUxY;7FSMDWe1-M6#2ptZUG1+p5447e{0uh0>qO*Nmu9HRo!s0~G8JC@7-l{qE?g8qv4?g7wt|pT*m-~_!OfJMzuq(qE=Q4$#%wY#T0!^=g{(7?Jkj~7sRJg645^7 zGVBONQL9S6ny4^D4ANf%(g0>q5R{)!$Kd&HkP-Cndc2?U07!WtqC{*FS&fQtM2lk^ zJG&(#6rqSm%X;dgydMl$mE*w$MAb>gx)3?unQa)G}f zjuPn6lf~1w7@YdDONjc@$tN=K0!!?Ez(P3YrYBk* zrRx>pnRA;Hiw&c7!fV6Ura?$Jcco<__d?^CAtCz{_Qq_dSSWEOb6R7xa6tr;h&Iwk zAPp&mgWps(BFG@>0&xnQ=`lg-Z74H=%R$4?-$Zp&bVkWxS>-53Z>5~Xf5wTJ=QIVH za;Pd7B!yJqjg5lsaC}R#w6>HDIold>b<88Afr$HimXI&FhZ5kMlyV$#w! zj^)(yJkzVdSEdo8JfHtTFWY|UjNK4?w#85n$LE-fi0SIl&LkTWDoAPMRo$op*uR~T zwtgKWKqxy>C`l{JFgNVKl5uiHRP@@|hLZlmjN`Vm2ObrgKi*CQeMy(0DD1H*AeG)1 zNk$$KU^)O(5>?#eVMs_v@yBW+n4#8#T9B0O1LMI%9vY0Coa00REnwTn^NP{-VaE~W z;f3^`P_Vhh-;7+&%keIYUgdEpL8S+~v7Y*EWd@rda3{liV9>~?o^^k?fd+Fw1Q>W_K*x5xlz0KZcj|tcb0qg4lG4FEEmc+NPhzRMz-yL25u~{FSL2 zMOgz9n~zw-sY35|zpzJH8FuI+lyEHjvmk0lv`^yz*dBNy>`aQs5Rs>6mVz$Zq=b(- zfv8CaUWdc0ZOvhpWpqK1bVe4xW#3ve^iP}_f2o`yt^3$m)5;T5U2|m+6AIo+;tz+r zPVB_cBPc zf;zV;(Jqgx(|Rxh$YC(QbZ$^L_Rz#mZ%2BS;KBZWWFX;!^oG?{1oc zx>&%N5mTBUUJo9i%}#)p5zLH?AtR*TXSb+F%W$8B`*4(g==f%Md&#(yYVAB z;$GXzO7j83Iz15`^KA|+AY&)|uNy$vs{~kkvl*=2i@I)*C0 zG+RB7R$dCkK?Pe^a1!lU)ufN4TpV$`gl5kfj`IsA}9*alK zGI6?wbsi~joa8=bKcbSrGay}(K+GMq8Bs@FOr(oGt_`Q#QP5*}ejo0P$cddNRng{t zec;hV3c_O^5xlIOA0aqn0+2ZEv!4_c4LP@?bV_NU4Gu zZh@#N!u@?ANyVwaCbe3g%O%NKrY+34TO4qyw8~U?(ucMc64=Wu`o{ zjF4jeWH6>EbF-S9D3>hO-zV9S2#QrlW2$n+ejw$@DME_^p1)H!Mq}h|79!y#*j(}y zv!jH=T7s^}W9^p%4niO*Q`gIMWH9u^f5i$gQIML=fLkoH#3&4CV|W&!}m4 zBLF-GIys6YYmz*SPDvi>!P|Mg^wE{&9l@1H+H6e^AP-8jI8)GXX2Btw67z@g1j&Sa zjN|e^A^DI2dVM4)ID;w`zry44?FQL$|Qg|h654brJIebqn}W&=?cQ1 z$!ka=pplWV;}wZB^Fx%UWS}VIhKCjfOv>f;|HABmj}#6ej2%b&x!eWALXaE@AfB}J zvu{OuwfLXPxySLLO06txivP=}ep0m*2cagB5dbk_+rj2SbrCte^AL_4#;CHpYYkpp zNhrkF-f}LCa&(i{Yh&ze=S5X_QD0rzO)2LOpu1zd@KJxZ8p`(W;!e7haG$SQ?+k2G z>{!Ul;gretES^p^7TsylA;m4ooAj|=LM%=TIks-LS(=g(gq3g)a{mEZPNo=H z;kc{=b5EonS(N8gmqLTBcThoXk=kllgrJhSsD|5LmC>|Q@L}Y?8FIu$$&yJIAwD`> zQWME-R7M*6Y#Vx8yNt zeLW6Yk7@TrOwr5po`@{VC&?!S@vZpdobUK;7HKC&Go@xY82Si4fbJ!`^3+5hgl#re8#xB%lf-Q z`Id}bhR?DUmkhkm=Ce_O7?o{J32E9G~wPyPjx%*dK=eWxk62WNbk;}Nz7 zg45@C!OTq$u1J|XGS92hgaG!RRpjUsYZ^NC>6{RTZ@EDEQhPJ2yYr@t@o#zpsA2z3u8zo zuHqKfEXK{GNT4LFy{*1B^6!;`ET3B`^qXL?TRn2%#vhu#W~nnVC^rki@;y5saa{>6 zB>nW@@;xIHUIV}|4e`w_1Ejs)S(LjwrNq0QojosyWPHCGn*ApQePK(oJ2wO%u;L|?^>FtX>A zV=h*4Je9EzaTcu6)p#0t*%O)pC1^gPpyd-HX|Z}E;EhsH4~G--5Gm<-7vKnm!xfMZ z{$lb#93=yJF`R_EC=d?k6G90^us`HPTtwOl+nju~%HW6)OGw6cAemtaZw?#63}21F zH}@k;Z6a3XM-k8 z4oowK?zs3oTz$YPX6HeUh=*u}<9Bb@rqU?MyjMTTSn-I2iDv;+?cIbQmhu{}`dLW@ z-K<*MkQAUmE8h+K#d0R5Id+bCYFTcScDj?bKvmIO2{K~J=~+b1z%wZr534ktDCDx5 zlw^#%9l937b1$S7uN)$XQjR^+3QBnr+%s!vO7h|51!^9IR|uR016O7qo|@y8h2S`2 zOEMPYlqLA=3MY;)#-m5SVJ|m=*l&oJ1A4{mm=lw?=t@s*+}$6Lji0zJ-RT7jD6AqazgEM!F71>&Q}>ol|lOhJ|rC3wY> zWU9hz7YSWM!YeF9i-@VmO!@YZWBG1^SiR%gqePgCyT35}pjKSCDmos_S*}bx6~txJ zjK9K%obgL2(1#Rg9(2zEX{QySc$Qpp=`fbu+5T{9^&=XRk}AVwv;~?H&t;U9@+W!? z)Z_({jJu-0!i{EbnoqK45qs$QT!5EAq+)>@&&=n{$} z!3#ak5EX>yTwg8Fl;+aLsVTN8Mjl}xdyTZ5z-i313xD0$#cWhZujr%o@7^g3;crAy zwxa|!Gw9-hU_F5}`R4p4EE8j@8MH@B-PwmdEBN*y!jKVSy#r-n_pUTllr_z5Xn@An zIpI@8*|4as8J5=5T%GBW3jM(e(y|U)@-~~27!RbwYpcBad{J75aHQ9+VvsmIrN7+7 za|}Oyuz=)yutRgWUfty8U5@6t`4D1bn)#HUg}9rr5e2Icf#WhecAgkDVd>mONYI>_ zn7x{|aem8MoNrLmCtQz1OTy5B06|6uTjh>ANAa8-h(TW5kQsJl1G2HoX|83DmtZ>@ zBszjMlgx}aiW}#}D7nc=Myajj;R8c?KdGz-P0WI(Hle{H^IW;naF5Z>h;T2K?Wd-0 z>9`?#jO8Z%pvD`iZG!kr?)026s);zhN?uSgorbUL5`?oM;K9`l?#O zUMFEMnUvTH%OlXfAVO^t=+{v~C8qgKu)3KylV{V}d8S8ZFh0Rj?m{c8XH(ZqqkGZ# zV7Mmze7C|jqMGld(2pAf<#FWn4$5{1Tv`QElP)TEpL5B)ygNbZ>W4fQF2t77+MO;$ zIp|FfuzVDc)M!kqH>fdnXR3pYrDbqwDwv7kP3@yO8qq|Jpm9&HEdUvD1IiezNP`6> zP)2J(h5D?z!e%dvR2qJ&E7d$k$5aIAl_jIDi=akfzLaU`5oDREducrInmPdTPJ_d@ z&5L7&HTlA6jh?23(eDI#d+^G$1Z#OWN5$v0l~_ZkQC1{%KMZQ$BDZE8iy%vou;Z1EUAVT<%6VTfW|s{ zXvvlS?Wfw3EXN(tt{iTH2SYhZ_76fdO`_~po9qU{;Swh`5d_3=D}41my6Ea8bfBAB z(FbkilI=qb#(XX1VGR3dN^JsY@f9iE;r<%(s+R9sX`AnMYF$Y!o@4mW5jwho{e*Bc zm^9#ewZ?(6ANg2Aj8$S0dEVFJF#p`)+m3rs%!jLv`lY3;xkO@g(yiDsdRf2?xsyrr z1ePqP^{*R5*~5VuPgRR8%f~2(uI!^Gn!5uQKEg_Bq6VxZx_FkqAImdNPwdB17E=mJ zu&8xRn_|#Y%vFz*i|Y0J%PgHUdEL>bf%%4uq1-Byh4Kse<;1mJo5K{AbdxrcPior{ zMf=*7H1QR!m}(NV*P-RTsCzQ3ZiBZvNy>J;hFrsjQEa#Obao)iKhQvHk7RhTvm?P~ z9K4_-$`>G)IA}&KY~0|vIg%VIuucs}+CuL5z&V%URD!4$QGZ{($T*Trel0!FprsCJ z?j*?z*md#3wiTfCppT5R8`q)cBvyl`MVjW(nO*DtC~S5R>5-n^@S;1qp2^)M3?(uv zRhY7svzH)B?vwkY<~HFi zDQ&dNw07CBFNIv_Zayls+P{%y=DTCKB4s611fMQ=`m3I|ESD(dV$p{UGippWM0e=Q=_ zd}qzPZX64aVrye>tmat853;-4g%h=eI43KYPGLt9-L2MeGQh+)^-RFg%jva`?52b5 zaJJtnEz--hUcv6HBo9rSO}iUS_z87~aBw(z-BUFFXy!g$noZixM_R|Bk-3Z24o^13 z=*v?G*`)-zCpjV=`02Z3kXGA-9+~FxV&Yl}6p=wH8UVP16^~mNrQ4|I&vq=CjV+ui zSz`63oQjLZ4r{USgd3{%N7{CTk~(&G6jCEKtIntET&Qtg)UH93`U#r6lIj=-M}?Ss z3|~x^*<^LxiHbGM?0!V$bHSq$TY5ES6xK8$>6s>DqnzG?S9T_@mLBQHgx?T5*(SM> zoG0jxxky9{QlTLBI&i)H%oU}~&5;Z3**5blw5%l7;y$+BBW!ZfJ4iw&E0DlZs8K@b zP89TwT$&vw^}QG3!?T~Jg>RDsKX0rxt7*SbTyNQ<6!=U)V=*+p5qjb9Y3DvjBRS>l zL7eM0_82~w?s&GhS~xmGTK2l$>iFW&FgI925154J(}l%TM5WB8r2p1r{LPJUE3Jd@ zD3OrAW7O6L;z9-Woq7#jjLK=F-~*f01bDyxY+o`dw%Bz0N>YK{l-zKr5}quF$8ri+i6L6TL=#YU>o=XRa)!3U(N1np z6}sdHl?NatX=vasPg}=nb%3gGIs2b)_C$Ka>vL7_`FjKQ= z{W?MN8gC`Wx~R@{U!qs;%Z=PnZ)blz!NQ=G3NqwxoL|#O+{#YAvMQP$(a%I%rSe8fF zl!GmQp3I%G5CQM38b}X$g~d?*5kWpmY%gXfI1{q-koqE{jwP$|iW_|n&mQjE6t)TK z)2Rzn@Ol%nJ4{x&&@&rpiEj@qlU?=&8Iwj%vxb#zw3JlyL%+9yvs294tAxBKIioUmb^_8^i`q95GkxH`SmVxJ zeqkmlb40i=f6~p%Mm}LsBIf(8*tYNM9m$^C@3>H{qZef({yCK3A@kO-TW%Vw@U%6NriNXM@}=OPb+>EJkdNQBd%^g-zCB zHJCaZ8tLa`EE7wQun?KRrRnlYp>?h!n3K>48i_Rt=QSP|x`)w-PRfiNUDJ_dnQU)o zUQRQ?eTo~WrA+Uz^EU)>k*AV71cy^d=8P~$Sla}7cNjpfP7F69l~@o1)H{~*4KE%_ zE-tok)U8%7qtrk#cDNV>U@C z6X(XJ&1>+4v6R%^4yBizR)&%{y^UR@X1aK;EW$9?lOF7S%F%^8UPN0ArG!UZ@~F93 z+C}KeC6pf#2PQZR(^y@rL7vy69o1KJ-0A5uLdV-pI*w1v;}x2#(^$}jsc{-C2X1Cwh5b7S{PNXtRYvTAI&k-k_& zb!|YCJ4G9J0(QBxMeKuqMphRoQc3Y?C(LAtRsv~`sd`17x7Ifdno}O#TYWaIif1n+ zG-uHRgGqfkNO=&fTp(?vvAcUoQ3oJpmRK+Wa=Wf?yWcE`pe_Uu$0CXDh5F@E^Nnn> ze4Z3ANOTSqI=0-1jkW13W7SriD|DuXdU6H=81}JxE0dH|wSi1>r!T;;C(9(vgCg@f zmw8~h=Rj737=`zW3X_?AVc$)^ulhl~50HzbhC1Q6CpZ+^8_UmiPJj@7`P-vZvEjnq z$uGS0o0+rZ^;ac!IaceULtgi-T~k+sNmNTwf~(PR0vU)9usyETz#b z(ujHJm2x+^9!A@gZIU1Hxj=dsy&TPAdQ0{nggns`I_)H*aRVTdyyBY@4%wbzMKG1k{>|zY?>;PDyy3)DXQOQBFP>W#sT> zua7?MsOQM^yIRej=A^M?1z=h~j2-sUAl7ZrZ&%GMey#VB5Srt!!!87fR9rFQge-v| zXxCKswCm}Gn+h-jo~tBwpU%3te%8Y#&!GQ{ddow7&0{{7OMfi8GX0Vg>4ig(ddp}B z;PX<_QxZ;w$uwC?lToE)KkDRX)h|X1M{hO0)_HxDQ7#%3q}YSmo4ZC~W>V3*Y5gx` z&-wU)#*a4HVXQDL1oFXhNcODy%S*RqA2mKLH`;tU=$-9bn`vO(p#P5S4c#B?g@@m$ zZ}nUE{;2kBz(vPX31rph8dk*Hz0Y{i9G_A50}Qkv)`>e<`)Y3e>QCwbZ+vq z`m52eum7;|M>%J@4OD0tCZ&$jb(UJgT6WLmyPB73=|Q)}LJcE=KR)q$=U*rL{C?>D zIwX$MPG2?i$19gCkp7F3JIn3BdlV}A`N%UxpAIkM*Uj`D^#~yGY8A3 zO>o0W^;9nj7(~C!K0Wnf;9H?xNF^nKp&^B^GN_wqIM3a-#mJYMPa0w*QraGK`({Si z?FJeHKo@snLb{ZrJqUmAgd9;0;my~E|^=c76R&SNRZV!d?D! z+pfqMGsa}*w6jIxwL=wrpg&s!QVq*Qlb@*!0=NoV$Ws-9+9J?Ss{gD^)Xt8g=O{ri zZH!q(DPz<+f;GKGw#Uy3=~48!Zw5s3q}dPKdCvS?p>7qP;>rd=Hf#E~L$+xdAlWc4 z6=a#!G6U~MSx8clmG@T&Sy(fxdlSkEfkx{=%FmV!Cc_9>XVvD-Z>9XyuNqXF2A$pv z2Kl_o54@(KD2wdV{lCmcI|%zB;Lw+l83F|wwX{9ALeSDO9v(B4M}aR#4gIOxwhO5s z!)!eq^yy6ofDyxh22ev>CC}0N^rQ};-Rg*{RRYDmK0V3H_TSd&cj|x>JLs*LbDdW- zq%>wTE6UYM$8VzrBgqu)?bZuNciWK7Umw671SbQP1;rn<8>hoX;ELK6G|xb(%y3c- zCnX))j)f1X8Si#JJ_J6UE?bbo3jVVQ8%50;{FXXc<Zv;+H*ozLs;sWecZ^MSN#|K5d8CWVFgpzP7_TpmtbAJw0MLC!9I$gR@jf7-Nnx z%S66^?6C4Q-TCRzD{Grn#F*>gJr89!gHc;Ij ziFv$VPYx6|gQxaiO?$Hfa(I3M?tT04M~xqcAmt8Vnp-Y%e14a6HEWu=4QXis)MD~R zANazR9Kn-{#+gfmFS~$poHg`>|NmKjzvP=$e!7|atsPVU>tmk<8wDfkmpmBJpbs4o zj4?MvJ$j4JH@$VB@5u>l0As(Xd${aW{Fif2<oqLg07oACQ{!-yZhdXP2=fZhI)^2CY7$xK? z(<l9 zk67<;CeC}B#-A!<99=K6*t6SU^r8P>J69eQWqRiQ?xVl2zrJs1j^;*`@r*NLCgYgQ zk)$TG$xbp!ZKgKYI?g7ktt4BM+H62Bn+Cb$l3PTMra^8HrMw%@#1a>=NKHR<22JrQ-j(8YSyIWiz$2W+ z+l-V-tF+hLYqSXm*WQ?v*6U2*FnwCUYmkRxYe?$kfo4sn`kq@a#DJ=gGl_a%P%?r z*zor34_n^KeiUeWt?jdzJvTE1BOb;*RodKdH~nD72<3Uquuq6Mq!PVKCNOiN8t*da z1ZGKW3(&vXwX0(H*q&MUjKg_~0eOVakk6!aMH8nqA6@rM+?%KA{@9oTjY)D`@;s71 zj`qEDWaS^G_U7X2NE7f|vT)qfqJBc8Qw5;(HPA6cDLEv*RtZ(Dz*V8pYzQ3l*>&%F z-GUzKIymW8=v$_`cC76046g>ixAu|kw`pgfXBm*`Xrl9^-_+i^z{GFVA2AB7{51b= zuYASYTb4C6<_*)2ueqB&6PO)gv<1t;+9WHQRGF1up^BEwX&<_FDWhGD0y|xJApfnj zeZf?s*Pu(7E=;F?JNb2^Q?A&?Gr8$~f)76o2-77xN2Ea+-UBYC->*q*yj1sSiRrZo z>41w>UHZp$<(edCXQ9)DMmp-j?_zyQP9Te_+}6d0+TdE)a~5uT)~C}M(hvMMI7mN zUX$46^)3l+1}<0>A{y8~E=ZNdNNfjI@EF3ts}qeVGDH)*(zJDIt8R?Q<1UGtxP`!H zM>Eu*C%KxHWnSggzqUdvT+AEY!WysmKgAF=+FbEwM@SQ0*Y*hWxXJy!$+3a8=C9{L zu`QLFcGc;OPTlX8MFG>6#D$V}-wvO~I4F8vQ?l!fdx=|tN3kGC-u>n%Z&*An48%~;Fcar*NxH} z|1{11BxV@N@So64&=pkv*F$`xS7RWH$ik0sNoB`~RJ0D7LF14N43zB8samQL*Q_IH zl!?j0lm>VPt*7l!?3!`_GQi2`FrG&i*i{y_1+!plJTueA8^Xtk7A}Nu<5{72WLkKd zsiuGqL=o`-(&PEOInj)2LTcsLP(~TdH$Rvhp>(&9cTRnT;w*oiY)#4{QhEbyR29??7HfkBweIyh|)uL=M~QgZ`0)n zXi{o)ik5_kYA^#FLqyP4fH1SD_;9F&%L6Sqftg`@gl1*3FnM=p=tU^^6)V3i_YX!v zzQUqTb`Jd{_mw4|RQHKpdY_QMnSX3}3hx7HicJu0tB7s!P#Tqt_Ml6&o=X&}l%P%35`x zNOQqdFlK|NQ0qlyEK&Pjtl-WGXhvo5nZ#{)8vhxdh~L~bqgq!M62bm@O^4?S{)9h{ zTIgnQoR{cXUi+lB&9zBmQ5(2=+6eHET|Ok%YuY3;>?C=K9>Tkr#~0`bI!vA{OqB(T zOoA~%gnV6@wR_g3gWvFtI$gJ)MXlGAx|yUy;t`KHrA?vNRH)3#F{gYFle9)_Q(Tkj zHPNm)E-OAW{?}#Et^>f#IAx-9mA_qADs7Wk)Zw3reAIXvYXV z!PH@9c%~3KTx6U+=VfnloQ4`m~ zMbHL#k~Di>-?tDrci8yOh=(B{>(Gkkx-1!O;O z_$y#p>sFR;VA^XN3SvM#hm~CJxCW?l5Uo3~`-G7EVa*^0K{& zepx$*Ttaj;+6B#9eMl-g2&I4`(@1CX(`6~bVnHo-o7XPMRJXgXYR)TGHBsCK1G)(R zl%!w-8$SV+S(GaUF+sQhNsukQ75xx9P@ z=w1>q6f5fjvvx)YnB2yE6P*WKW>k~TQ_kIFnb1uAC<@Mj=FvsOOx=Rpd6jIru#XLC zI?!pBofD@BH<-tgD6W$2=dW_jns!Mwds|Q<80Dp)$9vFCTvJLeU`AvD$>KL@X0>aQ z7H&e?!Ub!?=vmgHHo0xKr8uC0QE*Mx?Y!V?!)ka%vN5%uo)ry{NxKuB*OkWCQr#y( z5-#np5dm1Ih>;`;yBMQj^K>ofHW6tWRg7F=Eb6koEw9FT+Cf^^1OEBEbtVhUxNWW$ z_h$YR_jN)qTq7U*nv`K~lWaa-q_*1?~Mb3a+Itkf)h5vTC*#D?xhDQZhu;q)DST@wk;ALqq`-zQLRm&x;<& z+a&~E`~V#Wn+#J*v?Sa6?>C1Q!Q&`xi!eEWgE&-Fl{XA2I)_yGa!a@ks9vvgK!Z`2oQ5t|gih^c?RdZd7eB!{TsxlwS~eK^RLt z@p&N6RG)Kg{+hz_Fjch`!hy_)?SkrMRX98eMap=7j*4bd5DotivC zcs0<$X>erH-+aVtq~}G|vWx0k^%Z4Qr~i{ZU5B4Zqsc~pi#lJS=Rl=!8v1$#Ys9`` z;9^`O&}(!TenXJ}GGfg4nO~~A3Cz<~JJXN|xR1)BbCEp6hz`=n3;;U!yk2+E%o`Qj zcuQaqGV`P173ibWcrG3a*TBv2u~hWf@i}aDLe(ZZ<(w)raACZ+L(v+v9~#C}#C_;v zJQGe7f6ku8-;PA9kxX=s*AGQ;?UGvU1!<7>5!cM62#cKh-23^FT&#%<*WH#UdTc1; zJhB)my?6stf?e{QR;_ZYKCS!vu@t=A*mx1Nx#AAeuK$`(5xl zPsWqw!n@=uVLvqJa>}(m9E$Wwrv`=YMu>a>ts{DXTQ6e;XacVU>11xbeb%!CZO2c+ zYr-J*9KK`UBREAEB9bVX9LR2qMST)yH43NufSmyxs71k(-PXW4b-Sl~0NM)B0| ziPN0-6Iqdai=tGT%*`<~WF;F+S*RaX!Yyz%beqgXCb4GJw-7g>*-QmmBe}%4;8!&* z+>$s;m*st3m`cRDw@Gre*Q5p77|Eg>eeXoORp-KhI{2N0&i`~OIu@h)9)qochne;s(o*ksK>1A+#88;EnkPP?= zandc)<#X4w{IN*LOe9E~WOby8XJYG_F3gID_o|;lv!K!h|NYuGTLe$dNG|n|mw_A7 zku&5ledp(W>^x@pZX-YP=rk`$aYuL&t;VN|-hBM+a{o16+L(Ib5;}PYO07W7cas}l z1(`^--~9+^{yf#5hD`l2?{nCe4%^&^-(PRQik*$$P1?QTISVIy74hpXzze%FoYpvl z2`>A{`j);9YPt^@YvGPA*eE{r({PVGrj2iCQdrXiKcAAt?TOh_!RG9|Cd+#z(b=jV z!a>Vttyuj4VP|K&`A~FOawJbgC5|<+T%w z*xg;9l!LOIf`($@^hvrE4#kcekuIhMAlYLPbQ!7VuE_>Kf!&;mlF_bB3?A4)m;h?q z{3Apgn4yheV(t;HNkXt=Ay^q{rx&mqksT~Ia29os(1=*oa{ws!2*(9;SPPsh4#A=& zJ;Gz-f=MY5G7kFJ5!s$IfDNeER2za(KVt7IvG8FJOnd?5nxR+2e==r6P1#s zWGY%C3c>8a!eq1zdMD(~?E(0RYLdl2P6&m*jlLJrp}h*NRfmPX{0~W8mhn zA$&(CGs^1~-Vv;FDd@fuWFAYGHPU-`j3atYu&9tXZD&i|nl;({6torY6{etfrRC%> z-n;vr^qw>nYmk4gjpEL@XL?7z-2i&+=$|^7N8B(T$t}}YejY1X)TLARq^C#xCldap z_Us4AIez0v3%rilcovaSFa;3Uymu7W3h%f?PpKxQlVprU&o0pebQZr3fYut(f@_>0 zk?cXI&^c-jc(YrY%C>+(sfsdiL*i66+P6$N0LG6G?g#wmf0Bfezdrx+N##(J-_&JB<=azTS$75+m~5Br_>gjIg;bRAw@8%Mp9t_1hk|twZvnXSWE-mI<(hSjSwxg7-E)eIdoRSK_bxo@e!<*^b#d zJU1_tiyaPFJLdeN$ywmTTkrrS1 z-r@IlU)a;K?=OAvjop#wA8`=%Ed43FF9Vc`-T>LXXp4e zO@HBFZ|7+L<^TTOkF?+yfBmb^X~8ACZ|!N_zq;(;__gAvm+h`N+I^$=yUTW8(vmCo zUtGTQrJVz|Wq0MfukF5Zq=nzuU9!JIi>_S$-T|8yf9G)VTVNng(^|i=zv8IUppqA1 z2Y;~tkw%X84%h`ybH(1_hl^NF=Pxe*;HdbOqn+c$%a>H#=u3x-KiI>-M3*mdHF@bW z*!$7mffoJF;o_HH+0(=+n)(sj*HYQP@Rj}LZ|xniX8q6Y9PEDhUhy~Bzt^=IeQx&$ zI|s$zIM{!C@%wKTfBCK5SN0A+Dvo_-?`UU5OaAuaH+CvUp`(M{7ypOW{@m`n-(CKL z;#YW*zWL^&qdhI*9ytE^ojoo3%E9hS`!lro_m?j_Vh?ruzG59v@q;m1mn2A*qy(lk zdPn=6!|y2GIVSw&bu#`9kCi4i7g zgmGzRT+Y{8T^Kn&qaS86VbguG2BCw7w%;6g*_xv`anZTWgGy+10a2=N{^m z(ur)g*)@UB$u6?VH0T&>jUMy4uhn!9d+#Nkmw9OxD3%%a6I98^Yu4O{c_&QMC2{)Y zCttbAgRaWG&7Ro$`p8|{nGK5vmK`*E9@Xs1i`3DG&RM3)SDs=v zC9XC!$vxeI*wd(v&-9_97A)#v`@l9s_SC%nXvmQjPr1u1!`h;HK68w_clsC+SMiF7 zKrJ>Aw4dnKQTAu30r>H@A;fCj$9-WLS4I9C%B)X^HZk-wOJdg!W^9UiN=?w2E?Zfj z=D)=Kbp-n%YE)T8*i(s`9>eNPtY^hZzS_#niQ9Xb!WPEkln0q@RAs=Lq9#6X#W4+7 zE96~GU}q9XuKTD#?5$aS&TS;TU|2JX35^ zwi=!Z9_O?NpPHt^=9!$zvfxvxrMQ_15ES%yPOxw@DBh;SAWB)ZU|qMCvS2;1%5)ZX z2cKFCteF##cgpeO{?5Spvt1b&UZ!HLskbq7N8$%n6&O;^NBQ82B%fsBsuJMpJq)e@ zWLpWw|La;cKXz^|2*Uu13&uT<<1v&0kh29Ce-Q)t0y-7}UX%g_5buSdOFP8O*~UKz zbl=A?X?Twr_Q1qiEQW0DYIs5riD|gX5hlrMBS3SkC&pO4QW)Y$D26CmvQeQ)I(R*s zgTY7KsvkuK7!-=z!Bp4X%{ZH@oy?{n`MC4D2{X)-Q6}BFca~6I%geLA8iv6MfFQP@ zgQC1s(hIB!A3%=020%pt&#RzSK{zy{0B&**a7p9jZv%3qC>QV^Kt4bw0GWFrc^{DC zc*#OQDb){puLrLRNKzt`kmb-@z%h`IIPM}!k~Vy?x}uBlk52LM&c z^*#Zt=R9xNBijI*R7yCSYUUa}98FQ(JLX1!7zdczek7a+;sw`Q0bB#nz!MtI$q0SI zRg7U!6XAfJS;6rJMH6wsI%El8AGFuobMve>0U}hAWbC#i71)srD-{5hKY&+qY_N_e z0Dk*Jpa*l^4Y-fF@&HvNMc#)x2S7zq!XmCwrE0*)fO<92?Jw(C4XXH%;M!y#V6ks@ zoWTtjj}V=`CK&)h3_a6P3E>R0p~nVyU^iQP3IHk>_4F%z9{(T#bt0< z$~)gJl6Dx@n+(Z%h%wrS7}4w1%higOG>OJfc#uu)jMp^NV?0f!TP=<-Z4=BZ-Gz_p zWs67HW2krM^r6&`;goc&+0#XSkDSA+)X(R|FCwxqud|E%;WH-J&{rNo|71ku zOKy<;w6k`ft*gXU-cQz$kwh8Yr4wcldtzSUcO)KdkLw^Qk6qKrq(}AckWDoE z0UF>l!*UZ=Xx>1+wEk$*WL4BE$q`Rz1R5{vvJoU20ZCZk*~8-P5>!vi-z$?J^y|kE70~cQ8H6Oo8^WM*Q({+lTZu+Zj>O zF@MpM_Y&OYetk@XMxJ3O8?IM#S1;~2e_{HOk{q;2 zWL?2=|XP7O6-(5hKa-#^)WxInW&_jxN5N#+ePU9$ee(TXIgmr>aJajgvQ z36%b}`(73V7Ln~*NkCYuu3SOU?Qs?Lx)zAAb|{S!(`1o&aM)tnORe%b-m_o+h%9^ zW-s%s0Y3^~$KYx_LnU=$;?WdU|84nU|c1k{Y;6Myi&)BVSj=~`PS-aW8^iLpXD{pT>q>> z>RlZ>dTzEHLlNb1RtX+0^kNtxakv=g4Hsdlc(upPay7Dad^A=}uIr?kl}JnU<8+=J zs`QX&%`r*qIG+q!!cZx_qv5oIi5j2jtWT%6m}g3c{Vkf{)BQ<7E!JJ6mHd9%i6kvPQ40@Z_Xf2~}b_uj{XsPozqQ#i9D_88pSG z`fzqLR()0HA#3D*`o6G3T%eP8)Jqa}h;=5=Grs)wtIA|7<*utdQgtZmaE*n(v1=c* zwZTM~)d{zxi$WK{Q{m{LUk2EgNHX)E`JN#cwx)c~TsH`e#El^(rbhuDXX0ixKsh|9 zWnV!rdFOV!@0q&*tPPsx^IS)D#XOOyovq;~T_*fVe8HRLz2Hq&VeTsZTh=(Af6g~9 zLoPL)R$QYp&3)J?lPFn+a6xoK)F2o-#{NB;@qDno1xFf5@_*tP3j)F z5ZtvYyYv-NDhV;IOMXSnC%RreHa3iN@2J|^;8Rt-7y_4ZbGj2lvSCvX?g$ygkSBfM zE|2Jb6nx5Us4e(ZjMbc^oKC36WCnB&ZoW`j290B(d0B~BvR0{|yjlrepv-x#Dfm=$ z7lu?V_UEYYb{eCaB0Y5nt&7_4t{Nq~L)wv(K*ACbT~wK}7%Ct$*9Pg~=5FL=ekRF9A@~fWdXO zGe(^)p#@|Yz}qRJn?s^QRo&_gDIg2}AKAk-W1tlaz)?grheVvJQZFvh$y%auZb({$ z9!oKLht!5dbBkBplIfzs4vNvsfqDmBA+4!}=TWGu!)UATnqqN6oogG;PQ3$hg5?Lm zQDguhPW&Rc)=j`0evsHD15!y`cLrb@0C9VlOwX^F6Li&VB%Evq7z4lwWR$Bi;p9yK zUq}#%5WvjzbZ()@d?2Xm=z(sVI)Lp5_n{J(sTws<390QC*}#cV)!WS;Ra4jwb(g9! z$~~I`oY`huh58x5EXOm?A-Mq6a{!K_mH;_k>Ln++0Vs9K;Iw!^#jz{{xI$C4KpY$O zDxK@#8tVX?0NcI98pl%%^H?1HDqJTYfDjXppoh~%`ebb1ohWGvkheJtE4euR7rETU zNad(JAq#gosd^xv04$ELC!AacpqjS%#sn|m-Xx{m;gTaB7LY>%7a+N|7kBfaJfZRd zAbbG>073!CNWi^LDvawDqMIs7D(pUfy*1CID9jO8ccc9J&kP(DOidTdV>p~%= zXIV0-OA&{0ESDRohssg1wG@1&!^@%{Hcjd}P>W zoTPQ*-DRu0$K!yL5Y&J(N~R4>I*Z`b zj}P@~rs*!|4lz+?A2rZFs~%zMXYmwtRbocg8f|flAYct~oD~?oyD!zG`$ra=ju(2@ zPFb->k&_qj9(1zN+{xA*RNSS>zK7h`M=>84}C+}Cf( zi(FNw)K#u>+CH*2I<}84XXxP0EW&l?ICC#*_ma7UJ}cMU{IXpB!O zk&%f)eOhV==k&2o8!~0FepQ|bKqdjk1ME?xXMg{y+zo)ENDsh30KZ*I{Mz$-*NM3_ z-!<_fzzYBYN8-q_+dZf1*e-YD*ItV60Far0w*j&NGIps{?(x3TpYoFHWY3}=WqjfG z@@wzQM7a;0x8OskPZG|!$R;?(s{jP-KO>K;&uhp7Q`ny4M(DzoU3> z{?n!NSy>I! z1pszWqEz0&Fqxxo42Jy-rSS?JU7Fby@Is!*2Oa1E`)icWtK)osB~JR1VsN7MM~70Q zuGSRY=T$4}p8(g`O@~Z#JBpHVF zc?wKNQ7!M#=p}e#{}r?S0~YX4CFO9~DK z^?MGP?h6LKi#)HNO7^c&1MgtYI4ki))-LT%Bwy+Es3;@{07&+KqKAXNy$37&^m*>u z+UeH>mmZh&iIxFyLkTr`wGvwHfF2y_3+Ts@1bYG%^J>=_d;;-!6yHd9MrVpDCRFTt zcyFs;y+j4!FR(M1=P^G|v5My?s^sONpBKW1Y`9aiHmGWX&pqu5U*-1T7nUI8;Xqu? z%Pnc#;b#8~)$p<%am~&ix0>v22*IaB`%FQfr~}~j6}CaOyuNEkX47o4Qln()2IzjY?F(iQ|nu zz{_JCH-25vSM3G%9|%V30FCH+F8T8pFbg`s26OT!+u)M-v-cF2m=Vj^`_0 zZZh6PVm}ivxoX}fXuj}7+gFmS9Vl+I$|38`k_GIkZN0hJlW7sC+fq85?`{JyT!q^X~(<>f<6!Vq(RoN2LawWx;A@;xBd58yVAn-9h?pW-eW(89=rhdgF=-orw4D9$)4#u=LjJV*@3SPR!rxVhecixbNY^=ARaDR0dVcu zHABHXe3FVCKK2td&{`OZ?eJ{#$FlTE`D=0Z{9Ag(_}wI%AP&>{gJ@b9TMKf zl;OG$dAIRTQTPJC28-vbQ!y8ahZ#gs1|U9F2s2Va7?Otx;r8350au0AEq*12)$`*q z7kpzx2iZT1+TJ2+z=@_r?-zY~1)tIFJA&pcWol-j{_l60fwM6e_DKm9k5XjDv|!`<9Q!-dTs>q1yudedv$fHxeSVYP3jEt z^x2czKYepH!M(EZF9&oEFF2`B8Y~E{rCJ%)C_y2<6b;R~1ycv+i`o%!tMwN&_DsVsu*;tA z!3U4WmAD0K*p&S5Z@m4)-lo{D{j)c-Bs_W2UU}TMoUOK6fA;Z#%eGS{oJDLMf#!3; zM>q^j)jZ55x*Kqij`P`E{cz7EZf>?KoUo`nsv%JIb5pPvh4(MP?Ss}&xI5b*sy6)B z=V!;TdPC(EYlW&y-FenWqqEexVQ2qnuz!!9l=HEk>xPeyNn5IKjNp-E$060)s?14= zo!!6VFV0!38~L>AR+sHMustGnl|V1Y51#&BqCcx>f(UG@Q~i&@{ynifDLeD5!+8TU zvpe&yh8q;X?&Mc*d}z4u$$ z4S@oD@r{q)kyQV$M4e*DB?AX=+}*ibS)l|TX07Ies=iNTd)F`v Date: Wed, 3 Nov 2021 17:24:26 +0800 Subject: [PATCH 176/187] update UI version for cypress tests --- .github/workflows/cypress-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml index c9aa49402..0fd16c257 100644 --- a/.github/workflows/cypress-integration.yml +++ b/.github/workflows/cypress-integration.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 with: repository: conveyal/analysis-ui - ref: f0080035c1562ebe940d5e6a444420f423f1c6a7 + ref: 1701bd9aea859a4714bc3f35f5bcf767a3256a64 path: ui - uses: actions/checkout@v2 with: From 1767f23fdd877e5531ab8642f5e9d3f6833333a1 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 3 Nov 2021 23:39:01 +0800 Subject: [PATCH 177/187] add test csv file with excessive geographic extent --- .../resources/com/conveyal/analysis/datasource/cities.csv | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/test/resources/com/conveyal/analysis/datasource/cities.csv diff --git a/src/test/resources/com/conveyal/analysis/datasource/cities.csv b/src/test/resources/com/conveyal/analysis/datasource/cities.csv new file mode 100644 index 000000000..69701e1bf --- /dev/null +++ b/src/test/resources/com/conveyal/analysis/datasource/cities.csv @@ -0,0 +1,5 @@ +id,name,lon,lat,metropop +1,Hong Kong,114.1621809593,22.2783740471,7.501 +2,Shanghai,121.4682654779,31.2461374983,24.871 +3,Tokyo,139.7514493987,35.681394636,37.468 +4,Singapore,103.8451699322,1.2908769445,5.454 \ No newline at end of file From e0f0c4e5c0cf10942cee68a5e20e917fc8d2ae71 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 9 Nov 2021 10:56:17 +0800 Subject: [PATCH 178/187] Revert "Revert "Use getPriority method instead of a field"" This reverts commit f399f90ee496b4498d8d55f49f6d7966c02b46af. This is a double-revert, re-establishing the original use of getPriority methods instead of priority fields. The hastily applied patch where we shifted to using the field instead of methods was based on a false belief that old MapDBs had this field. In fact the field was a temporary addition during development, and the original state was without the field. --- .../com/conveyal/analysis/models/Bundle.java | 1 - .../conveyal/gtfs/error/DateParseError.java | 6 +++++- .../conveyal/gtfs/error/DuplicateKeyError.java | 6 +++++- .../gtfs/error/DuplicateStopError.java | 6 +++++- .../gtfs/error/DuplicateTripError.java | 6 +++++- .../conveyal/gtfs/error/EmptyFieldError.java | 5 ++++- .../conveyal/gtfs/error/EmptyTableError.java | 6 +++++- .../com/conveyal/gtfs/error/GTFSError.java | 18 ++++++++++-------- .../com/conveyal/gtfs/error/GeneralError.java | 5 ++++- .../gtfs/error/MisplacedStopError.java | 8 +++++--- .../gtfs/error/MissingColumnError.java | 5 ++++- .../conveyal/gtfs/error/MissingShapeError.java | 6 +++++- .../conveyal/gtfs/error/MissingTableError.java | 5 ++++- .../gtfs/error/NoAgencyInFeedError.java | 6 +++++- .../conveyal/gtfs/error/NumberParseError.java | 5 ++++- .../error/OverlappingTripsInBlockError.java | 6 +++++- .../com/conveyal/gtfs/error/RangeError.java | 5 ++++- .../gtfs/error/ReferentialIntegrityError.java | 6 +++++- .../gtfs/error/ReversedTripShapeError.java | 6 +++++- .../error/ShapeMissingCoordinatesError.java | 6 +++++- .../gtfs/error/TableInSubdirectoryError.java | 6 +++++- .../conveyal/gtfs/error/TimeParseError.java | 5 ++++- .../com/conveyal/gtfs/error/TimeZoneError.java | 6 +++++- .../com/conveyal/gtfs/error/URLParseError.java | 5 ++++- .../conveyal/gtfs/error/UnusedStopError.java | 6 +++++- 25 files changed, 117 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/Bundle.java b/src/main/java/com/conveyal/analysis/models/Bundle.java index 1f17abd93..31f2e6794 100644 --- a/src/main/java/com/conveyal/analysis/models/Bundle.java +++ b/src/main/java/com/conveyal/analysis/models/Bundle.java @@ -82,7 +82,6 @@ public static class GtfsErrorTypeSummary { public Priority priority; public GtfsErrorTypeSummary () { /* For deserialization. */ } public GtfsErrorTypeSummary (GTFSError error) { - this.priority = error.priority; this.type = error.errorType; } } diff --git a/src/main/java/com/conveyal/gtfs/error/DateParseError.java b/src/main/java/com/conveyal/gtfs/error/DateParseError.java index a02d55c70..566dee393 100644 --- a/src/main/java/com/conveyal/gtfs/error/DateParseError.java +++ b/src/main/java/com/conveyal/gtfs/error/DateParseError.java @@ -9,10 +9,14 @@ public class DateParseError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public DateParseError(String file, long line, String field) { - super(file, line, field, Priority.MEDIUM); + super(file, line, field); } @Override public String getMessage() { return "Could not parse date (format should be YYYYMMDD)."; } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/DuplicateKeyError.java b/src/main/java/com/conveyal/gtfs/error/DuplicateKeyError.java index fda04d977..9630449f4 100644 --- a/src/main/java/com/conveyal/gtfs/error/DuplicateKeyError.java +++ b/src/main/java/com/conveyal/gtfs/error/DuplicateKeyError.java @@ -9,10 +9,14 @@ public class DuplicateKeyError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public DuplicateKeyError(String file, long line, String field) { - super(file, line, field, Priority.MEDIUM); + super(file, line, field); } @Override public String getMessage() { return "Duplicate primary key."; } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/DuplicateStopError.java b/src/main/java/com/conveyal/gtfs/error/DuplicateStopError.java index 85007c8ae..1d448a1fb 100644 --- a/src/main/java/com/conveyal/gtfs/error/DuplicateStopError.java +++ b/src/main/java/com/conveyal/gtfs/error/DuplicateStopError.java @@ -13,7 +13,7 @@ public class DuplicateStopError extends GTFSError implements Serializable { public final DuplicateStops duplicateStop; public DuplicateStopError(DuplicateStops duplicateStop) { - super("stop", duplicateStop.getDuplicatedStop().sourceFileLine, "stop_lat,stop_lon", Priority.MEDIUM, duplicateStop.getDuplicatedStop().stop_id); + super("stop", duplicateStop.getDuplicatedStop().sourceFileLine, "stop_lat,stop_lon", duplicateStop.getDuplicatedStop().stop_id); this.message = duplicateStop.toString(); this.duplicateStop = duplicateStop; } @@ -21,4 +21,8 @@ public DuplicateStopError(DuplicateStops duplicateStop) { @Override public String getMessage() { return message; } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/DuplicateTripError.java b/src/main/java/com/conveyal/gtfs/error/DuplicateTripError.java index 7c35a2d32..964b5d5f9 100644 --- a/src/main/java/com/conveyal/gtfs/error/DuplicateTripError.java +++ b/src/main/java/com/conveyal/gtfs/error/DuplicateTripError.java @@ -20,7 +20,7 @@ public class DuplicateTripError extends GTFSError implements Serializable { String lastArrival; public DuplicateTripError(Trip trip, long line, String duplicateTripId, String patternName, String firstDeparture, String lastArrival) { - super("trips", line, "trip_id", Priority.MEDIUM, trip.trip_id); + super("trips", line, "trip_id", trip.trip_id); this.duplicateTripId = duplicateTripId; this.patternName = patternName; this.routeId = trip.route_id; @@ -33,4 +33,8 @@ public DuplicateTripError(Trip trip, long line, String duplicateTripId, String p @Override public String getMessage() { return String.format("Trip Ids %s & %s (route %s) are duplicates (pattern: %s, calendar: %s, from %s to %s)", duplicateTripId, affectedEntityId, routeId, patternName, serviceId, firstDeparture, lastArrival); } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/EmptyFieldError.java b/src/main/java/com/conveyal/gtfs/error/EmptyFieldError.java index badfb0ab0..4703df572 100644 --- a/src/main/java/com/conveyal/gtfs/error/EmptyFieldError.java +++ b/src/main/java/com/conveyal/gtfs/error/EmptyFieldError.java @@ -9,11 +9,14 @@ public class EmptyFieldError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public EmptyFieldError(String file, long line, String field) { - super(file, line, field, Priority.MEDIUM); + super(file, line, field); } @Override public String getMessage() { return String.format("No value supplied for a required column."); } + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/EmptyTableError.java b/src/main/java/com/conveyal/gtfs/error/EmptyTableError.java index f3fda5d4d..60f1e0187 100644 --- a/src/main/java/com/conveyal/gtfs/error/EmptyTableError.java +++ b/src/main/java/com/conveyal/gtfs/error/EmptyTableError.java @@ -11,10 +11,14 @@ public class EmptyTableError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public EmptyTableError(String file) { - super(file, 0, null, Priority.MEDIUM); + super(file, 0, null); } @Override public String getMessage() { return String.format("Table is present in zip file, but it has no entries."); } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/GTFSError.java b/src/main/java/com/conveyal/gtfs/error/GTFSError.java index 95a41e4c6..84680a82d 100644 --- a/src/main/java/com/conveyal/gtfs/error/GTFSError.java +++ b/src/main/java/com/conveyal/gtfs/error/GTFSError.java @@ -18,21 +18,17 @@ public abstract class GTFSError implements Comparable, Serializable { public final String field; public final String affectedEntityId; public final String errorType; - // NOTE: Do not remove this field. Though this field is somewhat redundant (since every instance of each class has - // the same priority) we have old MapDB files around that contain serialized errors. They would all break. - public final Priority priority; - public GTFSError(String file, long line, String field, Priority priority) { - this(file, line, field, priority, null); + public GTFSError(String file, long line, String field) { + this(file, line, field, null); } - public GTFSError(String file, long line, String field, Priority priority, String affectedEntityId) { + public GTFSError(String file, long line, String field, String affectedEntityId) { this.file = file; this.line = line; this.field = field; this.affectedEntityId = affectedEntityId; this.errorType = this.getClass().getSimpleName(); - this.priority = priority; } /** @@ -44,7 +40,6 @@ public GTFSError (String entityId) { this.field = null; this.errorType = null; this.affectedEntityId = entityId; - this.priority = Priority.UNKNOWN; } /** @@ -55,6 +50,13 @@ public final String getErrorCode () { return this.getClass().getSimpleName(); } + /** + * @return The Error priority level associated with this class. + */ + public Priority getPriority() { + return Priority.UNKNOWN; + } + /** * @return a Class object for the class of GTFS entity in which the error was found, * which also implies a table in the GTFS feed. diff --git a/src/main/java/com/conveyal/gtfs/error/GeneralError.java b/src/main/java/com/conveyal/gtfs/error/GeneralError.java index 54eb270b3..35219eb21 100644 --- a/src/main/java/com/conveyal/gtfs/error/GeneralError.java +++ b/src/main/java/com/conveyal/gtfs/error/GeneralError.java @@ -11,7 +11,7 @@ public class GeneralError extends GTFSError implements Serializable { private String message; public GeneralError(String file, long line, String field, String message) { - super(file, line, field, Priority.UNKNOWN); + super(file, line, field); this.message = message; } @@ -19,4 +19,7 @@ public GeneralError(String file, long line, String field, String message) { return message; } + @Override public Priority getPriority() { + return Priority.UNKNOWN; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/MisplacedStopError.java b/src/main/java/com/conveyal/gtfs/error/MisplacedStopError.java index 99189aa79..991289ec6 100644 --- a/src/main/java/com/conveyal/gtfs/error/MisplacedStopError.java +++ b/src/main/java/com/conveyal/gtfs/error/MisplacedStopError.java @@ -11,16 +11,18 @@ public class MisplacedStopError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; - public final Priority priority; public final Stop stop; public MisplacedStopError(String affectedEntityId, long line, Stop stop) { - super("stops", line, "stop_id", Priority.MEDIUM, affectedEntityId); - this.priority = Priority.HIGH; + super("stops", line, "stop_id", affectedEntityId); this.stop = stop; } @Override public String getMessage() { return String.format("Stop Id %s is misplaced.", affectedEntityId); } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/MissingColumnError.java b/src/main/java/com/conveyal/gtfs/error/MissingColumnError.java index fe9c28f6b..24022a66a 100644 --- a/src/main/java/com/conveyal/gtfs/error/MissingColumnError.java +++ b/src/main/java/com/conveyal/gtfs/error/MissingColumnError.java @@ -9,11 +9,14 @@ public class MissingColumnError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public MissingColumnError(String file, String field) { - super(file, 1, field, Priority.MEDIUM); + super(file, 1, field); } @Override public String getMessage() { return String.format("Missing required column."); } + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/MissingShapeError.java b/src/main/java/com/conveyal/gtfs/error/MissingShapeError.java index 3ddeb8c75..9487d8551 100644 --- a/src/main/java/com/conveyal/gtfs/error/MissingShapeError.java +++ b/src/main/java/com/conveyal/gtfs/error/MissingShapeError.java @@ -12,10 +12,14 @@ public class MissingShapeError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public MissingShapeError(Trip trip) { - super("trips", trip.sourceFileLine, "shape_id", Priority.LOW, trip.trip_id); + super("trips", trip.sourceFileLine, "shape_id", trip.trip_id); } @Override public String getMessage() { return "Trip " + affectedEntityId + " is missing a shape"; } + + @Override public Priority getPriority() { + return Priority.LOW; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/MissingTableError.java b/src/main/java/com/conveyal/gtfs/error/MissingTableError.java index 0ff547881..6a756a8b2 100644 --- a/src/main/java/com/conveyal/gtfs/error/MissingTableError.java +++ b/src/main/java/com/conveyal/gtfs/error/MissingTableError.java @@ -9,11 +9,14 @@ public class MissingTableError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public MissingTableError(String file) { - super(file, 0, null, Priority.MEDIUM); + super(file, 0, null); } @Override public String getMessage() { return String.format("This table is required by the GTFS specification but is missing."); } + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/NoAgencyInFeedError.java b/src/main/java/com/conveyal/gtfs/error/NoAgencyInFeedError.java index 1d06d727f..e82c6895d 100644 --- a/src/main/java/com/conveyal/gtfs/error/NoAgencyInFeedError.java +++ b/src/main/java/com/conveyal/gtfs/error/NoAgencyInFeedError.java @@ -7,10 +7,14 @@ */ public class NoAgencyInFeedError extends GTFSError { public NoAgencyInFeedError() { - super("agency", 0, "agency_id", Priority.HIGH); + super("agency", 0, "agency_id"); } @Override public String getMessage() { return String.format("No agency listed in feed (must have at least one)."); } + + @Override public Priority getPriority() { + return Priority.HIGH; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/NumberParseError.java b/src/main/java/com/conveyal/gtfs/error/NumberParseError.java index ff6bf6085..c691c911a 100644 --- a/src/main/java/com/conveyal/gtfs/error/NumberParseError.java +++ b/src/main/java/com/conveyal/gtfs/error/NumberParseError.java @@ -9,11 +9,14 @@ public class NumberParseError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public NumberParseError(String file, long line, String field) { - super(file, line, field, Priority.HIGH); + super(file, line, field); } @Override public String getMessage() { return String.format("Error parsing a number from a string."); } + @Override public Priority getPriority() { + return Priority.HIGH; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java b/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java index e4e6ea627..459c2527b 100644 --- a/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java +++ b/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java @@ -15,7 +15,7 @@ public class OverlappingTripsInBlockError extends GTFSError implements Serializa public final String routeId; public OverlappingTripsInBlockError(long line, String field, String affectedEntityId, String routeId, String[] tripIds) { - super("trips", line, field, Priority.MEDIUM, affectedEntityId); + super("trips", line, field, affectedEntityId); this.tripIds = tripIds; this.routeId = routeId; } @@ -23,4 +23,8 @@ public OverlappingTripsInBlockError(long line, String field, String affectedEnti @Override public String getMessage() { return String.format("Trip Ids %s overlap (route: %s) and share block ID %s", String.join(" & ", tripIds), routeId, affectedEntityId); } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/RangeError.java b/src/main/java/com/conveyal/gtfs/error/RangeError.java index 2411f40d7..42a0f51e6 100644 --- a/src/main/java/com/conveyal/gtfs/error/RangeError.java +++ b/src/main/java/com/conveyal/gtfs/error/RangeError.java @@ -11,7 +11,7 @@ public class RangeError extends GTFSError implements Serializable { final double min, max, actual; public RangeError(String file, long line, String field, double min, double max, double actual) { - super(file, line, field, Priority.LOW); + super(file, line, field); this.min = min; this.max = max; this.actual = actual; @@ -21,4 +21,7 @@ public RangeError(String file, long line, String field, double min, double max, return String.format("Number %s outside of acceptable range [%s,%s].", actual, min, max); } + @Override public Priority getPriority() { + return Priority.LOW; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/ReferentialIntegrityError.java b/src/main/java/com/conveyal/gtfs/error/ReferentialIntegrityError.java index 5ada67d82..0ef7edad0 100644 --- a/src/main/java/com/conveyal/gtfs/error/ReferentialIntegrityError.java +++ b/src/main/java/com/conveyal/gtfs/error/ReferentialIntegrityError.java @@ -12,7 +12,7 @@ public class ReferentialIntegrityError extends GTFSError implements Serializable public final String badReference; public ReferentialIntegrityError(String tableName, long row, String field, String badReference) { - super(tableName, row, field, Priority.HIGH); + super(tableName, row, field); this.badReference = badReference; } @@ -27,4 +27,8 @@ public int compareTo (GTFSError o) { @Override public String getMessage() { return String.format(badReference); } + + @Override public Priority getPriority() { + return Priority.HIGH; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/ReversedTripShapeError.java b/src/main/java/com/conveyal/gtfs/error/ReversedTripShapeError.java index 9ce7c714f..c29e2439e 100644 --- a/src/main/java/com/conveyal/gtfs/error/ReversedTripShapeError.java +++ b/src/main/java/com/conveyal/gtfs/error/ReversedTripShapeError.java @@ -14,11 +14,15 @@ public class ReversedTripShapeError extends GTFSError implements Serializable { public final String shapeId; public ReversedTripShapeError(Trip trip) { - super("trips", trip.sourceFileLine, "shape_id", Priority.HIGH, trip.trip_id); + super("trips", trip.sourceFileLine, "shape_id", trip.trip_id); this.shapeId = trip.shape_id; } @Override public String getMessage() { return "Trip " + affectedEntityId + " references reversed shape " + shapeId; } + + @Override public Priority getPriority() { + return Priority.HIGH; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/ShapeMissingCoordinatesError.java b/src/main/java/com/conveyal/gtfs/error/ShapeMissingCoordinatesError.java index 3a7df095a..8ece94a15 100644 --- a/src/main/java/com/conveyal/gtfs/error/ShapeMissingCoordinatesError.java +++ b/src/main/java/com/conveyal/gtfs/error/ShapeMissingCoordinatesError.java @@ -14,11 +14,15 @@ public class ShapeMissingCoordinatesError extends GTFSError implements Serializa public final String[] tripIds; public ShapeMissingCoordinatesError(ShapePoint shapePoint, String[] tripIds) { - super("shapes", shapePoint.sourceFileLine, "shape_id", Priority.MEDIUM, shapePoint.shape_id); + super("shapes", shapePoint.sourceFileLine, "shape_id", shapePoint.shape_id); this.tripIds = tripIds; } @Override public String getMessage() { return "Shape " + affectedEntityId + " is missing coordinates (affects " + tripIds.length + " trips)"; } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/gtfs/error/TableInSubdirectoryError.java b/src/main/java/com/conveyal/gtfs/error/TableInSubdirectoryError.java index ed4ac0d84..e415ef6c8 100644 --- a/src/main/java/com/conveyal/gtfs/error/TableInSubdirectoryError.java +++ b/src/main/java/com/conveyal/gtfs/error/TableInSubdirectoryError.java @@ -13,11 +13,15 @@ public class TableInSubdirectoryError extends GTFSError implements Serializable public final String directory; public TableInSubdirectoryError(String file, String directory) { - super(file, 0, null, Priority.HIGH); + super(file, 0, null); this.directory = directory; } @Override public String getMessage() { return String.format("All GTFS files (including %s.txt) should be at root of zipfile, not nested in subdirectory (%s)", file, directory); } + + @Override public Priority getPriority() { + return Priority.HIGH; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/TimeParseError.java b/src/main/java/com/conveyal/gtfs/error/TimeParseError.java index d6579e715..6dcc4b282 100644 --- a/src/main/java/com/conveyal/gtfs/error/TimeParseError.java +++ b/src/main/java/com/conveyal/gtfs/error/TimeParseError.java @@ -9,11 +9,14 @@ public class TimeParseError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public TimeParseError(String file, long line, String field) { - super(file, line, field, Priority.MEDIUM); + super(file, line, field); } @Override public String getMessage() { return "Could not parse time (format should be HH:MM:SS)."; } + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/TimeZoneError.java b/src/main/java/com/conveyal/gtfs/error/TimeZoneError.java index a1b9155c8..5a9710f2a 100644 --- a/src/main/java/com/conveyal/gtfs/error/TimeZoneError.java +++ b/src/main/java/com/conveyal/gtfs/error/TimeZoneError.java @@ -22,11 +22,15 @@ public class TimeZoneError extends GTFSError implements Serializable { * @param message description of issue with timezone reference */ public TimeZoneError(String tableName, long line, String field, String affectedEntityId, String message) { - super(tableName, line, field, Priority.MEDIUM, affectedEntityId); + super(tableName, line, field, affectedEntityId); this.message = message; } @Override public String getMessage() { return message + ". (" + field + ": " + affectedEntityId + ")"; } + + @Override public Priority getPriority() { + return Priority.MEDIUM; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/URLParseError.java b/src/main/java/com/conveyal/gtfs/error/URLParseError.java index 6a38818a5..6157e861c 100644 --- a/src/main/java/com/conveyal/gtfs/error/URLParseError.java +++ b/src/main/java/com/conveyal/gtfs/error/URLParseError.java @@ -9,11 +9,14 @@ public class URLParseError extends GTFSError implements Serializable { public static final long serialVersionUID = 1L; public URLParseError(String file, long line, String field) { - super(file, line, field, Priority.LOW); + super(file, line, field); } @Override public String getMessage() { return "Could not parse URL (format should be ://?#)."; } + @Override public Priority getPriority() { + return Priority.LOW; + } } diff --git a/src/main/java/com/conveyal/gtfs/error/UnusedStopError.java b/src/main/java/com/conveyal/gtfs/error/UnusedStopError.java index 16c237c93..be2c9ea47 100644 --- a/src/main/java/com/conveyal/gtfs/error/UnusedStopError.java +++ b/src/main/java/com/conveyal/gtfs/error/UnusedStopError.java @@ -12,11 +12,15 @@ public class UnusedStopError extends GTFSError implements Serializable { public final Stop stop; public UnusedStopError(Stop stop) { - super("stops", stop.sourceFileLine, "stop_id", Priority.LOW, stop.stop_id); + super("stops", stop.sourceFileLine, "stop_id", stop.stop_id); this.stop = stop; } @Override public String getMessage() { return String.format("Stop Id %s is not used in any trips.", affectedEntityId); } + + @Override public Priority getPriority() { + return Priority.LOW; + } } From fb24d5fb5a3ca809d0bf735901652fff54ee2df3 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 9 Nov 2021 12:38:28 +0800 Subject: [PATCH 179/187] Store errors in memory before writing to JSON We don't want to save them to the MapDB file anyway, and simply adding error instances to MapDB files causes (de)serializer configuration to be stored in the file. This makes it more fragile: any change to error class fields can break reopening MapDB files even after all instances of those errors have been cleared out. We can't readily stream the errors out to JSON without accumulating in memory, because we also want to summarize them for storage in Mongo. Conceivably we could build up the summary at the same time as streaming them out to JSON. --- .../analysis/controllers/BundleController.java | 2 +- src/main/java/com/conveyal/gtfs/GTFSFeed.java | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/controllers/BundleController.java b/src/main/java/com/conveyal/analysis/controllers/BundleController.java index 570d0138f..2cb8afd63 100644 --- a/src/main/java/com/conveyal/analysis/controllers/BundleController.java +++ b/src/main/java/com/conveyal/analysis/controllers/BundleController.java @@ -221,7 +221,7 @@ private Bundle create (Request req, Response res) { } catch (IOException e) { throw new RuntimeException(e); } - // Save some space in the MapDB after we've summarized the errors to Mongo and a JSON file. + // Release some memory after we've summarized the errors to Mongo and a JSON file. feed.errors.clear(); // Flush db files to disk diff --git a/src/main/java/com/conveyal/gtfs/GTFSFeed.java b/src/main/java/com/conveyal/gtfs/GTFSFeed.java index 6a742cc24..a642f2e5d 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSFeed.java +++ b/src/main/java/com/conveyal/gtfs/GTFSFeed.java @@ -150,10 +150,11 @@ public class GTFSFeed implements Cloneable, Closeable { /** * A place to accumulate errors while the feed is loaded. Tolerate as many errors as possible and keep on loading. - * TODO store these outside the mapdb for size? If we just don't create this map, old workers should not fail. - * Ideally we'd report the errors to the backend when it first builds the MapDB. + * Note that this set is in memory, not in the MapDB file. We save these into a JSON file to avoid shuttling + * all the serialized errors back and forth to workers. Older workers do have a MapDB table here, but when they + * try to reopen newer MapDB files without that table, even in read-only mode, they'll just receive an empty Set. */ - public final NavigableSet errors; + public final Set errors; // TODO eliminate if not used by Analysis /** Merged stop buffers polygon built lazily by getMergedBuffers() */ @@ -779,7 +780,8 @@ private GTFSFeed (DB db) { patternForTrip = db.getTreeMap("patternForTrip"); - errors = db.getTreeSet("errors"); + // Note that this is an in-memory Java HashSet instead of MapDB table (as it was in past versions). + errors = new HashSet<>(); } // One critical point when constructing the MapDB is the instance cache type and size. From bfe2946e88e877f789660e6605f4070af5ec73c5 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Wed, 10 Nov 2021 09:39:47 +0800 Subject: [PATCH 180/187] factor out setReadOnly helper method --- .../java/com/conveyal/file/LocalFileStorage.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/file/LocalFileStorage.java b/src/main/java/com/conveyal/file/LocalFileStorage.java index 44e8141bb..c1fd039dc 100644 --- a/src/main/java/com/conveyal/file/LocalFileStorage.java +++ b/src/main/java/com/conveyal/file/LocalFileStorage.java @@ -60,10 +60,7 @@ public void moveIntoStorage(FileStorageKey key, File sourceFile) { LOG.info("Could not move {} because of FileSystem restrictions (probably NTFS). Copied instead.", sourceFile.getName()); } - // Set the file to be read-only and accessible only by the current user. - if (!storedFile.setWritable(false, false)) { - LOG.error("Could not restrict permissions on {} to read-only by owner: ", sourceFile.getName()); - } + setReadOnly(storedFile); } catch (IOException e) { throw new RuntimeException(e); } @@ -110,4 +107,14 @@ public boolean exists(FileStorageKey key) { return getFile(key).exists(); } + /** + * Set the file to be read-only and accessible only by the current user. + * There are several ways to set read-only, but this one works on both POSIX and Windows systems. + */ + public static void setReadOnly (File file) { + if (!file.setWritable(false, false)) { + LOG.error("Could not restrict permissions on {} to read-only by owner: ", file.getName()); + } + } + } From eb15c3d3355a304bbafe271ce9b81b7151c5d6b3 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 9 Nov 2021 16:43:53 +0800 Subject: [PATCH 181/187] make GTFSFeed "writable" parameters consistent the outermost functions were acting as expected (writable vs. readonly) but the boolean parameters had reversed meanings on the inner methods. --- src/main/java/com/conveyal/gtfs/GTFSFeed.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/GTFSFeed.java b/src/main/java/com/conveyal/gtfs/GTFSFeed.java index a642f2e5d..49b4840bd 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSFeed.java +++ b/src/main/java/com/conveyal/gtfs/GTFSFeed.java @@ -794,7 +794,7 @@ private GTFSFeed (DB db) { // Initial tests show similar speeds for the default hashtable cache of 64k or 32k size and the hardRef cache. // By not calling any of the cacheEnable or cacheSize methods on the DB builder, we use the default values // that seem to perform well. - private static DB constructMapDb (File dbFile, boolean readOnly) { + private static DB constructMapDb (File dbFile, boolean writable) { DBMaker dbMaker; // TODO also allow for in-memory if (dbFile == null) { @@ -802,10 +802,10 @@ private static DB constructMapDb (File dbFile, boolean readOnly) { } else { dbMaker = DBMaker.newFileDB(dbFile); } - if (readOnly) { - dbMaker.readOnly(); - } else { + if (writable) { dbMaker.asyncWriteEnable(); + } else { + dbMaker.readOnly(); } try{ return dbMaker @@ -826,7 +826,7 @@ private static DB constructMapDb (File dbFile, boolean readOnly) { public static GTFSFeed reopenReadOnly (File file) { if (file.exists()) { - return new GTFSFeed(file, true); + return new GTFSFeed(file, false); } else { throw new GtfsLibException("Cannot reopen file, it does not exist."); } @@ -863,7 +863,7 @@ public static GTFSFeed newWritableFile (File dbFile) { if (dbFile != null && dbFile.exists() && dbFile.length() > 0) { throw new GtfsLibException("Cannot create new file, it already exists."); } - return new GTFSFeed(dbFile, false); + return new GTFSFeed(dbFile, true); } /** @@ -871,7 +871,7 @@ public static GTFSFeed newWritableFile (File dbFile) { * the GTFS file at the supplied filesystem path. This could probably be combined with some other factory methods. */ public static GTFSFeed writableTempFileFromGtfs (String file) { - GTFSFeed feed = new GTFSFeed(null, false); + GTFSFeed feed = new GTFSFeed(null, true); try { ZipFile zip = new ZipFile(file); feed.loadFromFile(zip, null); From 3a22930d40ba4afb4f66eb5341ef8115c10875e5 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 11 Nov 2021 10:16:22 +0800 Subject: [PATCH 182/187] set priority on GtfsErrorTypeSummary from method --- src/main/java/com/conveyal/analysis/models/Bundle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/analysis/models/Bundle.java b/src/main/java/com/conveyal/analysis/models/Bundle.java index 31f2e6794..912f066b4 100644 --- a/src/main/java/com/conveyal/analysis/models/Bundle.java +++ b/src/main/java/com/conveyal/analysis/models/Bundle.java @@ -83,6 +83,7 @@ public static class GtfsErrorTypeSummary { public GtfsErrorTypeSummary () { /* For deserialization. */ } public GtfsErrorTypeSummary (GTFSError error) { this.type = error.errorType; + this.priority = error.getPriority(); } } From 574836add7eba5178cc7c4034589649eff0ccc75 Mon Sep 17 00:00:00 2001 From: Trevor Gerhardt Date: Thu, 11 Nov 2021 10:28:12 +0800 Subject: [PATCH 183/187] Remove unused OverlappingTripsInBlockError --- .../error/OverlappingTripsInBlockError.java | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java diff --git a/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java b/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java deleted file mode 100644 index 459c2527b..000000000 --- a/src/main/java/com/conveyal/gtfs/error/OverlappingTripsInBlockError.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.conveyal.gtfs.error; - -import com.conveyal.gtfs.validator.model.Priority; - -import java.io.Serializable; - -/** - * Created by landon on 5/6/16. - */ -public class OverlappingTripsInBlockError extends GTFSError implements Serializable { - public static final long serialVersionUID = 1L; - - public final String[] tripIds; - public final Priority priority = Priority.HIGH; - public final String routeId; - - public OverlappingTripsInBlockError(long line, String field, String affectedEntityId, String routeId, String[] tripIds) { - super("trips", line, field, affectedEntityId); - this.tripIds = tripIds; - this.routeId = routeId; - } - - @Override public String getMessage() { - return String.format("Trip Ids %s overlap (route: %s) and share block ID %s", String.join(" & ", tripIds), routeId, affectedEntityId); - } - - @Override public Priority getPriority() { - return Priority.MEDIUM; - } -} From 3cf114dc86c2d5c21cc2aaed9b319eaef42e7191 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 12 Nov 2021 13:50:47 +0800 Subject: [PATCH 184/187] query parameter to show system tasks Allows us to observe progress of background tasks not associated with any user, such as the GtfsCacheWarmer. --- src/main/java/com/conveyal/analysis/BackendMain.java | 2 +- .../analysis/controllers/UserActivityController.java | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/BackendMain.java b/src/main/java/com/conveyal/analysis/BackendMain.java index 67fab4159..f543808ae 100644 --- a/src/main/java/com/conveyal/analysis/BackendMain.java +++ b/src/main/java/com/conveyal/analysis/BackendMain.java @@ -58,7 +58,7 @@ private static void startServerInternal (BackendComponents components, TaskActio LOG.info("Conveyal Analysis server is ready."); for (TaskAction taskAction : postStartupTasks) { components.taskScheduler.enqueue( - Task.create(Runnable.class.getSimpleName()).setHeavy(true).forUser("SYSTEM").withAction(taskAction) + Task.create(taskAction.getClass().getSimpleName()).setHeavy(true).forUser("SYSTEM").withAction(taskAction) ); } diff --git a/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java b/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java index e09d450a3..ac7b2ecb4 100644 --- a/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java +++ b/src/main/java/com/conveyal/analysis/controllers/UserActivityController.java @@ -3,10 +3,13 @@ import com.conveyal.analysis.UserPermissions; import com.conveyal.analysis.components.TaskScheduler; import com.conveyal.r5.analyst.progress.ApiTask; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import spark.Request; import spark.Response; import spark.Service; +import java.util.ArrayList; import java.util.List; import static com.conveyal.analysis.util.JsonUtil.toJson; @@ -44,7 +47,9 @@ private ResponseModel getActivity (Request req, Response res) { ResponseModel responseModel = new ResponseModel(); responseModel.systemStatusMessages = List.of(); responseModel.taskBacklog = taskScheduler.getBacklog(); - responseModel.taskProgress = taskScheduler.getTasksForUser(userPermissions.email); + boolean system = Boolean.parseBoolean(req.queryParams("system")); // false if param not present + String user = system ? "SYSTEM" : userPermissions.email; + responseModel.taskProgress = taskScheduler.getTasksForUser(user); return responseModel; } From 612abffac6d6d977336d2e37ad1ed89f5f66137e Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 12 Nov 2021 13:53:58 +0800 Subject: [PATCH 185/187] switch back to setPosixFilePermissions setReadable cannot readily clear non-owner read permissions --- .../com/conveyal/file/LocalFileStorage.java | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/file/LocalFileStorage.java b/src/main/java/com/conveyal/file/LocalFileStorage.java index c1fd039dc..2731d4e71 100644 --- a/src/main/java/com/conveyal/file/LocalFileStorage.java +++ b/src/main/java/com/conveyal/file/LocalFileStorage.java @@ -9,6 +9,8 @@ import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; import java.util.Set; import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; @@ -40,12 +42,13 @@ public LocalFileStorage (Config config) { /** * Move the File into the FileStorage by moving the passed in file to the Path represented by the FileStorageKey. + * It is possible that on some systems (Windows) the file cannot be moved and it will be copied instead, leaving + * the source file in place. */ @Override public void moveIntoStorage(FileStorageKey key, File sourceFile) { - // Get a pointer to the local file + // Get the destination file path inside FileStorage, and ensure all its parent directories exist. File storedFile = getFile(key); - // Ensure the directories exist storedFile.getParentFile().mkdirs(); try { try { @@ -109,11 +112,32 @@ public boolean exists(FileStorageKey key) { /** * Set the file to be read-only and accessible only by the current user. - * There are several ways to set read-only, but this one works on both POSIX and Windows systems. + * All files in our FileStorage are set to read-only as a safeguard against corruption under concurrent access. + * Because the method Files.setPosixFilePermissions fails on Windows with an UnsupportedOperationException, + * we attempted to use the portable File.setReadable and File.setWritable methods to cover both POSIX and Windows + * filesystems, but these require multiple calls in succession to achieve fine grained control on POSIX filesystems. + * Specifically, there is no way to atomically set a file readable by its owner but non-readable by all other users. + * The setReadable/Writable ownerOnly parameter just leaves group and others permissions untouched and unchanged. + * To get the desired result on systems with user-group-other permissions granularity, you have to do something like: + * success &= file.setReadable(false, false); + * success &= file.setWritable(false, false); + * success &= file.setReadable(true, true); + * + * Instead, we first do the POSIX atomic call, which should cover all deployment environments, then fall back on the + * NIO call to cover any development environments using other filesystems. */ public static void setReadOnly (File file) { - if (!file.setWritable(false, false)) { - LOG.error("Could not restrict permissions on {} to read-only by owner: ", file.getName()); + try { + try { + Files.setPosixFilePermissions(file.toPath(), EnumSet.of(PosixFilePermission.OWNER_READ)); + } catch (UnsupportedOperationException e) { + LOG.warn("POSIX permissions unsupported on this filesystem. Falling back on portable NIO methods."); + if (!(file.setReadable(true) && file.setWritable(false))) { + LOG.error("Could not set read-only permissions on file {}", file); + } + } + } catch (Exception e) { + LOG.error("Could not set read-only permissions on file {}", file, e); } } From 1ba313704e121942084532eb63939fe4550d5c57 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 12 Nov 2021 14:55:48 +0800 Subject: [PATCH 186/187] Detect and rebuild problematic MapDB files Also update logback.xml to show date and abbreviate logger packages --- src/main/java/com/conveyal/gtfs/GTFSCache.java | 11 ++++++++++- src/main/resources/logback.xml | 10 ++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/GTFSCache.java b/src/main/java/com/conveyal/gtfs/GTFSCache.java index 0cb2168d2..d78c5138c 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSCache.java +++ b/src/main/java/com/conveyal/gtfs/GTFSCache.java @@ -107,7 +107,16 @@ public FileStorageKey getFileKey (String id, String extension) { // Ensure both MapDB files are local, pulling them down from remote storage as needed. fileStorage.getFile(dbKey); fileStorage.getFile(dbpKey); - return GTFSFeed.reopenReadOnly(fileStorage.getFile(dbKey)); + try { + return GTFSFeed.reopenReadOnly(fileStorage.getFile(dbKey)); + } catch (GtfsLibException e) { + if (e.getCause().getMessage().contains("Could not set field value: priority")) { + // Swallow exception and fall through - rebuild bad MapDB and upload to S3. + LOG.warn("Detected poisoned MapDB containing GTFSError.priority serializer. Rebuilding."); + } else { + throw e; + } + } } FileStorageKey zipKey = getFileKey(bundeScopedFeedId, "zip"); if (!fileStorage.exists(zipKey)) { diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 955c22851..00e0cdabf 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,17 +1,19 @@ + + - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + %date [%thread] %-5level %logger{10} - %msg%n - + + + From 11f6ac77d2dc44ae07c98ce9ae5907d8447b55b9 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 12 Nov 2021 15:49:17 +0800 Subject: [PATCH 187/187] tolerate ancient feedIds containing underscores also fixed misspelled variable name --- .../java/com/conveyal/gtfs/GTFSCache.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/GTFSCache.java b/src/main/java/com/conveyal/gtfs/GTFSCache.java index d78c5138c..5593b77a5 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSCache.java +++ b/src/main/java/com/conveyal/gtfs/GTFSCache.java @@ -100,9 +100,9 @@ public FileStorageKey getFileKey (String id, String extension) { } /** This method should only ever be called by the cache loader. */ - private @Nonnull GTFSFeed retrieveAndProcessFeed(String bundeScopedFeedId) throws GtfsLibException { - FileStorageKey dbKey = getFileKey(bundeScopedFeedId, "db"); - FileStorageKey dbpKey = getFileKey(bundeScopedFeedId, "db.p"); + private @Nonnull GTFSFeed retrieveAndProcessFeed(String bundleScopedFeedId) throws GtfsLibException { + FileStorageKey dbKey = getFileKey(bundleScopedFeedId, "db"); + FileStorageKey dbpKey = getFileKey(bundleScopedFeedId, "db.p"); if (fileStorage.exists(dbKey) && fileStorage.exists(dbpKey)) { // Ensure both MapDB files are local, pulling them down from remote storage as needed. fileStorage.getFile(dbKey); @@ -118,7 +118,7 @@ public FileStorageKey getFileKey (String id, String extension) { } } } - FileStorageKey zipKey = getFileKey(bundeScopedFeedId, "zip"); + FileStorageKey zipKey = getFileKey(bundleScopedFeedId, "zip"); if (!fileStorage.exists(zipKey)) { throw new GtfsLibException("Original GTFS zip file could not be found: " + zipKey); } @@ -128,12 +128,19 @@ public FileStorageKey getFileKey (String id, String extension) { try { File tempDbFile = FileUtils.createScratchFile("db"); File tempDbpFile = new File(tempDbFile.getAbsolutePath() + ".p"); - // An unpleasant hack since we do not have separate references to the GTFS ID and Bundle ID here, + // An unpleasant hack since we do not have separate references to the GTFS feed ID and Bundle ID here, // only a concatenation of the two joined with an underscore. We have to force-override feed ID because // references to its contents (e.g. in scenarios) are scoped only by the feed ID not the bundle ID. - final String[] feedAndBundleId = bundeScopedFeedId.split("_"); - checkState(feedAndBundleId.length == 2, "Expected underscore-joined feedId and bundleId."); - GTFSFeed.newFileFromGtfs(tempDbFile, fileStorage.getFile(zipKey), feedAndBundleId[0]); + // The bundle ID is expected to always be an underscore-free UUID, but old feed IDs may contain underscores + // (yielding filenames like old_feed_id_bundleuuid) so we look for the last underscore as a split point. + // GTFS feeds may now be referenced by multiple bundles with different IDs, so the last part of the file + // name is rather arbitrary - it's just the bundleId with which this feed was first associated. + // We don't really need to scope newer feeds by bundleId since they now have globally unique UUIDs. + int splitIndex = bundleScopedFeedId.lastIndexOf("_"); + checkState(splitIndex > 0 && splitIndex < bundleScopedFeedId.length() - 1, + "Expected underscore-joined feedId and bundleId."); + String feedId = bundleScopedFeedId.substring(0, splitIndex); + GTFSFeed.newFileFromGtfs(tempDbFile, fileStorage.getFile(zipKey), feedId); // The DB file should already be closed and flushed to disk. // Put the DB and DB.p files in local cache, and mirror to remote storage if configured. fileStorage.moveIntoStorage(dbKey, tempDbFile);