Merge branch 'dev' into dev-flex-2024

# Conflicts:
#	gtfs.yml
#	i18n/english.yml
#	i18n/polish.yml
#	lib/common/actions/index.js
#	lib/editor/actions/map/stopStrategies.js
#	lib/editor/actions/tripPattern.js
#	lib/editor/components/EditorInput.js
#	lib/editor/components/HourMinuteInput.js
#	lib/editor/components/pattern/NormalizeStopTimesModal.js
#	lib/editor/components/pattern/PatternStopCard.js
#	lib/editor/components/pattern/PatternStopContainer.js
#	lib/editor/components/pattern/PatternStopsPanel.js
#	lib/editor/components/timetable/TimetableEditor.js
#	lib/editor/components/timetable/TimetableGrid.js
#	lib/editor/components/timetable/TimetableHeader.js
#	lib/editor/components/timetable/TripSeriesModal.js
#	lib/editor/util/map.js
#	lib/editor/util/validation/validation.js
#	lib/gtfs/util/index.js
#	lib/manager/components/transform/FeedTransformation.js
#	lib/types/index.js

Author: miles-grant-ibigroup

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # See https://help.github.com/articles/about-codeowners/
 
-# An IBI Group OTP/Data Tools member is required to approve PR merges
+# An Arcadis OTP/TRANSIT-data-tools member is required to approve PR merges
 * @ibi-group/otp-data-tools

.github/issue_template.md

@@ -1,4 +1,4 @@
-_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be dealt with in a timely manner. [IBI Group](https://www.ibigroup.com/) is able to provide technical support for custom deployments of this software. Please contact [Ritesh Warade](mailto:ritesh.warade@ibigroup.com?subject=Data%20Tools%20inquiry%20via%20GitHub&body=Name:%20%0D%0AAgency/Company:%20%0D%0ABest%20date/time%20for%20a%20demo/discussion:%20%0D%0ADescription%20of%20needs:%20) if your company or organization is interested in opening a support contract with us. Please remove this note when creating the issue._
+_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be dealt with in a timely manner. [Arcadis IBI Group](https://www.ibigroup.com/) is able to provide technical support for custom deployments of this software. Please contact [Jon Campbell](mailto:jon.campbell@ibigroup.com?subject=Data%20Tools%20inquiry%20via%20GitHub&body=Name:%20%0D%0AAgency/Company:%20%0D%0ABest%20date/time%20for%20a%20demo/discussion:%20%0D%0ADescription%20of%20needs:%20) if your company or organization is interested in opening a support contract with us. Please remove this note when creating the issue._
 
 ## Observed behavior (please include a screenshot if possible)
 

.gitignore

@@ -8,6 +8,9 @@ tmp/
 .tags
 *.pid
 
+# Ignore mobility data jar for building rules
+*.jar
+
 # Optional npm cache directory
 .npm
 
@@ -27,6 +30,7 @@ env.yml
 env.yml-original
 .env
 !configurations/test/env.yml
+!docker/server/env.yml
 scripts/*client.json
 *.pem
 

.readthedocs.yml

@@ -0,0 +1,14 @@
+# Read the Docs configuration file for MkDocs projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+mkdocs:
+  configuration: mkdocs.yml

README.md

@@ -1,6 +1,22 @@
 # datatools-ui
 
-The core application for IBI Group's transit Data Tools suite. This application provides GTFS editing, management, validation, and deployment to OpenTripPlanner.
+[![Join the chat at https://matrix.to/#/#transit-data-tools:gitter.im](https://badges.gitter.im/repo.png)](https://matrix.to/#/#transit-data-tools:gitter.im)
+
+The core application for IBI Group's TRANSIT-Data-Tools suite. This application provides GTFS editing, management, validation, and deployment to OpenTripPlanner.
+
+## Quick Start
+
+A pre-configured datatools instance can be lauched via Docker by running
+
+```bash
+cd docker
+cp ../configurations/default/env.yml.tmp ../configurations/default/env.yml
+docker-compose up
+```
+
+from the datatools-ui directory. Datatools will then be running on port `9966`.
+
+Deployment functionality will not work, and persistence may only work in certain cases (look into Docker volumes for more info).
 
 ## Configuration
 
@@ -12,6 +28,10 @@ View the [latest release documentation](http://data-tools-docs.ibi-transit.com/e
 
 Note: `dev` branch docs (which refer to the default `branch` and are more up-to-date and accurate for most users) can be found [here](http://data-tools-docs.ibi-transit.com/en/dev/).
 
+## Getting in touch
+
+We have a Gitter [space](https://matrix.to/#/#transit-data-tools:gitter.im) for the full TRANSIT-Data-Tools project where you can post questions and comments.
+
 ## Shoutouts 🙏
 
 <img src="browserstack-logo-600x315.png" height="80" title="BrowserStack Logo" alt="BrowserStack Logo" />

__tests__/e2e/server/Dockerfile

@@ -1,5 +1,6 @@
 # syntax=docker/dockerfile:1
-FROM maven:3.8.6-openjdk-8
+FROM maven:3.8.7-openjdk-18
+
 WORKDIR /datatools
 
 ARG E2E_AUTH0_USERNAME
@@ -32,6 +33,7 @@ ARG AWS_SECRET_ACCESS_KEY
 
 # Grab latest dev build of Datatools Server
 RUN git clone https://github.com/ibi-group/datatools-server.git
+RUN microdnf install wget
 WORKDIR /datatools/datatools-server
 # TODO: Remove once dev-flex is merged into dev!
 RUN git checkout dev-flex

__tests__/end-to-end.js

@@ -2779,6 +2779,10 @@ describe('end-to-end', () => {
 
       // wait for snapshots tab to load and publish snapshot
       await waitForAndClick('[data-test-id="publish-snapshot-button"]')
+
+      // wait for snapshot export modal and click "no" to proprietary file export
+      await waitForAndClick('[data-test-id="export-patterns-modal-no"]')
+
       // wait for version to get created
       await waitAndClearCompletedJobs()
 

__tests__/test-utils/mock-data/manager.js

@@ -71,6 +71,7 @@ export function makeMockDeployment (
     peliasCsvFiles: [],
     peliasResetDb: null,
     peliasUpdate: null,
+    peliasSynonymsBase64: null,
     pinnedfeedVersionIds: [],
     projectBounds: {east: 0, west: 0, north: 0, south: 0},
     projectId: project.id,
@@ -109,13 +110,14 @@ export const mockProject = {
   pinnedDeploymentId: null,
   peliasWebhookUrl: null,
   routerConfig: {
-    carDropoffTime: null,
-    numItineraries: null,
+    driveDistanceReluctance: null,
+    itineraryFilters: {nonTransitGeneralizedCostLimit: null},
     requestLogFile: null,
     stairsReluctance: null,
     updaters: null,
     walkSpeed: null
   },
+  sharedStopsConfig: null,
   useCustomOsmBounds: false,
   user: null
 }
@@ -128,7 +130,6 @@ export const mockFeedWithVersion = {
   externalProperties: {},
   id: 'mock-feed-with-version-id',
   isPublic: false,
-  lastFetched: 1543389038810,
   lastUpdated: 1543389038810,
   latestValidation: {
     agencies: null,
@@ -167,6 +168,38 @@ export const mockFeedWithVersion = {
   versionCount: 1
 }
 
+// a mock feed source summary
+export const mockFeedSourceSummaryWithVersion = {
+  deployable: false,
+  id: 'mock-feed-with-version-id',
+  isPublic: false,
+  lastUpdated: 1543389038810,
+  latestValidation: {
+    agencies: null,
+    agencyCount: 1,
+    avgDailyRevenueTime: 0,
+    bounds: {
+      north: 39.0486949672717,
+      south: 38.92884,
+      east: -76.481211,
+      west: -76.5673055566884
+    },
+    endDate: '20190801',
+    errorCount: 78,
+    feedVersionId: 'mock-feed-version-id',
+    loadFailureReason: null,
+    loadStatus: 'SUCCESS',
+    routeCount: 10,
+    startDate: '20180801',
+    stopCount: 237,
+    stopTimesCount: 11170,
+    tripCount: 415
+  },
+  labelIds: [],
+  name: 'test feed with a version',
+  projectId: mockProject.id
+}
+
 // a mock feed with no versions
 export const mockFeedWithoutVersion = {
   dateCreated: 1544831411569,
@@ -175,7 +208,6 @@ export const mockFeedWithoutVersion = {
   externalProperties: {},
   id: 'mock-feed-without-version-id',
   isPublic: false,
-  lastFetched: null,
   name: 'test feed with no version',
   labelIds: [],
   noteCount: 0,
@@ -340,6 +372,7 @@ export const mockFeedVersion = {
     feedVersionId: 'mock-feed-version-id',
     loadFailureReason: null,
     loadStatus: 'SUCCESS',
+    mobilityDataResult: {},
     routeCount: 10,
     startDate: '20180801',
     stopCount: 237,

configurations/default/env.yml.tmp

@@ -2,6 +2,7 @@ AUTH0_CLIENT_ID: your-auth0-client-id
 AUTH0_CONNECTION_NAME: your-auth0-connection-name
 AUTH0_DOMAIN: your-auth0-domain
 BUGSNAG_KEY: optional-bugsnag-key
+MAP_BASE_URL: optional-map-tile-url
 MAPBOX_ACCESS_TOKEN: your-mapbox-access-token
 MAPBOX_MAP_ID: mapbox/outdoors-v11
 MAPBOX_ATTRIBUTION: <a href="https://www.mapbox.com/about/maps/" target="_blank">&copy; Mapbox &copy; OpenStreetMap</a> <a href="https://www.mapbox.com/map-feedback/" target="_blank">Improve this map</a>
@@ -22,3 +23,4 @@ GRAPH_HOPPER_KEY: your-graph-hopper-key
 #      - 83
 GOOGLE_ANALYTICS_TRACKING_ID: optional-ga-key
 # GRAPH_HOPPER_POINT_LIMIT: 10 # Defaults to 30
+DISABLE_AUTH: true

docker/docker-compose.yml

@@ -0,0 +1,71 @@
+version: "3.8"
+
+x-common-variables: &common-variables
+  - BUGSNAG_KEY=${BUGSNAG_KEY}
+  - S3_BUCKET=${S3_BUCKET}
+  - LOGS_S3_BUCKET=${LOGS_S3_BUCKET}
+  - MS_TEAMS_WEBHOOK_URL=${MS_TEAMS_WEBHOOK_URL}
+  - MAPBOX_ACCESS_TOKEN=${MAPBOX_ACCESS_TOKEN}
+  - GITHUB_SHA=${GITHUB_SHA}
+  - GITHUB_REF_SLUG=${GITHUB_REF_SLUG}
+  - TRANSITFEEDS_KEY=${TRANSITFEEDS_KEY}
+  - GITHUB_REPOSITORY=${GITHUB_REPOSITORY}
+  - GITHUB_WORKSPACE=${GITHUB_WORKSPACE}
+  - GITHUB_RUN_ID=${GITHUB_RUN_ID}
+  - AUTH0_CLIENT_ID=${AUTH0_CLIENT_ID}
+  - AUTH0_PUBLIC_KEY=${AUTH0_PUBLIC_KEY}
+  - AUTH0_CONNECTION_NAME=${AUTH0_CONNECTION_NAME}
+  - AUTH0_DOMAIN=${AUTH0_DOMAIN}
+  - AUTH0_API_CLIENT=${AUTH0_API_CLIENT}
+  - AUTH0_API_SECRET=${AUTH0_API_SECRET}
+  - OSM_VEX=${OSM_VEX}
+  - SPARKPOST_KEY=${SPARKPOST_KEY}
+  - SPARKPOST_EMAIL=${SPARKPOST_EMAIL}
+  - GTFS_DATABASE_URL=jdbc:postgresql://postgres/dmtest
+  - GTFS_DATABASE_USER=root
+  - GTFS_DATABASE_PASSWORD=e2e
+  - MONGO_DB_NAME=data_manager
+  - MONGO_HOST=mongo:27017
+  - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+  - AWS_REGION=${AWS_REGION}
+  - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+  - DISABLE_AUTH=true
+
+services:
+  datatools-server:
+    image: ghcr.io/ibi-group/datatools-server:dev
+    restart: always
+    environment: *common-variables
+    volumes:
+      - type: bind
+        source: ./server/
+        target: /config
+    ports:
+      - "4000:4000"
+  datatools-ui:
+    build:
+      context: ../
+      dockerfile: ./docker/ui/Dockerfile
+      args: *common-variables
+    restart: always
+    environment: *common-variables
+    ports:
+      - "9966:9966"
+  mongo:
+    image: mongo
+    restart: always
+    volumes:
+      - dt-mongo:/data/db
+  postgres:
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_USER: root
+      POSTGRES_PASS: e2e
+      POSTGRES_DB: dmtest
+    image: postgres
+    restart: always
+    volumes:
+      - dt-postgres:/var/lib/postgresql/data
+volumes:
+  dt-postgres:
+  dt-mongo:

docker/server/env.yml

@@ -0,0 +1,5 @@
+DISABLE_AUTH: TRUE
+GTFS_DATABASE_URL: jdbc:postgresql://postgres/dmtest
+MONGO_DB_NAME: data_manager
+MONGO_HOST: mongo
+AUTH0_CLIENT_ID: disable_auth

docker/server/server.yml

@@ -0,0 +1,67 @@
+application:
+  title: Data Tools
+  logo: https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png
+  logo_large: https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png
+  client_assets_url: https://example.com
+  shortcut_icon_url: https://d2tyb7byn1fef9.cloudfront.net/ibi-logo-original%402x.png
+  public_url: http://localhost:9966
+  notifications_enabled: false
+  docs_url: http://conveyal-data-tools.readthedocs.org
+  support_email: [email protected]
+  port: 4000
+  data:
+    gtfs: /tmp
+    use_s3_storage: false
+    s3_region: us-east-1
+    gtfs_s3_bucket: bucket-name
+modules:
+  enterprise:
+    enabled: false
+  editor:
+    enabled: true
+  deployment:
+    enabled: true
+    ec2:
+      enabled: false
+      default_ami: ami-your-ami-id
+      tag_key: a-tag-key-to-add-to-all-instances
+      tag_value: a-tag-value-to-add-to-all-instances
+    # Note: using a cloudfront URL for these download URLs will greatly
+    # increase download/deploy speed.
+    otp_download_url: https://optional-otp-repo.com
+  user_admin:
+    enabled: false
+  gtfsapi:
+    enabled: true
+    load_on_fetch: false
+    # use_extension: mtc
+    # update_frequency: 30 # in seconds
+  manager:
+    normalizeFieldTransformation:
+      # Enter capitalization exceptions (e.g. acronyms), in the desired case, and separated by commas.
+      defaultCapitalizationExceptions:
+        - ACE
+        - BART
+      # Enter substitutions (e.g. substitute '@' with 'at'), one dashed entry for each substitution, with:
+      # - pattern: the regex string pattern that will be replaced,
+      # - replacement: the replacement string for that pattern,
+      # - normalizeSpace: if true, the resulting field value will include one space before and after the replacement string.
+      #       Note: if the replacement must be blank, then normalizeSpace should be set to false
+      #       and whitespace management should be handled in pattern instead.
+      # Substitutions are executed in order they appear in the list.
+      defaultSubstitutions:
+        - description: "Replace '@' with 'at', and normalize space."
+          pattern: "@"
+          replacement: at
+          normalizeSpace: true
+        - description: "Replace '+' (\\+ in regex) and '&' with 'and', and normalize space."
+          pattern: "[\\+&]"
+          replacement: and
+          normalizeSpace: true
+extensions:
+  transitland:
+    enabled: true
+    api: https://transit.land/api/v1/feeds
+  transitfeeds:
+    enabled: true
+    api: http://api.transitfeeds.com/v1/getFeeds

docker/ui/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:14
+WORKDIR /datatools-build
+
+ARG BUGSNAG_KEY
+
+RUN cd /datatools-build
+COPY package.json yarn.lock patches /datatools-build/
+RUN yarn
+COPY . /datatools-build/ 
+COPY configurations/default /datatools-config/
+
+
+# Copy the tmp file to the env.yml if no env.yml is present
+RUN cp -R -u -p /datatools-config/env.yml.tmp /datatools-config/env.yml
+
+CMD yarn run mastarm build --env dev --serve --proxy http://datatools-server:4000/api # 

docs/dev/api_interaction.md

@@ -1,4 +1,4 @@
-# API Interaction Transcript
+# API Interaction
 The following is a set of instructions on API calls needed to upload and validate
 a feed, wait for the tasks' completion, and then browse its contents. All of the
 endpoints needed to load and process a GTFS file are REST-based. The endpoints