Merge branch 'main' into addressable-index

api/config.ts

@@ -13,6 +13,7 @@ const environment = environmentSchema.parse(process.env);
 // so be careful not to expose any secrets here.
 const config = {
   pipelineDate: '2024-12-10',
+  addressablesIndex: 'addressables',
   articlesIndex: 'articles',
   eventsIndex: 'events',
   venuesIndex: 'venues',

api/package.json

@@ -6,7 +6,8 @@
   "scripts": {
     "start": "NODE_ENV=production tsx ./server.ts",
     "dev": "AWS_PROFILE=catalogue-developer NODE_ENV=development nodemon - exec 'tsx' ./server.ts",
-    "test": "jest"
+    "test": "jest",
+    "check_holiday_closures": "AWS_PROFILE=catalogue-developer tsx ./scripts/holiday_closure_test.ts"
   },
   "dependencies": {
     "@elastic/elasticsearch": "^8.6.0",

api/scripts/holiday_closure_test.ts

@@ -0,0 +1,112 @@
+/* 
+Before the end-of-year closure, we test that the date picker dropdown in the Request item dialog is going to display the correct options
+to ensure that users can only requests items when they can be available, on a day the library is open
+Once the Modified opening times have been added to Prismic and published,
+set dateNow as the date to be tested and run this script with
+yarn check_holiday_closures
+*/
+
+import { DateTime } from 'luxon';
+
+import { getNextOpeningDates } from '@weco/content-api/src/controllers/utils';
+import { getElasticClient } from '@weco/content-common/services/elasticsearch';
+import {
+  ElasticsearchVenue,
+  NextOpeningDate,
+  Venue,
+} from '@weco/content-common/types/venue';
+
+// set this to the date to be tested, here and in utils.ts
+// eg. if you want to see what the dates on the date picker will be on December 14, 2024 at 8:30am
+// const dateNow = new Date("2024-12-14T08:30:00.000Z");
+const dateNow = new Date();
+
+const formatDate = (openingDay: NextOpeningDate) =>
+  openingDay.open
+    ? new Date(openingDay.open).toUTCString().slice(0, 11) + '\n'
+    : '';
+
+const compareDates = (library: NextOpeningDate, deepstore: NextOpeningDate) => {
+  const libraryDate =
+    library.open && DateTime.fromJSDate(new Date(library.open)).startOf('day');
+  const deepstoreDate =
+    deepstore.open &&
+    DateTime.fromJSDate(new Date(deepstore.open)).startOf('day');
+  if (libraryDate === undefined || deepstoreDate === undefined) {
+    throw new Error('One of these dates is undefined!');
+  } else {
+    return libraryDate > deepstoreDate;
+  }
+};
+
+// the 2 functions below reproduce the logic used in the catalogue-api/items to generate available dates
+
+const applyItemsApiLibraryLogic = (openingTimes: NextOpeningDate[]) => {
+  // the library is open today if the 1st date in NextOpeningDate[] is the same as today
+  const isLibraryOpenToday =
+    new Date(dateNow).getDate() ===
+    (openingTimes[0].open && new Date(openingTimes[0].open).getDate());
+  if (dateNow.getHours() < 10 || !isLibraryOpenToday) {
+    return openingTimes.slice(1, -1).map(formatDate);
+  } else {
+    return openingTimes.slice(2, -1).map(formatDate);
+  }
+};
+
+const applyItemsApiDeepstoreLogic = (
+  libraryOpeningTimes: NextOpeningDate[],
+  deepstoreOpeningTimes: NextOpeningDate[]
+) => {
+  const firstDeepstoreAvailability = deepstoreOpeningTimes.slice(10, -1)[0];
+  const subsequentLibraryAvailabilities = libraryOpeningTimes.filter(
+    libraryOpeningTime => {
+      return compareDates(libraryOpeningTime, firstDeepstoreAvailability);
+    }
+  );
+  return subsequentLibraryAvailabilities.map(formatDate);
+};
+
+const run = async () => {
+  const elasticClient = await getElasticClient({
+    serviceName: 'api',
+    pipelineDate: '2023-03-24',
+    hostEndpointAccess: 'public',
+  });
+
+  const searchResponse = await elasticClient.search<ElasticsearchVenue>({
+    index: 'venues',
+    _source: ['display'],
+    query: {
+      bool: {
+        filter: [{ terms: { 'filter.title': ['library', 'deepstore'] } }],
+      },
+    },
+  });
+  const venuesData = searchResponse.hits.hits.flatMap(hit =>
+    hit._source ? [hit._source.display] : []
+  );
+
+  const {
+    regularOpeningDays: libraryRegularOpeningDays,
+    exceptionalClosedDays: libraryHolidayclosures,
+  } = venuesData.find(venue => venue.title === 'Library') as Venue;
+  const {
+    regularOpeningDays: deepstoreRegularOpeningDays,
+    exceptionalClosedDays: deepstoreHolidayclosures,
+  } = venuesData.find(venue => venue.title === 'Deepstore') as Venue;
+
+  const onsiteItemsPickup = applyItemsApiLibraryLogic(
+    getNextOpeningDates(libraryRegularOpeningDays, libraryHolidayclosures)
+  ).slice(0, 12);
+  const offsiteItemsPickup = applyItemsApiDeepstoreLogic(
+    getNextOpeningDates(libraryRegularOpeningDays, libraryHolidayclosures),
+    getNextOpeningDates(deepstoreRegularOpeningDays, deepstoreHolidayclosures)
+  ).slice(0, 12);
+
+  const library = `Onsite items requested on ${dateNow} will be available on: \n${onsiteItemsPickup.join('\r')}`;
+  const deepstore = `Deepstore items requested on ${dateNow} will be available on: \n${offsiteItemsPickup.join('\r')}`;
+  console.log(library);
+  console.log(deepstore);
+};
+
+run();

api/src/app.ts

@@ -5,6 +5,7 @@ import { Config } from '@weco/content-api/config';
 import { logStream } from '@weco/content-common/services/logging';
 
 import {
+  addressablesController,
   articleController,
   articlesController,
   errorHandler,
@@ -20,6 +21,7 @@ const createApp = (clients: Clients, config: Config) => {
 
   app.use(morgan('short', { stream: logStream('http') }));
 
+  app.get('/all', addressablesController(clients, config));
   app.get('/articles', articlesController(clients, config));
   app.get('/articles/:id', articleController(clients, config));
   app.get('/events', eventsController(clients, config));

api/src/controllers/addressables.ts

@@ -0,0 +1,72 @@
+import { errors as elasticErrors } from '@elastic/elasticsearch';
+import { RequestHandler } from 'express';
+import asyncHandler from 'express-async-handler';
+
+import { Config } from '@weco/content-api/config';
+import { ifDefined } from '@weco/content-api/src/helpers';
+import { resultListResponse } from '@weco/content-api/src/helpers/responses';
+import { addressablesQuery } from '@weco/content-api/src/queries/addressables';
+import { Clients, Displayable } from '@weco/content-api/src/types';
+import { ResultList } from '@weco/content-api/src/types/responses';
+
+import { HttpError } from './error';
+import { paginationElasticBody, PaginationQueryParameters } from './pagination';
+
+type QueryParams = {
+  query?: string;
+} & PaginationQueryParameters;
+
+type AddressablesHandler = RequestHandler<
+  never,
+  ResultList,
+  never,
+  QueryParams
+>;
+
+const addressablesController = (
+  clients: Clients,
+  config: Config
+): AddressablesHandler => {
+  const index = config.addressablesIndex;
+  const resultList = resultListResponse(config);
+
+  return asyncHandler(async (req, res) => {
+    const { query: queryString } = req.query;
+
+    try {
+      const searchResponse = await clients.elastic.search<Displayable>({
+        index,
+        _source: ['display'],
+        query: {
+          bool: {
+            must: ifDefined(queryString, addressablesQuery),
+          },
+        },
+
+        ...paginationElasticBody(req.query),
+      });
+
+      res.status(200).json(resultList(req, searchResponse));
+    } catch (e) {
+      if (
+        e instanceof elasticErrors.ResponseError &&
+        // This is an error we see from very long (spam) queries which contain
+        // many many terms and so overwhelm the multi_match query. The check
+        // for length is a heuristic so that if we get legitimate `too_many_nested_clauses`
+        // errors, we're still alerted to them
+        e.message.includes('too_many_nested_clauses') &&
+        encodeURIComponent(queryString || '').length > 2000
+      ) {
+        throw new HttpError({
+          status: 400,
+          label: 'Bad Request',
+          description:
+            'Your query contained too many terms, please try again with a simpler query',
+        });
+      }
+      throw e;
+    }
+  });
+};
+
+export default addressablesController;

api/src/controllers/index.ts

@@ -1,3 +1,4 @@
+import addressablesController from './addressables';
 import articleController from './article';
 import articlesController from './articles';
 import eventController from './event';
@@ -7,6 +8,7 @@ import venuesController from './venue';
 
 export { errorHandler } from './error';
 export {
+  addressablesController,
   articlesController,
   articleController,
   eventController,

api/test/addressables.test.ts

@@ -0,0 +1,17 @@
+import { mockedApi } from './fixtures/api';
+
+describe('GET /all', () => {
+  it('returns a list of documents', async () => {
+    const docs = Array.from({ length: 10 }).map((_, i) => ({
+      id: `id-${i}`,
+      display: {
+        title: `test doc ${i}`,
+      },
+    }));
+    const api = mockedApi(docs);
+
+    const response = await api.get(`/all`);
+    expect(response.statusCode).toBe(200);
+    expect(response.body.results).toStrictEqual(docs.map(d => d.display));
+  });
+});

api/test/fixtures/api.ts

@@ -21,20 +21,31 @@ const elastic404 = () =>
     warnings: [],
   });
 
+const testAddressablesIndex = 'test-addressables';
+const testArticlesIndex = 'test-articles';
+const testEventsIndex = 'test-events';
+const testVenuesIndex = 'test-venue';
+
+export const mockConfig = {
+  pipelineDate: '2222-22-22',
+  addressablesIndex: testAddressablesIndex,
+  articlesIndex: testArticlesIndex,
+  eventsIndex: testEventsIndex,
+  venuesIndex: testVenuesIndex,
+  publicRootUrl: new URL('http://test.test/test'),
+};
+
 export const mockedApi = <T extends Displayable & Identified>(
   documents: T[]
 ) => {
-  const testArticlesIndex = 'test-article-index';
-  const testEventsIndex = 'test-event-index';
-  const testVenuesIndex = 'test-venue-index';
-
   const documentsMap = new Map(documents.map(d => [d.id, d]));
 
   const elasticClientGet = jest.fn(
     ({ id, index }: Parameters<ElasticClient['get']>[0]) => {
       if (
         documentsMap.has(id) &&
-        (index === testArticlesIndex ||
+        (index === testAddressablesIndex ||
+          index === testArticlesIndex ||
           index === testEventsIndex ||
           index === testVenuesIndex)
       ) {
@@ -50,6 +61,7 @@ export const mockedApi = <T extends Displayable & Identified>(
   const elasticClientSearch = jest.fn(
     (params: Parameters<ElasticClient['search']>[0]) => {
       if (
+        params?.index === testAddressablesIndex ||
         params?.index === testArticlesIndex ||
         params?.index === testEventsIndex ||
         params?.index === testVenuesIndex
@@ -75,13 +87,7 @@ export const mockedApi = <T extends Displayable & Identified>(
         search: elasticClientSearch,
       } as unknown as ElasticClient,
     },
-    {
-      pipelineDate: '2222-22-22',
-      articlesIndex: testArticlesIndex,
-      eventsIndex: testEventsIndex,
-      venuesIndex: testVenuesIndex,
-      publicRootUrl: new URL('http://test.test/test'),
-    }
+    mockConfig
   );
 
   return supertest.agent(app);

api/test/query.test.ts

@@ -1,23 +1,19 @@
 import { Client as ElasticClient } from '@elastic/elasticsearch';
 import { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
 import supertest from 'supertest';
-import { URL, URLSearchParams } from 'url';
+import { URLSearchParams } from 'url';
 
 import createApp from '@weco/content-api/src/app';
 
+import { mockConfig } from './fixtures/api';
+
 const elasticsearchRequestForURL = async (
   url: string
 ): Promise<SearchRequest> => {
   const searchSpy = jest.fn();
   const app = createApp(
     { elastic: { search: searchSpy } as unknown as ElasticClient },
-    {
-      pipelineDate: '2222-22-22',
-      articlesIndex: 'test-articles',
-      eventsIndex: 'test-events',
-      venuesIndex: 'test-venues',
-      publicRootUrl: new URL('http://test.test/test'),
-    }
+    mockConfig
   );
   const api = supertest.agent(app);
   await api.get(url);

pipeline/src/extractTransformLoad.ts

@@ -21,6 +21,7 @@ import { Clients } from './types';
 type ETLParameters<PrismicDocument, ElasticsearchDocument> = {
   indexConfig: IndexConfig;
   graphQuery: string;
+  filters?: string[];
   parentDocumentTypes: Set<string>;
   transformer: (prismicDoc: PrismicDocument) => ElasticsearchDocument[];
 };
@@ -59,6 +60,7 @@ export const createETLPipeline =
         getPrismicDocuments(clients.prismic, {
           publicationWindow: toBoundedWindow(event),
           graphQuery: etlParameters.graphQuery,
+          filters: etlParameters.filters,
           after,
         })
       ).pipe(tap(document => seenIds.add(document.id))),

pipeline/src/handler.ts

@@ -1,3 +1,4 @@
+import * as prismic from '@prismicio/client';
 import { Handler } from 'aws-lambda';
 
 import { WindowEvent } from './event';
@@ -19,6 +20,7 @@ import { Clients } from './types';
 
 const loadAddressables = createETLPipeline({
   graphQuery: addressablesQuery,
+  filters: [prismic.filter.not('document.tags', ['delist'])],
   indexConfig: addressables,
   parentDocumentTypes: new Set([
     'articles',

pipeline/src/helpers/prismic.ts

@@ -19,6 +19,7 @@ const PRISMIC_MAX_PAGE_SIZE = 100;
 type GetPrismicDocumentsParams = {
   publicationWindow: TimeWindow;
   graphQuery: string;
+  filters?: string[];
   after?: string;
 };
 
@@ -34,7 +35,7 @@ const fields = {
 
 export const getPrismicDocuments = async (
   client: prismic.Client,
-  { publicationWindow, graphQuery, after }: GetPrismicDocumentsParams
+  { publicationWindow, graphQuery, after, filters }: GetPrismicDocumentsParams
 ): Promise<PrismicPage<prismic.PrismicDocument>> => {
   const startDate = publicationWindow.start;
   const endDate = publicationWindow.end;
@@ -48,7 +49,9 @@ export const getPrismicDocuments = async (
       endDate
         ? prismic.filter.dateBefore(fields.lastPublicationDate, endDate)
         : [],
-    ].flat(),
+    ]
+      .concat(filters ?? [])
+      .flat(),
     orderings: {
       field: fields.lastPublicationDate,
       direction: 'desc',