Skip to content

Commit

Permalink
Merge branch 'main' into addressables-unpublisher
Browse files Browse the repository at this point in the history
  • Loading branch information
davidpmccormick authored Dec 12, 2024
2 parents b9c0c2e + 869f5f2 commit 013e50e
Show file tree
Hide file tree
Showing 42 changed files with 3,540 additions and 3,099 deletions.
2 changes: 1 addition & 1 deletion .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
env:
LIVE_PIPELINE: '2023-03-24'
LIVE_PIPELINE: '2024-12-10'
steps:
- label: 'autoformat'
command: '.buildkite/scripts/autoformat.sh'
Expand Down
3 changes: 2 additions & 1 deletion api/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ const environment = environmentSchema.parse(process.env);
// This configuration is exposed via the public healthcheck endpoint,
// so be careful not to expose any secrets here.
const config = {
pipelineDate: '2023-03-24',
pipelineDate: '2024-12-10',
addressablesIndex: 'addressables',
articlesIndex: 'articles',
eventsIndex: 'events',
venuesIndex: 'venues',
Expand Down
3 changes: 2 additions & 1 deletion api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"scripts": {
"start": "NODE_ENV=production tsx ./server.ts",
"dev": "AWS_PROFILE=catalogue-developer NODE_ENV=development nodemon - exec 'tsx' ./server.ts",
"test": "jest"
"test": "jest",
"check_holiday_closures": "AWS_PROFILE=catalogue-developer tsx ./scripts/holiday_closure_test.ts"
},
"dependencies": {
"@elastic/elasticsearch": "^8.6.0",
Expand Down
112 changes: 112 additions & 0 deletions api/scripts/holiday_closure_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
Before the end-of-year closure, we test that the date picker dropdown in the Request item dialog is going to display the correct options
to ensure that users can only requests items when they can be available, on a day the library is open
Once the Modified opening times have been added to Prismic and published,
set dateNow as the date to be tested and run this script with
yarn check_holiday_closures
*/

import { DateTime } from 'luxon';

import { getNextOpeningDates } from '@weco/content-api/src/controllers/utils';
import { getElasticClient } from '@weco/content-common/services/elasticsearch';
import {
ElasticsearchVenue,
NextOpeningDate,
Venue,
} from '@weco/content-common/types/venue';

// set this to the date to be tested, here and in utils.ts
// eg. if you want to see what the dates on the date picker will be on December 14, 2024 at 8:30am
// const dateNow = new Date("2024-12-14T08:30:00.000Z");
const dateNow = new Date();

const formatDate = (openingDay: NextOpeningDate) =>
openingDay.open
? new Date(openingDay.open).toUTCString().slice(0, 11) + '\n'
: '';

const compareDates = (library: NextOpeningDate, deepstore: NextOpeningDate) => {
const libraryDate =
library.open && DateTime.fromJSDate(new Date(library.open)).startOf('day');
const deepstoreDate =
deepstore.open &&
DateTime.fromJSDate(new Date(deepstore.open)).startOf('day');
if (libraryDate === undefined || deepstoreDate === undefined) {
throw new Error('One of these dates is undefined!');
} else {
return libraryDate > deepstoreDate;
}
};

// the 2 functions below reproduce the logic used in the catalogue-api/items to generate available dates

const applyItemsApiLibraryLogic = (openingTimes: NextOpeningDate[]) => {
// the library is open today if the 1st date in NextOpeningDate[] is the same as today
const isLibraryOpenToday =
new Date(dateNow).getDate() ===
(openingTimes[0].open && new Date(openingTimes[0].open).getDate());
if (dateNow.getHours() < 10 || !isLibraryOpenToday) {
return openingTimes.slice(1, -1).map(formatDate);
} else {
return openingTimes.slice(2, -1).map(formatDate);
}
};

const applyItemsApiDeepstoreLogic = (
libraryOpeningTimes: NextOpeningDate[],
deepstoreOpeningTimes: NextOpeningDate[]
) => {
const firstDeepstoreAvailability = deepstoreOpeningTimes.slice(10, -1)[0];
const subsequentLibraryAvailabilities = libraryOpeningTimes.filter(
libraryOpeningTime => {
return compareDates(libraryOpeningTime, firstDeepstoreAvailability);
}
);
return subsequentLibraryAvailabilities.map(formatDate);
};

const run = async () => {
const elasticClient = await getElasticClient({
serviceName: 'api',
pipelineDate: '2024-12-10',
hostEndpointAccess: 'public',
});

const searchResponse = await elasticClient.search<ElasticsearchVenue>({
index: 'venues',
_source: ['display'],
query: {
bool: {
filter: [{ terms: { 'filter.title': ['library', 'deepstore'] } }],
},
},
});
const venuesData = searchResponse.hits.hits.flatMap(hit =>
hit._source ? [hit._source.display] : []
);

const {
regularOpeningDays: libraryRegularOpeningDays,
exceptionalClosedDays: libraryHolidayclosures,
} = venuesData.find(venue => venue.title === 'Library') as Venue;
const {
regularOpeningDays: deepstoreRegularOpeningDays,
exceptionalClosedDays: deepstoreHolidayclosures,
} = venuesData.find(venue => venue.title === 'Deepstore') as Venue;

const onsiteItemsPickup = applyItemsApiLibraryLogic(
getNextOpeningDates(libraryRegularOpeningDays, libraryHolidayclosures)
).slice(0, 12);
const offsiteItemsPickup = applyItemsApiDeepstoreLogic(
getNextOpeningDates(libraryRegularOpeningDays, libraryHolidayclosures),
getNextOpeningDates(deepstoreRegularOpeningDays, deepstoreHolidayclosures)
).slice(0, 12);

const library = `Onsite items requested on ${dateNow} will be available on: \n${onsiteItemsPickup.join('\r')}`;
const deepstore = `Deepstore items requested on ${dateNow} will be available on: \n${offsiteItemsPickup.join('\r')}`;
console.log(library);
console.log(deepstore);
};

run();
2 changes: 2 additions & 0 deletions api/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { Config } from '@weco/content-api/config';
import { logStream } from '@weco/content-common/services/logging';

import {
addressablesController,
articleController,
articlesController,
errorHandler,
Expand All @@ -20,6 +21,7 @@ const createApp = (clients: Clients, config: Config) => {

app.use(morgan('short', { stream: logStream('http') }));

app.get('/all', addressablesController(clients, config));
app.get('/articles', articlesController(clients, config));
app.get('/articles/:id', articleController(clients, config));
app.get('/events', eventsController(clients, config));
Expand Down
72 changes: 72 additions & 0 deletions api/src/controllers/addressables.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { errors as elasticErrors } from '@elastic/elasticsearch';
import { RequestHandler } from 'express';
import asyncHandler from 'express-async-handler';

import { Config } from '@weco/content-api/config';
import { ifDefined } from '@weco/content-api/src/helpers';
import { resultListResponse } from '@weco/content-api/src/helpers/responses';
import { addressablesQuery } from '@weco/content-api/src/queries/addressables';
import { Clients, Displayable } from '@weco/content-api/src/types';
import { ResultList } from '@weco/content-api/src/types/responses';

import { HttpError } from './error';
import { paginationElasticBody, PaginationQueryParameters } from './pagination';

type QueryParams = {
query?: string;
} & PaginationQueryParameters;

type AddressablesHandler = RequestHandler<
never,
ResultList,
never,
QueryParams
>;

const addressablesController = (
clients: Clients,
config: Config
): AddressablesHandler => {
const index = config.addressablesIndex;
const resultList = resultListResponse(config);

return asyncHandler(async (req, res) => {
const { query: queryString } = req.query;

try {
const searchResponse = await clients.elastic.search<Displayable>({
index,
_source: ['display'],
query: {
bool: {
must: ifDefined(queryString, addressablesQuery),
},
},

...paginationElasticBody(req.query),
});

res.status(200).json(resultList(req, searchResponse));
} catch (e) {
if (
e instanceof elasticErrors.ResponseError &&
// This is an error we see from very long (spam) queries which contain
// many many terms and so overwhelm the multi_match query. The check
// for length is a heuristic so that if we get legitimate `too_many_nested_clauses`
// errors, we're still alerted to them
e.message.includes('too_many_nested_clauses') &&
encodeURIComponent(queryString || '').length > 2000
) {
throw new HttpError({
status: 400,
label: 'Bad Request',
description:
'Your query contained too many terms, please try again with a simpler query',
});
}
throw e;
}
});
};

export default addressablesController;
2 changes: 2 additions & 0 deletions api/src/controllers/error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ export class HttpError extends Error {
}
}

// unused var: https://github.com/ladjs/supertest/issues/416#issuecomment-514508137
// eslint-disable-next-line @typescript-eslint/no-unused-vars
export const errorHandler: ErrorRequestHandler = (err, req, res, next) => {
if (err instanceof HttpError) {
res.status(err.status).json(err.responseJson);
Expand Down
2 changes: 2 additions & 0 deletions api/src/controllers/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import addressablesController from './addressables';
import articleController from './article';
import articlesController from './articles';
import eventController from './event';
Expand All @@ -7,6 +8,7 @@ import venuesController from './venue';

export { errorHandler } from './error';
export {
addressablesController,
articlesController,
articleController,
eventController,
Expand Down
21 changes: 21 additions & 0 deletions api/src/queries/addressables.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';

export const addressablesQuery = (
queryString: string
): QueryDslQueryContainer => ({
multi_match: {
query: queryString,
fields: [
'id',
'uid',
'query.title.*^100',
'query.contributors.*^10',
'query.contributors.keyword^100',
'query.body.*',
'query.description.*',
],
operator: 'or',
type: 'cross_fields',
minimum_should_match: '-25%',
},
});
17 changes: 17 additions & 0 deletions api/test/addressables.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { mockedApi } from './fixtures/api';

describe('GET /all', () => {
it('returns a list of documents', async () => {
const docs = Array.from({ length: 10 }).map((_, i) => ({
id: `id-${i}`,
display: {
title: `test doc ${i}`,
},
}));
const api = mockedApi(docs);

const response = await api.get(`/all`);
expect(response.statusCode).toBe(200);
expect(response.body.results).toStrictEqual(docs.map(d => d.display));
});
});
30 changes: 18 additions & 12 deletions api/test/fixtures/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,31 @@ const elastic404 = () =>
warnings: [],
});

const testAddressablesIndex = 'test-addressables';
const testArticlesIndex = 'test-articles';
const testEventsIndex = 'test-events';
const testVenuesIndex = 'test-venue';

export const mockConfig = {
pipelineDate: '2222-22-22',
addressablesIndex: testAddressablesIndex,
articlesIndex: testArticlesIndex,
eventsIndex: testEventsIndex,
venuesIndex: testVenuesIndex,
publicRootUrl: new URL('http://test.test/test'),
};

export const mockedApi = <T extends Displayable & Identified>(
documents: T[]
) => {
const testArticlesIndex = 'test-article-index';
const testEventsIndex = 'test-event-index';
const testVenuesIndex = 'test-venue-index';

const documentsMap = new Map(documents.map(d => [d.id, d]));

const elasticClientGet = jest.fn(
({ id, index }: Parameters<ElasticClient['get']>[0]) => {
if (
documentsMap.has(id) &&
(index === testArticlesIndex ||
(index === testAddressablesIndex ||
index === testArticlesIndex ||
index === testEventsIndex ||
index === testVenuesIndex)
) {
Expand All @@ -50,6 +61,7 @@ export const mockedApi = <T extends Displayable & Identified>(
const elasticClientSearch = jest.fn(
(params: Parameters<ElasticClient['search']>[0]) => {
if (
params?.index === testAddressablesIndex ||
params?.index === testArticlesIndex ||
params?.index === testEventsIndex ||
params?.index === testVenuesIndex
Expand All @@ -75,13 +87,7 @@ export const mockedApi = <T extends Displayable & Identified>(
search: elasticClientSearch,
} as unknown as ElasticClient,
},
{
pipelineDate: '2222-22-22',
articlesIndex: testArticlesIndex,
eventsIndex: testEventsIndex,
venuesIndex: testVenuesIndex,
publicRootUrl: new URL('http://test.test/test'),
}
mockConfig
);

return supertest.agent(app);
Expand Down
12 changes: 4 additions & 8 deletions api/test/query.test.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,19 @@
import { Client as ElasticClient } from '@elastic/elasticsearch';
import { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
import supertest from 'supertest';
import { URL, URLSearchParams } from 'url';
import { URLSearchParams } from 'url';

import createApp from '@weco/content-api/src/app';

import { mockConfig } from './fixtures/api';

const elasticsearchRequestForURL = async (
url: string
): Promise<SearchRequest> => {
const searchSpy = jest.fn();
const app = createApp(
{ elastic: { search: searchSpy } as unknown as ElasticClient },
{
pipelineDate: '2222-22-22',
articlesIndex: 'test-articles',
eventsIndex: 'test-events',
venuesIndex: 'test-venues',
publicRootUrl: new URL('http://test.test/test'),
}
mockConfig
);
const api = supertest.agent(app);
await api.get(url);
Expand Down
13 changes: 13 additions & 0 deletions infrastructure/pipeline.tf
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,19 @@ module "pipeline" {
logging_cluster_id = local.logging_cluster_id
network_config = local.network_config
lambda_alarm_topic_arn = local.catalogue_lambda_alarn_topic_arn
deployment_template_id = "aws-io-optimized-v3"
unpublish_event_rule = module.webhook.unpublish_event_rule
}

module "pipeline_2024-12-10" {
source = "./pipeline_stack"

pipeline_date = "2024-12-10"
window_duration_minutes = 15
deployment_template_id = "aws-storage-optimized"
logging_cluster_id = local.logging_cluster_id
network_config = local.network_config
lambda_alarm_topic_arn = local.catalogue_lambda_alarn_topic_arn

unpublish_event_rule = module.webhook.unpublish_event_rule
}
Loading

0 comments on commit 013e50e

Please sign in to comment.