Point to new pipeline

wellcomecollection · Dec 12, 2024 · 3884dc7 · 3884dc7
1 parent fd72d75
commit 3884dc7
Show file tree

Hide file tree

Showing 7 changed files with 112 additions and 32 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,5 +1,5 @@
 env:
-  LIVE_PIPELINE: '2023-03-24'
+  LIVE_PIPELINE: '2024-12-10'
 steps:
   - label: 'autoformat'
     command: '.buildkite/scripts/autoformat.sh'

diff --git a/api/config.ts b/api/config.ts
@@ -12,7 +12,7 @@ const environment = environmentSchema.parse(process.env);
 // This configuration is exposed via the public healthcheck endpoint,
 // so be careful not to expose any secrets here.
 const config = {
-  pipelineDate: '2023-03-24',
+  pipelineDate: '2024-12-10',
   addressablesIndex: 'addressables',
   articlesIndex: 'articles',
   eventsIndex: 'events',

diff --git a/api/scripts/holiday_closure_test.ts b/api/scripts/holiday_closure_test.ts
@@ -1,4 +1,4 @@
-/* 
+/*
 Before the end-of-year closure, we test that the date picker dropdown in the Request item dialog is going to display the correct options
 to ensure that users can only requests items when they can be available, on a day the library is open
 Once the Modified opening times have been added to Prismic and published,
@@ -69,7 +69,7 @@ const applyItemsApiDeepstoreLogic = (
 const run = async () => {
   const elasticClient = await getElasticClient({
     serviceName: 'api',
-    pipelineDate: '2023-03-24',
+    pipelineDate: '2024-12-10',
     hostEndpointAccess: 'public',
   });
 

diff --git a/pipeline/src/handler.ts b/pipeline/src/handler.ts
@@ -1,4 +1,4 @@
-// import * as prismic from '@prismicio/client';
+import * as prismic from '@prismicio/client';
 import { Handler } from 'aws-lambda';
 
 import { WindowEvent } from './event';
@@ -10,32 +10,32 @@ import {
   webcomicsQuery,
   wrapQueries,
 } from './graph-queries';
-// import { addressablesQuery } from './graph-queries/addressables';
-import { articles, events, venues } from './indices';
-// import { transformAddressable } from './transformers/addressables';
+import { addressablesQuery } from './graph-queries/addressables';
+import { addressables, articles, events, venues } from './indices';
+import { transformAddressable } from './transformers/addressables';
 import { transformArticle } from './transformers/article';
 import { transformEventDocument } from './transformers/eventDocument';
 import { transformVenue } from './transformers/venue';
 import { Clients } from './types';
 
-// const loadAddressables = createETLPipeline({
-//   graphQuery: addressablesQuery,
-//   filters: [prismic.filter.not('document.tags', ['delist'])],
-//   indexConfig: addressables,
-//   parentDocumentTypes: new Set([
-//     'articles',
-//     'books',
-//     'events',
-//     'exhibitions',
-//     'exhibition-texts',
-//     'exhibition-highlight-tours',
-//     'pages',
-//     'projects',
-//     'seasons',
-//     'visual-stories',
-//   ]),
-//   transformer: transformAddressable,
-// });
+const loadAddressables = createETLPipeline({
+  graphQuery: addressablesQuery,
+  filters: [prismic.filter.not('document.tags', ['delist'])],
+  indexConfig: addressables,
+  parentDocumentTypes: new Set([
+    'articles',
+    'books',
+    'events',
+    'exhibitions',
+    'exhibition-texts',
+    'exhibition-highlight-tours',
+    'pages',
+    'projects',
+    'seasons',
+    'visual-stories',
+  ]),
+  transformer: transformAddressable,
+});
 
 const loadArticles = createETLPipeline({
   graphQuery: wrapQueries(articlesQuery, webcomicsQuery),
@@ -64,9 +64,9 @@ export const createHandler =
     if (!event.contentType) {
       throw new Error('Event contentType must be specified!');
     }
-    // if (event.contentType === 'all' || event.contentType === 'addressables') {
-    //   await loadAddressables(clients, event);
-    // }
+    if (event.contentType === 'all' || event.contentType === 'addressables') {
+      await loadAddressables(clients, event);
+    }
     if (event.contentType === 'all' || event.contentType === 'articles') {
       await loadArticles(clients, event);
     }

diff --git a/pipeline/src/local.ts b/pipeline/src/local.ts
@@ -23,7 +23,7 @@ const windowEvent: WindowEvent = {
 };
 
 getElasticClient({
-  pipelineDate: '2023-03-24',
+  pipelineDate: '2024-12-10',
   serviceName: 'pipeline',
   hostEndpointAccess: 'public',
 }).then(elasticClient => {

diff --git a/pipeline/test/extractTransformLoad.test.ts b/pipeline/test/extractTransformLoad.test.ts
@@ -8,8 +8,14 @@ import {
   webcomicsQuery,
   wrapQueries,
 } from '@weco/content-pipeline/src/graph-queries';
+import { addressablesQuery } from '@weco/content-pipeline/src/graph-queries/addressables';
 import { isFilledLinkToDocument } from '@weco/content-pipeline/src/helpers/type-guards';
-import { articles, events } from '@weco/content-pipeline/src/indices';
+import {
+  addressables,
+  articles,
+  events,
+} from '@weco/content-pipeline/src/indices';
+import { transformAddressable } from '@weco/content-pipeline/src/transformers/addressables';
 import { transformArticle } from '@weco/content-pipeline/src/transformers/article';
 import { transformEventDocument } from '@weco/content-pipeline/src/transformers/eventDocument';
 import { ArticlePrismicDocument } from '@weco/content-pipeline/src/types/prismic';
@@ -216,3 +222,77 @@ describe('Extract, transform and load articles', () => {
     );
   });
 });
+
+describe('Extract, transform and load addressables', () => {
+  it('fetches addressables from prismic and indexes them into ES', async () => {
+    const allDocs = getSnapshots(
+      [
+        'articles',
+        'books',
+        'events',
+        'exhibition-highlight-tours',
+        'exhibitions',
+        'exhibition-texts',
+        'pages',
+        'projects',
+        'seasons',
+        'visual-stories',
+      ],
+      true
+    );
+
+    const elasticIndexCreator = jest.fn().mockResolvedValue(true);
+    const [elasticBulkHelper, getIndexedDocuments] = createElasticBulkHelper();
+    const elasticClient = {
+      indices: {
+        create: elasticIndexCreator,
+      },
+      helpers: {
+        bulk: elasticBulkHelper,
+      },
+    } as unknown as ElasticClient;
+
+    const prismicClient = {
+      get: prismicGet(allDocs),
+    } as unknown as prismic.Client;
+
+    const addressablePipeline = createETLPipeline({
+      indexConfig: addressables,
+      graphQuery: addressablesQuery,
+      parentDocumentTypes: new Set([
+        'articles',
+        'books',
+        'events',
+        'exhibition-highlight-tours',
+        'exhibitions',
+        'exhibition-texts',
+        'pages',
+        'projects',
+        'seasons',
+        'visual-stories',
+      ]),
+      transformer: transformAddressable,
+    });
+
+    await addressablePipeline(
+      { elastic: elasticClient, prismic: prismicClient },
+      { contentType: 'addressables' }
+    );
+    expect(elasticIndexCreator).toHaveBeenCalled();
+    expect(elasticBulkHelper).toHaveBeenCalled();
+
+    const expectedIdsFromAllDocs = allDocs
+      .map(d => {
+        if (d.type === 'exhibition-highlight-tours') {
+          return [`${d.id}/${d.type}/audio`, `${d.id}/${d.type}/bsl`];
+        } else {
+          return `${d.id}/${d.type}`;
+        }
+      })
+      .flat();
+    const indexedDocs = getIndexedDocuments();
+    const indexedIds = indexedDocs.map(d => d.id);
+
+    expect(indexedIds).toIncludeSameMembers(expectedIdsFromAllDocs);
+  });
+});
diff --git a/unpublisher/src/local.ts b/unpublisher/src/local.ts
@@ -10,7 +10,7 @@ import { createHandler } from './handler';
 const [_1, _2, ...deletionIds] = argv;
 
 getElasticClient({
-  pipelineDate: '2023-03-24',
+  pipelineDate: '2024-12-10',
   serviceName: 'unpublisher',
   hostEndpointAccess: 'public',
 }).then(async elasticClient => {