From 6f1290d267f181e3bfa862302e22032c55586b16 Mon Sep 17 00:00:00 2001 From: mbret Date: Wed, 18 Dec 2024 01:15:42 +0100 Subject: [PATCH 1/3] feat: improved collection metadata --- package-lock.json | 1 + packages/api/package.json | 3 +- .../handler.ts | 142 +++++----- .../src/collections.ts | 47 ++++ .../src/parameters.ts | 37 +++ .../src/refreshMetadata.ts | 248 ++++++++---------- .../refreshMetadataLongProcess/handler.ts | 4 +- .../syncDataSourceLongProcess/handler.ts | 4 +- .../api/src/functions/syncReports/handler.ts | 4 +- packages/api/src/libs/auth.ts | 15 +- .../src/libs/google/withConfiguredGoogle.ts | 16 ++ .../metadata/comicvine/getSeriesMetadata.ts | 13 +- packages/api/src/libs/supabase/deleteLock.ts | 27 ++ packages/api/src/libs/utils.ts | 36 +++ .../CollectionActionsDrawer.tsx | 4 +- .../getCollectionComputedMetadata.ts | 21 +- .../src/collections/useUpdateCollection.ts | 4 +- 17 files changed, 387 insertions(+), 239 deletions(-) create mode 100644 packages/api/src/functions/refreshMetadataCollectionLongProcess/src/collections.ts create mode 100644 packages/api/src/functions/refreshMetadataCollectionLongProcess/src/parameters.ts create mode 100644 packages/api/src/libs/google/withConfiguredGoogle.ts diff --git a/package-lock.json b/package-lock.json index 959abf58..2a9ef662 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34357,6 +34357,7 @@ "node-fetch": "^3.2.0", "node-unrar-js": "^2.0.2", "nodemailer": "^6.9.1", + "rxjs": "^7.5.7", "sharp": "0.33.5", "unzipper": "^0.12.3", "yup": "^1.3.2" diff --git a/packages/api/package.json b/packages/api/package.json index fa2293c7..40b6fcfa 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -45,7 +45,8 @@ "unzipper": "^0.12.3", "yup": "^1.3.2", "@sentry/aws-serverless": "^8.32.0", - "@sentry/profiling-node": "^8.32.0" + "@sentry/profiling-node": "^8.32.0", + "rxjs": "^7.5.7" }, "devDependencies": { "@types/node": "^20", diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts index 12a9475f..85b30dfd 100644 --- a/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts @@ -1,92 +1,82 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withToken } from "@libs/auth" -import { configure as configureGoogleDataSource } from "@libs/plugins/google" +import { getAuthToken } from "@libs/auth" import schema from "./schema" import { findOne, getNanoDbForUser } from "@libs/couch/dbHelpers" -import { getParametersValue } from "@libs/ssm" -import { deleteLock } from "@libs/supabase/deleteLock" +import { withDeleteLock } from "@libs/supabase/deleteLock" import { supabase } from "@libs/supabase/client" -import { Logger } from "@libs/logger" import { refreshMetadata } from "./src/refreshMetadata" import { withMiddy } from "@libs/middy/withMiddy" +import { from, lastValueFrom, map, mergeMap, of, switchMap } from "rxjs" +import { parameters$ } from "./src/parameters" +import { + onBeforeError, + switchMapCombineOuter, + switchMapMergeOuter +} from "@libs/utils" +import { withConfiguredGoogle } from "@libs/google/withConfiguredGoogle" +import { markCollectionAsError } from "./src/collections" const lambda: ValidatedEventAPIGatewayProxyEvent = async ( event ) => { - const [ - client_id = ``, - client_secret = ``, - googleApiKey = ``, - jwtPrivateKey = ``, - comicVineApiKey = `` - ] = await getParametersValue({ - Names: [ - "GOOGLE_CLIENT_ID", - "GOOGLE_CLIENT_SECRET", - "GOOGLE_API_KEY", - "jwt-private-key", - "COMiCVINE_API_KEY" - ], - WithDecryption: true - }) - - configureGoogleDataSource({ - client_id, - client_secret - }) - - const soft = event.body.soft === true - const authorization = event.body.authorization ?? `` - const rawCredentials = event.body.credentials ?? JSON.stringify({}) - const credentials = JSON.parse(rawCredentials) - - const { name: userName } = await withToken( - { - headers: { - authorization - } - }, - jwtPrivateKey - ) - - const collectionId: string | undefined = event.body.collectionId - - if (!collectionId) { - throw new Error(`Unable to parse event.body -> ${event.body}`) - } - + const collectionId = event.body.collectionId ?? "" const lockId = `metadata-collection_${collectionId}` - const db = await getNanoDbForUser(userName, jwtPrivateKey) - - const collection = await findOne(db, "obokucollection", { - selector: { _id: collectionId } - }) - - if (!collection) throw new Error(`Unable to find book ${collectionId}`) - - try { - await refreshMetadata(collection, { - googleApiKey, - db, - credentials, - soft, - comicVineApiKey - }) - } catch (e) { - await deleteLock(supabase, lockId) - - throw e - } - - await deleteLock(supabase, lockId) - - Logger.info(`lambda executed with success for ${collection._id}`) + const result = await lastValueFrom( + of(event).pipe( + map((event) => { + const soft = event.body.soft === true + const authorization = event.body.authorization ?? `` + const rawCredentials = event.body.credentials ?? JSON.stringify({}) + const credentials = JSON.parse(rawCredentials) + + return { + soft, + authorization, + credentials + } + }), + switchMapMergeOuter(() => parameters$), + withConfiguredGoogle, + switchMapMergeOuter((params) => + getAuthToken(params.authorization, params.jwtPrivateKey) + ), + switchMapCombineOuter(({ name: userName, jwtPrivateKey }) => + from(getNanoDbForUser(userName, jwtPrivateKey)) + ), + switchMap(([params, db]) => + from( + findOne(db, "obokucollection", { + selector: { _id: collectionId } + }) + ).pipe( + mergeMap((collection) => { + if (!collection) + throw new Error(`Unable to find book ${collectionId}`) + + return from( + refreshMetadata(collection, { + db, + ...params + }) + ) + }), + onBeforeError(() => markCollectionAsError({ db, collectionId })) + ) + ), + map(() => { + console.info(`lambda executed with success for ${collectionId}`) + + return { + statusCode: 200, + body: JSON.stringify({}) + } + }), + withDeleteLock(supabase, lockId) + ) + ) - return { - statusCode: 200, - body: JSON.stringify({}) - } + return result } export const main = withMiddy(lambda, { diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/collections.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/collections.ts new file mode 100644 index 00000000..ff15d77c --- /dev/null +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/collections.ts @@ -0,0 +1,47 @@ +import { atomicUpdate } from "@libs/couch/dbHelpers" +import { CollectionDocType } from "@oboku/shared/src/db/docTypes" +import nano from "nano" +import { from } from "rxjs" + +export const markCollectionAsFetching = ({ + db, + collectionId +}: { + db: nano.DocumentScope + collectionId: string +}) => + from( + atomicUpdate(db, "obokucollection", collectionId, (old) => { + const wasAlreadyInitialized = + old.metadataUpdateStatus === "fetching" && old.lastMetadataStartedAt + + if (wasAlreadyInitialized) return old + + return { + ...old, + metadataUpdateStatus: "fetching" as const, + lastMetadataStartedAt: new Date().toISOString() + } + }) + ) + +export const markCollectionAsError = ({ + db, + collectionId +}: { + db: nano.DocumentScope + collectionId: string +}) => + from( + atomicUpdate( + db, + "obokucollection", + collectionId, + (old) => + ({ + ...old, + metadataUpdateStatus: "idle", + lastMetadataUpdateError: "unknown" + }) satisfies CollectionDocType + ) + ) diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/parameters.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/parameters.ts new file mode 100644 index 00000000..cb4cbda4 --- /dev/null +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/parameters.ts @@ -0,0 +1,37 @@ +import { map } from "rxjs/operators" + +import { from } from "rxjs" + +import { getParametersValue } from "@libs/ssm" +import { defer } from "rxjs" + +export const parameters$ = defer(() => + from( + getParametersValue({ + Names: [ + "GOOGLE_CLIENT_ID", + "GOOGLE_CLIENT_SECRET", + "GOOGLE_API_KEY", + "jwt-private-key", + "COMiCVINE_API_KEY" + ], + WithDecryption: true + }) + ).pipe( + map( + ([ + client_id = ``, + client_secret = ``, + googleApiKey = ``, + jwtPrivateKey = ``, + comicVineApiKey = `` + ]) => ({ + client_id, + client_secret, + googleApiKey, + jwtPrivateKey, + comicVineApiKey + }) + ) + ) +) diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts index 92e03347..413d63f2 100644 --- a/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts @@ -10,6 +10,8 @@ import { Logger } from "@libs/logger" import { pluginFacade } from "@libs/plugins/facade" import { computeMetadata } from "@libs/collections/computeMetadata" import { saveOrUpdateCover } from "./saveOrUpdateCover" +import { from, lastValueFrom, of, switchMap } from "rxjs" +import { markCollectionAsFetching } from "./collections" export const refreshMetadata = async ( collection: CollectionDocType, @@ -27,153 +29,123 @@ export const refreshMetadata = async ( comicVineApiKey: string } ) => { - if (collection.type !== "series") { - Logger.info(`Ignoring ${collection._id} because of type ${collection.type}`) + const { isCollectionAlreadyUpdatedFromLink, linkMetadataInfo } = + await lastValueFrom( + markCollectionAsFetching({ db, collectionId: collection._id }).pipe( + // we always get updated link data + // we take a chance to update the collection from its resource + switchMap(() => + collection.linkResourceId && collection.linkType + ? from( + pluginFacade.getMetadata({ + link: { + resourceId: collection.linkResourceId, + type: collection.linkType, + data: null + }, + credentials + }) + ) + : of(undefined) + ), + switchMap((linkMetadataInfo) => { + const linkModifiedAt = linkMetadataInfo?.modifiedAt + ? new Date(linkMetadataInfo.modifiedAt) + : undefined + const collectionMetadataUpdatedAt = collection?.lastMetadataUpdatedAt + ? new Date(collection?.lastMetadataUpdatedAt) + : undefined + + const isCollectionAlreadyUpdatedFromLink = + linkModifiedAt && + collectionMetadataUpdatedAt && + linkModifiedAt.getTime() < collectionMetadataUpdatedAt.getTime() + + return of({ linkMetadataInfo, isCollectionAlreadyUpdatedFromLink }) + }) + ) + ) + + /** + * @important + * In case of soft refresh, we only update if the link is updated + * or if there is no link but the collection has not yet fetched metadata. + * This soft mode is mostly used during sync. + */ + if (soft && isCollectionAlreadyUpdatedFromLink) { + Logger.info(`${collection._id} already has metadata, ignoring it!`) return } - try { - await atomicUpdate(db, "obokucollection", collection._id, (old) => { - const wasAlreadyInitialized = - old.metadataUpdateStatus === "fetching" && old.lastMetadataStartedAt - - if (wasAlreadyInitialized) return old - - return { - ...old, - metadataUpdateStatus: "fetching" as const, - lastMetadataStartedAt: new Date().toISOString() - } - }) - - // we always get updated link data - // we take a chance to update the collection from its resource - const linkMetadataInfo = - collection.linkResourceId && collection.linkType - ? await pluginFacade.getMetadata({ - link: { - resourceId: collection.linkResourceId, - type: collection.linkType, - data: null - }, - credentials - }) - : undefined - - const linkModifiedAt = linkMetadataInfo?.modifiedAt - ? new Date(linkMetadataInfo.modifiedAt) - : undefined - const collectionMetadataUpdatedAt = collection?.lastMetadataUpdatedAt - ? new Date(collection?.lastMetadataUpdatedAt) - : undefined - - const isCollectionAlreadyUpdatedFromLink = - linkModifiedAt && - collectionMetadataUpdatedAt && - linkModifiedAt.getTime() < collectionMetadataUpdatedAt.getTime() - - /** - * @important - * In case of soft refresh, we only update if the link is updated - * or if there is no link but the collection has not yet fetched metadata. - * This soft mode is mostly used during sync. - */ - - if (soft && isCollectionAlreadyUpdatedFromLink) { - Logger.info(`${collection._id} already has metadata, ignoring it!`) - - return { - statusCode: 200, - body: JSON.stringify({}) - } - } - - if (soft && !linkMetadataInfo && collection.lastMetadataUpdatedAt) { - Logger.info( - `${collection._id} does not have link and is already refreshed, ignoring it!` - ) - - return { - statusCode: 200, - body: JSON.stringify({}) - } - } - - const metadataUser = collection.metadata?.find( - (item) => item.type === "user" + if (soft && !linkMetadataInfo && collection.lastMetadataUpdatedAt) { + Logger.info( + `${collection._id} does not have link and is already refreshed, ignoring it!` ) - const { title: userTitle, startYear: userStartYear } = computeMetadata([ - metadataUser - ]) - const directivesFromLink = directives.extractDirectivesFromName( - linkMetadataInfo?.name ?? "" - ) + return + } - const title = directives.removeDirectiveFromString( - directivesFromLink.metadataTitle ?? - linkMetadataInfo?.name ?? - userTitle ?? - "" - ) + const metadataUser = collection.metadata?.find((item) => item.type === "user") + const { title: userTitle, startYear: userStartYear } = computeMetadata([ + metadataUser + ]) + + const directivesFromLink = directives.extractDirectivesFromName( + linkMetadataInfo?.name ?? "" + ) + + const title = directives.removeDirectiveFromString( + directivesFromLink.metadataTitle ?? + linkMetadataInfo?.name ?? + userTitle ?? + "" + ) + + const year = directivesFromLink.year ?? userStartYear + + const externalMetadatas = + collection.type === "series" + ? await fetchMetadata( + { title, year: year ? String(year) : undefined }, + { withGoogle: true, googleApiKey, comicVineApiKey } + ) + : [] + + const linkMetadata: CollectionMetadata = { + type: "link", + ...collection.metadata?.find((item) => item.type === "link"), + title: linkMetadataInfo?.name + } - const year = directivesFromLink.year ?? userStartYear + // try to get latest collection to stay as fresh as possible + const currentCollection = await findOne(db, "obokucollection", { + selector: { _id: collection._id } + }) - const externalMetadatas = await fetchMetadata( - { title, year: year ? String(year) : undefined }, - { withGoogle: true, googleApiKey, comicVineApiKey } - ) + if (!currentCollection) throw new Error("Unable to find collection") - const linkMetadata: CollectionMetadata = { - type: "link", - ...collection.metadata?.find((item) => item.type === "link"), - title: linkMetadataInfo?.name - } - - // try to get latest collection to stay as fresh as possible - const currentCollection = await findOne(db, "obokucollection", { - selector: { _id: collection._id } - }) - - if (!currentCollection) throw new Error("Unable to find collection") - - const userMetadata = - currentCollection.metadata?.filter((entry) => entry.type === "user") ?? [] - const metadata = [...userMetadata, ...externalMetadatas, linkMetadata] - - // cannot be done in // since metadata status will trigger cover refresh - await saveOrUpdateCover(currentCollection, { - _id: currentCollection._id, - metadata - }) - - await atomicUpdate( - db, - "obokucollection", - collection._id, - (old) => - ({ - ...old, - lastMetadataUpdatedAt: new Date().toISOString(), - metadataUpdateStatus: "idle", - lastMetadataUpdateError: null, - metadata - }) satisfies CollectionDocType - ) - } catch (error) { - await atomicUpdate( - db, - "obokucollection", - collection._id, - (old) => - ({ - ...old, - metadataUpdateStatus: "idle", - lastMetadataUpdateError: "unknown" - }) satisfies CollectionDocType - ) + const userMetadata = + currentCollection.metadata?.filter((entry) => entry.type === "user") ?? [] + const metadata = [...userMetadata, ...externalMetadatas, linkMetadata] - throw error - } + // cannot be done in // since metadata status will trigger cover refresh + await saveOrUpdateCover(currentCollection, { + _id: currentCollection._id, + metadata + }) + + await atomicUpdate( + db, + "obokucollection", + collection._id, + (old) => + ({ + ...old, + lastMetadataUpdatedAt: new Date().toISOString(), + metadataUpdateStatus: "idle", + lastMetadataUpdateError: null, + metadata + }) satisfies CollectionDocType + ) } diff --git a/packages/api/src/functions/refreshMetadataLongProcess/handler.ts b/packages/api/src/functions/refreshMetadataLongProcess/handler.ts index 4cd9b51e..59ae1618 100644 --- a/packages/api/src/functions/refreshMetadataLongProcess/handler.ts +++ b/packages/api/src/functions/refreshMetadataLongProcess/handler.ts @@ -2,7 +2,7 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" import fs from "fs" import path from "path" import { OFFLINE, TMP_DIR } from "../../constants" -import { withToken } from "@libs/auth" +import { getAuthTokenAsync } from "@libs/auth" import { configure as configureGoogleDataSource } from "@libs/plugins/google" import schema from "./schema" import { atomicUpdate, findOne, getNanoDbForUser } from "@libs/couch/dbHelpers" @@ -55,7 +55,7 @@ const lambda: ValidatedEventAPIGatewayProxyEvent = async ( const credentials = JSON.parse(rawCredentials) - const { name: userName } = await withToken( + const { name: userName } = await getAuthTokenAsync( { headers: { authorization diff --git a/packages/api/src/functions/syncDataSourceLongProcess/handler.ts b/packages/api/src/functions/syncDataSourceLongProcess/handler.ts index ec1be808..b60a8e82 100644 --- a/packages/api/src/functions/syncDataSourceLongProcess/handler.ts +++ b/packages/api/src/functions/syncDataSourceLongProcess/handler.ts @@ -1,6 +1,6 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" import { configure as configureGoogleDataSource } from "@libs/plugins/google" -import { withToken } from "@libs/auth" +import { getAuthTokenAsync } from "@libs/auth" import schema from "./schema" import { createHttpError } from "@libs/httpErrors" import { getNanoDbForUser } from "@libs/couch/dbHelpers" @@ -31,7 +31,7 @@ const lambda: ValidatedEventAPIGatewayProxyEvent = async ( const credentials = JSON.parse(event.body.credentials ?? JSON.stringify({})) const authorization = event.body.authorization ?? `` - const { name } = await withToken( + const { name } = await getAuthTokenAsync( { headers: { authorization diff --git a/packages/api/src/functions/syncReports/handler.ts b/packages/api/src/functions/syncReports/handler.ts index 193d3b0a..461fae31 100644 --- a/packages/api/src/functions/syncReports/handler.ts +++ b/packages/api/src/functions/syncReports/handler.ts @@ -1,5 +1,5 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withToken } from "@libs/auth" +import { getAuthTokenAsync } from "@libs/auth" import schema from "./schema" import { getParametersValue } from "@libs/ssm" import { supabase } from "@libs/supabase/client" @@ -15,7 +15,7 @@ const lambda: ValidatedEventAPIGatewayProxyEvent = async ( WithDecryption: true }) - const { name } = await withToken( + const { name } = await getAuthTokenAsync( { headers: { authorization diff --git a/packages/api/src/libs/auth.ts b/packages/api/src/libs/auth.ts index 69591dc3..d4bf7d4e 100644 --- a/packages/api/src/libs/auth.ts +++ b/packages/api/src/libs/auth.ts @@ -1,6 +1,7 @@ import jwt from "jsonwebtoken" import { APIGatewayProxyEvent } from "aws-lambda" import { createHttpError } from "./httpErrors" +import { from } from "rxjs" const isAuthorized = async ({ privateKey, @@ -58,7 +59,7 @@ export const generateAdminToken = async ({ return jwt.sign(data, privateKey, { algorithm: "RS256" }) } -export const withToken = async ( +export const getAuthTokenAsync = async ( event: Pick, privateKey: string ) => { @@ -68,3 +69,15 @@ export const withToken = async ( return await isAuthorized({ authorization, privateKey }) } + +export const getAuthToken = (authorization: string, privateKey: string) => + from( + getAuthTokenAsync( + { + headers: { + authorization + } + }, + privateKey + ) + ) diff --git a/packages/api/src/libs/google/withConfiguredGoogle.ts b/packages/api/src/libs/google/withConfiguredGoogle.ts new file mode 100644 index 00000000..6e1f9756 --- /dev/null +++ b/packages/api/src/libs/google/withConfiguredGoogle.ts @@ -0,0 +1,16 @@ +import { configure as configureGoogleDataSource } from "@libs/plugins/google" +import { Observable, tap } from "rxjs" + +export const withConfiguredGoogle = < + T extends { client_id: string; client_secret: string } +>( + stream: Observable +) => + stream.pipe( + tap(({ client_id, client_secret }) => + configureGoogleDataSource({ + client_id, + client_secret + }) + ) + ) diff --git a/packages/api/src/libs/metadata/comicvine/getSeriesMetadata.ts b/packages/api/src/libs/metadata/comicvine/getSeriesMetadata.ts index 81f3de79..aa348a1e 100644 --- a/packages/api/src/libs/metadata/comicvine/getSeriesMetadata.ts +++ b/packages/api/src/libs/metadata/comicvine/getSeriesMetadata.ts @@ -1,4 +1,4 @@ - import { CollectionMetadata } from "@oboku/shared" +import { CollectionMetadata } from "@oboku/shared" import { Logger } from "@libs/logger" import { URL } from "url" import axios from "axios" @@ -25,7 +25,7 @@ type Result = { issue_number: string } id: number - image: { + image?: { icon_url: string medium_url: string screen_url: string @@ -34,7 +34,7 @@ type Result = { super_url: string thumb_url: string tiny_url: string - original_url: string + original_url?: string image_tags: string } last_issue: { @@ -93,7 +93,12 @@ export const getSeriesMetadata = async ({ description: result?.description, aliases: result?.aliases ? [result?.aliases] : undefined, startYear: result?.start_year ? Number(result?.start_year) : undefined, - publisherName: result?.publisher.name + publisherName: result?.publisher.name, + ...(result.image?.original_url && { + cover: { + uri: result.image?.original_url + } + }) } satisfies CollectionMetadata } catch (e) { Logger.error(e) diff --git a/packages/api/src/libs/supabase/deleteLock.ts b/packages/api/src/libs/supabase/deleteLock.ts index 8d0005ab..d01eae05 100644 --- a/packages/api/src/libs/supabase/deleteLock.ts +++ b/packages/api/src/libs/supabase/deleteLock.ts @@ -1,5 +1,14 @@ import { Logger } from "@libs/logger" import { SupabaseClient } from "@supabase/supabase-js" +import { + catchError, + from, + ignoreElements, + map, + Observable, + switchMap, + tap +} from "rxjs" export const deleteLock = async (supabase: SupabaseClient, lockId: string) => { Logger.info(`releasing lock ${lockId}`) @@ -10,3 +19,21 @@ export const deleteLock = async (supabase: SupabaseClient, lockId: string) => { Logger.error(response.error) } } + +export const withDeleteLock = + (supabase: SupabaseClient, lockId: string) => + (stream: Observable) => { + return stream.pipe( + switchMap((value) => + from(deleteLock(supabase, lockId)).pipe(map(() => value)) + ), + catchError((error) => + from(deleteLock(supabase, lockId)).pipe( + tap(() => { + throw error + }), + ignoreElements() + ) + ) + ) + } diff --git a/packages/api/src/libs/utils.ts b/packages/api/src/libs/utils.ts index 287b63b4..60f3699c 100644 --- a/packages/api/src/libs/utils.ts +++ b/packages/api/src/libs/utils.ts @@ -2,6 +2,14 @@ import { APIGatewayProxyEvent } from "aws-lambda" import fs from "fs" import unzipper from "unzipper" import { READER_SUPPORTED_MIME_TYPES } from "@oboku/shared" +import { + catchError, + ignoreElements, + map, + Observable, + switchMap, + tap +} from "rxjs" export const waitForRandomTime = (min: number, max: number) => new Promise((resolve) => @@ -134,3 +142,31 @@ export function mergeSkippingUndefined( return result as T } + +export const onBeforeError = + (callback: (error: unknown) => Observable) => + (stream: Observable) => + stream.pipe( + catchError((error) => + callback(error).pipe( + tap(() => { + throw error + }), + ignoreElements() + ) + ) + ) + +export const switchMapMergeOuter = ( + project: (value: T) => Observable +) => + switchMap((outer: T) => + project(outer).pipe(map((inner) => ({ ...outer, ...inner }))) + ) + +export const switchMapCombineOuter = ( + project: (value: T) => Observable +) => + switchMap((outer: T) => + project(outer).pipe(map((inner) => [outer, inner])) + ) diff --git a/packages/web/src/collections/CollectionActionsDrawer/CollectionActionsDrawer.tsx b/packages/web/src/collections/CollectionActionsDrawer/CollectionActionsDrawer.tsx index e233c35e..02e93296 100644 --- a/packages/web/src/collections/CollectionActionsDrawer/CollectionActionsDrawer.tsx +++ b/packages/web/src/collections/CollectionActionsDrawer/CollectionActionsDrawer.tsx @@ -29,12 +29,12 @@ import { useRefreshCollectionMetadata } from "../useRefreshCollectionMetadata" import { useRemoveCollection } from "../useRemoveCollection" import { useUpdateCollectionBooks } from "../useUpdateCollectionBooks" import { useCollection } from "../useCollection" +import { COLLECTION_EMPTY_ID } from "../../constants.shared" export const CollectionActionsDrawer: FC<{}> = () => { const { openedWith, lastId: collectionId } = useSignalValue( collectionActionDrawerState ) - const [ isEditCollectionDialogOpenedWithId, setIsEditCollectionDialogOpenedWithId @@ -138,7 +138,7 @@ export const CollectionActionsDrawer: FC<{}> = () => { - {collection?.type === "series" && ( + {collection && collection._id !== COLLECTION_EMPTY_ID && ( { refreshCollectionMetadata(collectionId ?? ``) diff --git a/packages/web/src/collections/getCollectionComputedMetadata.ts b/packages/web/src/collections/getCollectionComputedMetadata.ts index 856e2f4c..42af890c 100644 --- a/packages/web/src/collections/getCollectionComputedMetadata.ts +++ b/packages/web/src/collections/getCollectionComputedMetadata.ts @@ -19,16 +19,17 @@ export const getCollectionComputedMetadata = ( const list = item?.metadata ?? [] const orderedList = [...list].sort((a, b) => { - // mangadex has higher priority - if (a.type === "mangadex") return 1 - - // mangaupdates has priority - if (a.type === "mangaupdates" && b.type === "biblioreads") return 1 - - /** - * link is the raw format, we don't want it to be on top - */ - return a.type === "link" ? -1 : 1 + const priority: Record = { + user: 5, + mangadex: 4, + mangaupdates: 3, + biblioreads: 2, + link: 1, + googleBookApi: 0, + comicvine: -1 + } + + return (priority[a.type] || 0) - (priority[b.type] || 0) }) const reducedMetadata = orderedList.reduce((acc, { type, ...item }) => { diff --git a/packages/web/src/collections/useUpdateCollection.ts b/packages/web/src/collections/useUpdateCollection.ts index f7b76087..1fc19438 100644 --- a/packages/web/src/collections/useUpdateCollection.ts +++ b/packages/web/src/collections/useUpdateCollection.ts @@ -19,7 +19,9 @@ export const useUpdateCollection = () => { ...old, ...rest, metadata: old.metadata?.map((entry) => - entry.type === "user" ? { ...entry, title: name } : entry + entry.type === "user" + ? { ...entry, title: name ?? entry.title } + : entry ) })) } From e47fb4d367e999569daf81aa4eefdf2b4b8dfbd5 Mon Sep 17 00:00:00 2001 From: mbret Date: Wed, 18 Dec 2024 19:23:08 +0100 Subject: [PATCH 2/3] fix: fixed collection update --- .../handler.ts | 13 ++-- .../src/refreshMetadata.ts | 4 +- .../refreshMetadataLongProcess/handler.ts | 8 +- packages/api/src/libs/couch/dbHelpers.ts | 40 ++-------- packages/api/src/libs/couch/findOne.ts | 74 +++++++++++++++++++ packages/api/src/libs/plugins/helpers.ts | 12 ++- .../sync/collections/repairCollectionBooks.ts | 12 ++- 7 files changed, 111 insertions(+), 52 deletions(-) create mode 100644 packages/api/src/libs/couch/findOne.ts diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts index 85b30dfd..913c3481 100644 --- a/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts @@ -46,14 +46,15 @@ const lambda: ValidatedEventAPIGatewayProxyEvent = async ( ), switchMap(([params, db]) => from( - findOne(db, "obokucollection", { - selector: { _id: collectionId } - }) + findOne( + "obokucollection", + { + selector: { _id: collectionId } + }, + { throwOnNotFound: true, db } + ) ).pipe( mergeMap((collection) => { - if (!collection) - throw new Error(`Unable to find book ${collectionId}`) - return from( refreshMetadata(collection, { db, diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts index 413d63f2..6d364bf4 100644 --- a/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts @@ -119,9 +119,9 @@ export const refreshMetadata = async ( } // try to get latest collection to stay as fresh as possible - const currentCollection = await findOne(db, "obokucollection", { + const currentCollection = await findOne("obokucollection", { selector: { _id: collection._id } - }) + }, { db }) if (!currentCollection) throw new Error("Unable to find collection") diff --git a/packages/api/src/functions/refreshMetadataLongProcess/handler.ts b/packages/api/src/functions/refreshMetadataLongProcess/handler.ts index 59ae1618..a9f153f6 100644 --- a/packages/api/src/functions/refreshMetadataLongProcess/handler.ts +++ b/packages/api/src/functions/refreshMetadataLongProcess/handler.ts @@ -72,7 +72,7 @@ const lambda: ValidatedEventAPIGatewayProxyEvent = async ( const db = await getNanoDbForUser(userName, jwtPrivateKey) - const book = await findOne(db, "book", { selector: { _id: bookId } }) + const book = await findOne("book", { selector: { _id: bookId } }, { db }) if (!book) throw new Error(`Unable to find book ${bookId}`) @@ -85,7 +85,11 @@ const lambda: ValidatedEventAPIGatewayProxyEvent = async ( const firstLinkId = (book.links || [])[0] || "-1" - const link = await findOne(db, "link", { selector: { _id: firstLinkId } }) + const link = await findOne( + "link", + { selector: { _id: firstLinkId } }, + { db } + ) if (!link) throw new Error(`Unable to find link ${firstLinkId}`) diff --git a/packages/api/src/libs/couch/dbHelpers.ts b/packages/api/src/libs/couch/dbHelpers.ts index 6a94109a..0bde1356 100644 --- a/packages/api/src/libs/couch/dbHelpers.ts +++ b/packages/api/src/libs/couch/dbHelpers.ts @@ -13,6 +13,9 @@ import { User } from "../couchDbEntities" import { waitForRandomTime } from "../utils" import { COUCH_DB_URL } from "../../constants" import { generatePassword } from "../authentication/generatePassword" +import { findOne } from "./findOne" + +export { findOne } export const createUser = async ( db: createNano.ServerScope, @@ -104,39 +107,6 @@ export const insert = async < return doc } -export const findOne = async < - M extends DocType["rx_model"], - D extends ModelOf ->( - db: createNano.DocumentScope, - rxModel: M, - query: SafeMangoQuery -) => { - const { fields, ...restQuery } = query - const fieldsWithRequiredFields = fields - if (Array.isArray(fieldsWithRequiredFields)) { - fieldsWithRequiredFields.push(`rx_model`) - } - - const response = await retryFn(() => - db.find({ - ...restQuery, - fields: fields as string[], - selector: { rx_model: rxModel, ...(query?.selector as any) }, - limit: 1 - }) - ) - - if (response.docs.length === 0) return null - - const doc = response - .docs[0] as createNano.MangoResponse["docs"][number] & D - - if (rxModel !== doc.rx_model) throw new Error(`Invalid document type`) - - return doc -} - export const findAllDataSources = async ( db: createNano.DocumentScope ) => { @@ -246,7 +216,7 @@ export const getOrCreateTagFromName = ( ) => { return retryFn(async () => { // Get all tag ids and create one if it does not exist - const existingTag = await findOne(db, "tag", { selector: { name } }) + const existingTag = await findOne("tag", { selector: { name } }, { db }) if (existingTag) { return existingTag._id } @@ -276,7 +246,7 @@ export const createTagFromName = ( silent: boolean ) => { return retryFn(async () => { - const existingTag = await findOne(db, "tag", { selector: { name } }) + const existingTag = await findOne("tag", { selector: { name } }, { db }) if (existingTag) { if (silent) { diff --git a/packages/api/src/libs/couch/findOne.ts b/packages/api/src/libs/couch/findOne.ts new file mode 100644 index 00000000..e5058511 --- /dev/null +++ b/packages/api/src/libs/couch/findOne.ts @@ -0,0 +1,74 @@ +import createNano from "nano" +import { SafeMangoQuery, DocType, ModelOf } from "@oboku/shared" +import { retryFn } from "./dbHelpers" + +type FindOneOptionsBase = { + db: createNano.DocumentScope +} + +type FindOneOptionsWithThrow = FindOneOptionsBase & { + throwOnNotFound: true +} + +type FindOneOptionsWithoutThrow = FindOneOptionsBase & { + throwOnNotFound?: false +} + +type FindOneOptions = FindOneOptionsWithThrow | FindOneOptionsWithoutThrow + +export function findOne>( + rxModel: M, + query: SafeMangoQuery, + options: FindOneOptionsWithThrow +): Promise< + D & { + _id: string + _rev: string + } +> + +export function findOne>( + rxModel: M, + query: SafeMangoQuery, + options: FindOneOptionsWithoutThrow +): Promise< + | (D & { + _id: string + _rev: string + }) + | null +> + +export async function findOne< + M extends DocType["rx_model"], + D extends ModelOf +>(rxModel: M, query: SafeMangoQuery, options: FindOneOptions) { + const { fields, ...restQuery } = query + const fieldsWithRequiredFields = fields + if (Array.isArray(fieldsWithRequiredFields)) { + fieldsWithRequiredFields.push(`rx_model`) + } + + const response = await retryFn(() => + options.db.find({ + ...restQuery, + fields: fields as string[], + selector: { rx_model: rxModel, ...(query?.selector as any) }, + limit: 1 + }) + ) + + if (response.docs.length === 0) { + if (options.throwOnNotFound) { + throw new Error("Document not found") + } + return null + } + + const doc = response + .docs[0] as createNano.MangoResponse["docs"][number] & D + + if (rxModel !== doc.rx_model) throw new Error(`Invalid document type`) + + return doc +} diff --git a/packages/api/src/libs/plugins/helpers.ts b/packages/api/src/libs/plugins/helpers.ts index 97b877d0..580b660a 100644 --- a/packages/api/src/libs/plugins/helpers.ts +++ b/packages/api/src/libs/plugins/helpers.ts @@ -23,9 +23,13 @@ export const createHelpers = ( refreshBookMetadata: (opts: { bookId: string }) => refreshBookMetadata(opts).catch(console.error), getDataSourceData: async (): Promise> => { - const dataSource = await findOne(db, "datasource", { - selector: { _id: dataSourceId } - }) + const dataSource = await findOne( + "datasource", + { + selector: { _id: dataSourceId } + }, + { db } + ) let data = {} try { if (dataSource?.data) { @@ -40,7 +44,7 @@ export const createHelpers = ( findOne: >( model: M, query: SafeMangoQuery - ) => findOne(db, model, query), + ) => findOne(model, query, { db }), find: >( model: M, query: SafeMangoQuery diff --git a/packages/api/src/libs/sync/collections/repairCollectionBooks.ts b/packages/api/src/libs/sync/collections/repairCollectionBooks.ts index 220d5344..70c86c1a 100644 --- a/packages/api/src/libs/sync/collections/repairCollectionBooks.ts +++ b/packages/api/src/libs/sync/collections/repairCollectionBooks.ts @@ -29,9 +29,15 @@ export const repairCollectionBooks = async ({ ctx: Context collectionId: string }) => { - const collection = await findOne(ctx.db, "obokucollection", { - selector: { _id: collectionId } - }) + const collection = await findOne( + "obokucollection", + { + selector: { _id: collectionId } + }, + { + db: ctx.db + } + ) if (collection) { const [booksHavingCollectionAttached, booksFromCollectionList] = From e9ecec9b81bd8694fef5fb0f201a640d0baf83b1 Mon Sep 17 00:00:00 2001 From: mbret Date: Wed, 18 Dec 2024 19:57:11 +0100 Subject: [PATCH 3/3] feat: added first book cover on collection --- packages/web/src/books/Cover.tsx | 21 +++++----- packages/web/src/books/useBookCover.ts | 39 +++++++++++++++++++ .../details/CollectionDetailsScreen.tsx | 6 +-- .../src/collections/useCollectionCoverUri.ts | 16 +++++++- 4 files changed, 67 insertions(+), 15 deletions(-) create mode 100644 packages/web/src/books/useBookCover.ts diff --git a/packages/web/src/books/Cover.tsx b/packages/web/src/books/Cover.tsx index c5d96ebf..dfc3568c 100644 --- a/packages/web/src/books/Cover.tsx +++ b/packages/web/src/books/Cover.tsx @@ -8,6 +8,7 @@ import { API_URL } from "../constants.web" import { useLocalSettings } from "../settings/states" import { useSignalValue } from "reactjrx" import { authStateSignal } from "../auth/authState" +import { useBookCover } from "./useBookCover" const useBookCoverState = ({ bookId }: { bookId: string }) => { const blurredTags = useBlurredTagIds().data ?? [] @@ -94,13 +95,7 @@ export const Cover: FC = memo( urlParams.append("format", "image/jpeg") - const originalJpgSrc = book - ? `${API_URL}/covers/cover-${auth?.nameHex}-${book._id}?${urlParams.toString()}` - : undefined - - const coverSrc = originalSrc && !hasError ? originalSrc : placeholder - const coverSrcJpg = - originalJpgSrc && !hasError ? originalJpgSrc : placeholder + const { coverSrc, coverSrcJpg } = useBookCover({ bookId }) useEffect(() => { setHasError(false) @@ -135,11 +130,17 @@ export const Cover: FC = memo( }) }} > - - + + { + const auth = useSignalValue(authStateSignal) + const { data: book } = useBook({ id: bookId }) + const assetHash = book?.lastMetadataUpdatedAt?.toString() + + const urlParams = new URLSearchParams({ + ...(assetHash && { + hash: assetHash + }) + }) + + const originalSrc = bookId + ? `${API_URL}/covers/cover-${auth?.nameHex}-${bookId}?${urlParams.toString()}` + : undefined + + urlParams.append("format", "image/jpeg") + + const originalJpgSrc = bookId + ? `${API_URL}/covers/cover-${auth?.nameHex}-${bookId}?${urlParams.toString()}` + : undefined + + const coverSrc = originalSrc + const coverSrcJpg = originalJpgSrc + + const hasCoverMetadata = !!book?.metadata?.find( + (metadata) => metadata.coverLink + ) + + return { + coverSrc, + coverSrcJpg, + hasCoverMetadata + } +} diff --git a/packages/web/src/collections/details/CollectionDetailsScreen.tsx b/packages/web/src/collections/details/CollectionDetailsScreen.tsx index bc4b9ad6..baf60bd9 100644 --- a/packages/web/src/collections/details/CollectionDetailsScreen.tsx +++ b/packages/web/src/collections/details/CollectionDetailsScreen.tsx @@ -18,7 +18,7 @@ import { useLocalSettings } from "../../settings/states" import { Report } from "../../debug/report.shared" import { useCollectionComputedMetadata } from "../useCollectionComputedMetadata" import { useCollectionCoverUri } from "../useCollectionCoverUri" -import placeholder from "../../assets/cover-placeholder.png" +import coverPlaceholder from "../../assets/cover-placeholder.png" import { StatusChip } from "../series/StatusChip" import { useWindowScroll } from "react-use" @@ -137,7 +137,7 @@ export const CollectionDetailsScreen = () => { style={{ backgroundImage: useOptimizedTheme ? undefined - : `url(${hasCover ? (coverUri ?? placeholder) : CollectionBgSvg})`, + : `url(${hasCover ? (coverUri ?? coverPlaceholder) : CollectionBgSvg})`, backgroundRepeat: "no-repeat", backgroundSize: "cover", backgroundPosition: "center" @@ -161,7 +161,7 @@ export const CollectionDetailsScreen = () => { | null ) => { const assetHash = collection?.lastMetadataUpdatedAt?.toString() - + const { coverSrc: firstBookCoverSrc, hasCoverMetadata } = useBookCover({ + bookId: collection?.books[0] + }) const urlParams = new URLSearchParams({ ...(assetHash && { hash: assetHash @@ -17,7 +20,16 @@ export const useCollectionCoverUri = ( const hasCover = !!collection.metadata?.find((metadata) => metadata.cover) - if (!hasCover) return { uri: undefined, hasCover: false } + if (!hasCover) { + if (firstBookCoverSrc && hasCoverMetadata) { + return { + uri: firstBookCoverSrc, + hasCover: true + } + } + + return { uri: undefined, hasCover: false } + } return { uri: `${API_URL}/covers/${getCollectionCoverKey(collection?._id)}?${urlParams.toString()}`,