From 8fdbeb8279df9d45c34ea2c657478720a69a8d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aaron=20D=C3=B6tsch?= <42084688+aarondoet@users.noreply.github.com> Date: Wed, 1 Jan 2025 17:05:49 +0000 Subject: [PATCH 1/4] Generate zip file on the fly --- backend/src/file/file.controller.ts | 2 +- backend/src/file/file.service.ts | 26 ++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/backend/src/file/file.controller.ts b/backend/src/file/file.controller.ts index feb036dd3..e7d4e96b1 100644 --- a/backend/src/file/file.controller.ts +++ b/backend/src/file/file.controller.ts @@ -54,7 +54,7 @@ export class FileController { @Res({ passthrough: true }) res: Response, @Param("shareId") shareId: string, ) { - const zipStream = this.fileService.getZip(shareId); + const zipStream = await this.fileService.getZip(shareId); res.set({ "Content-Type": "application/zip", diff --git a/backend/src/file/file.service.ts b/backend/src/file/file.service.ts index e5de341fe..a079a4e58 100644 --- a/backend/src/file/file.service.ts +++ b/backend/src/file/file.service.ts @@ -2,8 +2,11 @@ import { Injectable } from "@nestjs/common"; import { LocalFileService } from "./local.service"; import { S3FileService } from "./s3.service"; import { ConfigService } from "src/config/config.service"; -import { Readable } from "stream"; +import { PassThrough, Readable } from "stream"; import { PrismaService } from "../prisma/prisma.service"; +import * as archiver from "archiver"; +import * as fs from "fs"; +import { SHARE_DIRECTORY } from "src/constants"; @Injectable() export class FileService { @@ -59,9 +62,24 @@ export class FileService { return storageService.deleteAllFiles(shareId); } - getZip(shareId: string) { - const storageService = this.getStorageService(); - return storageService.getZip(shareId) as Readable; + async getZip(shareId: string) { + const passThrough = new PassThrough(); + const path = `${SHARE_DIRECTORY}/${shareId}`; + + const files = await this.prisma.file.findMany({ where: { shareId } }); + const archive = archiver("zip", { + zlib: { level: this.configService.get("share.zipCompressionLevel") }, + }); + + for (const file of files) { + archive.append(fs.createReadStream(`${path}/${file.id}`), { + name: file.name, + }); + } + + archive.pipe(passThrough); + archive.finalize(); + return passThrough as Readable; } private async streamToUint8Array(stream: Readable): Promise { From b3e3914c39f61dfabbdc486986e466a79ec68290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aaron=20D=C3=B6tsch?= <42084688+aarondoet@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:22:24 +0000 Subject: [PATCH 2/4] refactor: remove isZipReady field from Share model and related components --- .../migration.sql | 29 ++++++++++++++++ backend/prisma/schema.prisma | 1 - backend/src/share/dto/shareMetaData.dto.ts | 3 -- backend/src/share/share.service.ts | 8 +---- .../components/share/DownloadAllButton.tsx | 34 ++----------------- frontend/src/types/share.type.ts | 1 - 6 files changed, 32 insertions(+), 44 deletions(-) create mode 100644 backend/prisma/migrations/20250108150730_remove_is_zip_ready/migration.sql diff --git a/backend/prisma/migrations/20250108150730_remove_is_zip_ready/migration.sql b/backend/prisma/migrations/20250108150730_remove_is_zip_ready/migration.sql new file mode 100644 index 000000000..027344f1f --- /dev/null +++ b/backend/prisma/migrations/20250108150730_remove_is_zip_ready/migration.sql @@ -0,0 +1,29 @@ +/* + Warnings: + + - You are about to drop the column `isZipReady` on the `Share` table. All the data in the column will be lost. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Share" ( + "id" TEXT NOT NULL PRIMARY KEY, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "name" TEXT, + "uploadLocked" BOOLEAN NOT NULL DEFAULT false, + "views" INTEGER NOT NULL DEFAULT 0, + "expiration" DATETIME NOT NULL, + "description" TEXT, + "removedReason" TEXT, + "creatorId" TEXT, + "reverseShareId" TEXT, + "storageProvider" TEXT NOT NULL DEFAULT 'LOCAL', + CONSTRAINT "Share_creatorId_fkey" FOREIGN KEY ("creatorId") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "Share_reverseShareId_fkey" FOREIGN KEY ("reverseShareId") REFERENCES "ReverseShare" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); +INSERT INTO "new_Share" ("createdAt", "creatorId", "description", "expiration", "id", "name", "removedReason", "reverseShareId", "storageProvider", "uploadLocked", "views") SELECT "createdAt", "creatorId", "description", "expiration", "id", "name", "removedReason", "reverseShareId", "storageProvider", "uploadLocked", "views" FROM "Share"; +DROP TABLE "Share"; +ALTER TABLE "new_Share" RENAME TO "Share"; +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index b473283e3..def4cb6bd 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -80,7 +80,6 @@ model Share { name String? uploadLocked Boolean @default(false) - isZipReady Boolean @default(false) views Int @default(0) expiration DateTime description String? diff --git a/backend/src/share/dto/shareMetaData.dto.ts b/backend/src/share/dto/shareMetaData.dto.ts index 0733fefa1..aac4b1b2b 100644 --- a/backend/src/share/dto/shareMetaData.dto.ts +++ b/backend/src/share/dto/shareMetaData.dto.ts @@ -4,9 +4,6 @@ export class ShareMetaDataDTO { @Expose() id: string; - @Expose() - isZipReady: boolean; - from(partial: Partial) { return plainToClass(ShareMetaDataDTO, partial, { excludeExtraneousValues: true, diff --git a/backend/src/share/share.service.ts b/backend/src/share/share.service.ts index f13d8fade..b6d188eef 100644 --- a/backend/src/share/share.service.ts +++ b/backend/src/share/share.service.ts @@ -147,12 +147,6 @@ export class ShareService { "You need at least on file in your share to complete it.", ); - // Asynchronously create a zip of all files - if (share.files.length > 1) - this.createZip(id).then(() => - this.prisma.share.update({ where: { id }, data: { isZipReady: true } }), - ); - // Send email for each recipient for (const recipient of share.recipients) { await this.emailService.sendMailToShareRecipients( @@ -200,7 +194,7 @@ export class ShareService { async revertComplete(id: string) { return this.prisma.share.update({ where: { id }, - data: { uploadLocked: false, isZipReady: false }, + data: { uploadLocked: false }, }); } diff --git a/frontend/src/components/share/DownloadAllButton.tsx b/frontend/src/components/share/DownloadAllButton.tsx index f5e581a62..39a52025e 100644 --- a/frontend/src/components/share/DownloadAllButton.tsx +++ b/frontend/src/components/share/DownloadAllButton.tsx @@ -1,14 +1,10 @@ import { Button } from "@mantine/core"; -import { useEffect, useState } from "react"; +import { useState } from "react"; import { FormattedMessage } from "react-intl"; -import useTranslate from "../../hooks/useTranslate.hook"; import shareService from "../../services/share.service"; -import toast from "../../utils/toast.util"; const DownloadAllButton = ({ shareId }: { shareId: string }) => { - const [isZipReady, setIsZipReady] = useState(false); const [isLoading, setIsLoading] = useState(false); - const t = useTranslate(); const downloadAll = async () => { setIsLoading(true); @@ -17,37 +13,11 @@ const DownloadAllButton = ({ shareId }: { shareId: string }) => { .then(() => setIsLoading(false)); }; - useEffect(() => { - shareService - .getMetaData(shareId) - .then((share) => setIsZipReady(share.isZipReady)) - .catch(() => {}); - - const timer = setInterval(() => { - shareService - .getMetaData(shareId) - .then((share) => { - setIsZipReady(share.isZipReady); - if (share.isZipReady) clearInterval(timer); - }) - .catch(() => clearInterval(timer)); - }, 5000); - return () => { - clearInterval(timer); - }; - }, []); - return ( diff --git a/frontend/src/types/share.type.ts b/frontend/src/types/share.type.ts index abecf13ae..ba40a9442 100644 --- a/frontend/src/types/share.type.ts +++ b/frontend/src/types/share.type.ts @@ -31,7 +31,6 @@ export type CreateShare = { export type ShareMetaData = { id: string; - isZipReady: boolean; }; export type MyShare = Omit & { From 2855358632a1da21d1343e4b74939bca6de31432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aaron=20D=C3=B6tsch?= <42084688+aarondoet@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:24:31 +0000 Subject: [PATCH 3/4] Always render DownloadAllButton in Share component Also render button when there is only one file. I prefer this for the sake of consistency, this could be removed again because it basically is unnecessary. --- frontend/src/pages/share/[shareId]/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/share/[shareId]/index.tsx b/frontend/src/pages/share/[shareId]/index.tsx index 52d014ddc..229478a1c 100644 --- a/frontend/src/pages/share/[shareId]/index.tsx +++ b/frontend/src/pages/share/[shareId]/index.tsx @@ -108,7 +108,7 @@ const Share = ({ shareId }: { shareId: string }) => { {share?.name || share?.id} {share?.description} - {share?.files.length > 1 && } + Date: Wed, 8 Jan 2025 15:36:16 +0000 Subject: [PATCH 4/4] Remove download preparation notifications from translation files --- frontend/src/i18n/translations/de-DE.ts | 1 - frontend/src/i18n/translations/en-US.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/frontend/src/i18n/translations/de-DE.ts b/frontend/src/i18n/translations/de-DE.ts index 8607a8325..5c09f6426 100644 --- a/frontend/src/i18n/translations/de-DE.ts +++ b/frontend/src/i18n/translations/de-DE.ts @@ -282,7 +282,6 @@ export default { "share.modal.password": "Passwort", "share.modal.error.invalid-password": "Ungültiges Passwort", "share.button.download-all": "Alles herunterladen", - "share.notify.download-all-preparing": "Die Freigabe wird vorbereitet. Bitte versuche es in ein paar Minuten erneut.", "share.modal.file-link": "Dateilink", "share.table.name": "Name", "share.table.size": "Größe", diff --git a/frontend/src/i18n/translations/en-US.ts b/frontend/src/i18n/translations/en-US.ts index 1b3a968d5..bceae039a 100644 --- a/frontend/src/i18n/translations/en-US.ts +++ b/frontend/src/i18n/translations/en-US.ts @@ -384,8 +384,6 @@ export default { "share.modal.error.invalid-password": "Invalid password", "share.button.download-all": "Download all", - "share.notify.download-all-preparing": - "The share is being prepared. Please try again in a few minutes.", "share.modal.file-link": "File link", "share.table.name": "Name",