From 8251edd0cae9d6b52aa060d0be99a5c5ebd12df4 Mon Sep 17 00:00:00 2001 From: Ioan Lucut Date: Tue, 8 Aug 2023 21:33:43 +0300 Subject: [PATCH] Add a partial deployment capability --- .env | 1 + .env.remote | 3 +- .run/Run migrator.run.xml | 5 - .run/[Local] Run migrator.run.xml | 5 + README.md | 3 +- package.json | 2 +- runner.ts | 3 +- src/constants.ts | 2 + src/core.spec.ts | 41 ++++++ src/core.ts | 42 ++++-- src/index.ts | 1 - src/inventorySnapshotRunner.ts | 141 ------------------ src/manifestGenerator.ts | 26 ++++ src/migratorRunner.ts | 228 +++++++++++++++++++++++++++--- src/songsParser.spec.ts | 6 + src/songsParser.ts | 11 +- src/types.ts | 22 ++- 17 files changed, 341 insertions(+), 201 deletions(-) delete mode 100644 .run/Run migrator.run.xml create mode 100644 .run/[Local] Run migrator.run.xml delete mode 100644 src/inventorySnapshotRunner.ts create mode 100644 src/manifestGenerator.ts diff --git a/.env b/.env index 7bffd96..4aacdf1 100644 --- a/.env +++ b/.env @@ -1,2 +1,3 @@ +TZ=Europe/Bucharest SOURCE_DIR=../bes-lyrics/verified OUT_DIR=./pp7-songs diff --git a/.env.remote b/.env.remote index 29b1234..40c912e 100644 --- a/.env.remote +++ b/.env.remote @@ -1,2 +1,3 @@ +TZ=Europe/Bucharest SOURCE_DIR=../bes-lyrics/verified -OUT_DIR=/Users/ilucut/WORK/BES/CLOUD DATA/ProPresenter_Generated/ProPresenter_Generated_Version_10 +OUT_DIR=/Users/ilucut/WORK/BES/CLOUD DATA/ProPresenter_Generated diff --git a/.run/Run migrator.run.xml b/.run/Run migrator.run.xml deleted file mode 100644 index df0b636..0000000 --- a/.run/Run migrator.run.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.run/[Local] Run migrator.run.xml b/.run/[Local] Run migrator.run.xml new file mode 100644 index 0000000..145f37e --- /dev/null +++ b/.run/[Local] Run migrator.run.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/README.md b/README.md index d0f472e..3dc6d3a 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,7 @@ const CONFIG = { migrateSongsToPP7Format({ sourceDir: process.env.SOURCE_DIR as string, - outDir: process.env.OUT_DIR as string, - clearOutputDirFirst: true, + baseOutDir: process.env.OUT_DIR as string, config: CONFIG, }); ``` diff --git a/package.json b/package.json index 4729a47..3c8c729 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.ts", "scripts": { "test:ci": "NODE_ENV=test jest --runInBand --no-cache", - "test:watch": "NODE_ENV=test TZ='Europe/Berlin' jest --watch --logHeapUsage", + "test:watch": "NODE_ENV=test TZ='Europe/Bucharest' jest --watch --logHeapUsage", "test": "is-ci-cli test:ci test:watch", "migrate:local": "ts-node runner.ts", "migrate:remote": "dotenv -e .env.remote ts-node runner.ts" diff --git a/runner.ts b/runner.ts index 715349b..f72bdac 100644 --- a/runner.ts +++ b/runner.ts @@ -32,8 +32,7 @@ const CONFIG = { (async () => { await migrateSongsToPP7Format({ sourceDir: process.env.SOURCE_DIR as string, - outDir: process.env.OUT_DIR as string, - clearOutputDirFirst: true, + baseOutDir: process.env.OUT_DIR as string, config: CONFIG, }); })(); diff --git a/src/constants.ts b/src/constants.ts index 788a6c8..033dd41 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -12,6 +12,8 @@ export const SLASH = '/'; export const HASH = '#'; +export const DASH = '-'; + export const TXT_EXTENSION = '.txt'; export const TEST_FILE = 'TEMPLATE.txt'; diff --git a/src/core.spec.ts b/src/core.spec.ts index 7c2f7a3..8a8da48 100644 --- a/src/core.spec.ts +++ b/src/core.spec.ts @@ -1,10 +1,17 @@ import { + parseDateFromVersionedDir, getMetaSectionsFromTitle, getSongInSectionTuples, getTitleWithoutMeta, + getVersionedDir, + getClosestVersionedDir, } from './core'; import { SIMPLE_SONG_MOCK_FILE_CONTENT } from '../mocks'; +jest + .useFakeTimers() + .setSystemTime(new Date('2023-02-19T11:12:13.000Z').getTime()); + describe('core', () => { describe('getSongInSectionTuples', () => { it('should work correctly', () => { @@ -74,4 +81,38 @@ describe('core', () => { `); }); }); + + describe('getClosestVersionedDir', () => { + it('should work correctly', () => { + expect(getVersionedDir()).toMatchInlineSnapshot(`"2023-02-19-12:12:13"`); + }); + }); + + describe('parseDateFromVersionedDir', () => { + it('should work correctly', () => { + expect( + parseDateFromVersionedDir('2023-02-19-11:12:13'), + ).toMatchInlineSnapshot(`2023-02-19T10:12:13.000Z`); + + expect( + parseDateFromVersionedDir('2023-08-08-19:17:25'), + ).toMatchInlineSnapshot(`2023-08-08T17:17:25.000Z`); + }); + }); + + describe('getClosestVersionedDir', () => { + it('should work correctly', () => { + expect( + getClosestVersionedDir( + parseDateFromVersionedDir('2023-26-20-11:12:13'), + [ + parseDateFromVersionedDir('2023-02-20-11:12:13'), + parseDateFromVersionedDir('2023-02-19-11:12:13'), + parseDateFromVersionedDir('2023-02-23-11:12:13'), + parseDateFromVersionedDir('2023-02-24-11:12:13'), + ], + ), + ).toMatchInlineSnapshot(`2023-02-24T10:12:13.000Z`); + }); + }); }); diff --git a/src/core.ts b/src/core.ts index 2df324f..5e4ffde 100644 --- a/src/core.ts +++ b/src/core.ts @@ -13,6 +13,7 @@ import { SequenceChar, SongMeta } from './types'; import { COLON, COMMA, + DASH, DOUBLE_LINE_TUPLE, EMPTY_STRING, HASH, @@ -130,18 +131,41 @@ ${tuples export const convertSequenceToNumber = (sequenceOrderQualifier: string) => parseInt(sequenceOrderQualifier) || MISSING_SEQUENCE_NUMBER; -export const getTodayAsDateString = () => { - const now = new Date(); - - return ( - ('0' + now.getDate()).slice(-2) + - '-' + - ('0' + (now.getMonth() + 1)).slice(-2) + - '-' + - now.getFullYear() +export const getVersionedDir = (now = new Date()) => + now.getFullYear() + + DASH + + ('0' + (now.getMonth() + 1)).slice(-2) + + DASH + + ('0' + now.getDate()).slice(-2) + + DASH + + ('0' + now.getHours()).slice(-2) + + COLON + + ('0' + now.getMinutes()).slice(-2) + + COLON + + ('0' + now.getSeconds()).slice(-2); + +export const parseDateFromVersionedDir = (versionFolder: string) => { + const [year, month, day, time] = versionFolder.split(DASH); + const [hour, minute, second] = time.split(COLON); + + return new Date( + parseInt(year), + parseInt(month) - 1, + parseInt(day), + parseInt(hour), + parseInt(minute), + parseInt(second), ); }; +export const getClosestVersionedDir = (diffDate: Date, dates: Date[]) => + first( + dates.sort((a, b) => { + // @ts-ignore + return Math.abs(diffDate - a) - Math.abs(diffDate - b); // sort a before b when the distance is smaller + }), + ); + export const assertUniqueness = (array: string[]) => assert.equal( size(uniq(array)), diff --git a/src/index.ts b/src/index.ts index b137404..f979d07 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ -export * from './inventorySnapshotRunner'; export * from './migratorRunner'; export * from './proPresenter7SongConverter'; export * from './songsParser'; diff --git a/src/inventorySnapshotRunner.ts b/src/inventorySnapshotRunner.ts deleted file mode 100644 index a76b0a1..0000000 --- a/src/inventorySnapshotRunner.ts +++ /dev/null @@ -1,141 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import dotenv from 'dotenv'; -import recursive from 'recursive-readdir'; -import { isEqual, last, reject } from 'lodash'; -import { - assertUniqueness, - getMetaSectionsFromTitle, - getSongInSectionTuples, - getTitleWithoutMeta, - getTodayAsDateString, - logFileWithLinkInConsole, - logProcessingFile, -} from './core'; -import assert from 'node:assert'; -import { - SongInventoryChange, - SongMeta, - SongsInventory, - SongsInventoryEntry, -} from './types'; -import { TXT_EXTENSION } from './constants'; - -dotenv.config(); - -const inventoryChangelogFilename = path.join( - __dirname, - '..', - 'inventoryChangelog.json', -); - -const snapshotInventory = async (dir: string) => { - console.log(`"Reprocessing file names from ${dir} directory.."`); - - const currentInventory = JSON.parse( - fs.readFileSync(inventoryChangelogFilename).toString(), - ) as SongsInventory; - - const { songs } = currentInventory; - - const songsAsHashMap = songs.reduce( - (accumulator, entry) => ({ - ...accumulator, - [entry.id]: entry, - }), - {} as { - [id: string]: SongsInventoryEntry; - }, - ); - - const songsMeta = (await recursive(dir)) - .filter((filePath) => path.extname(filePath) === TXT_EXTENSION) - .map((filePath) => { - const filename = path.basename(filePath); - const fileContent = fs.readFileSync(filePath).toString(); - - logProcessingFile(filename, 'inventory generation'); - logFileWithLinkInConsole(filePath); - - const rawTitle = getSongInSectionTuples(fileContent)[1]; - - assert.ok( - rawTitle.includes(SongMeta.CONTENT_HASH), - `The ${SongMeta.CONTENT_HASH} should be defined.`, - ); - - return { - filename, - metaSections: getMetaSectionsFromTitle(rawTitle), - titleWithoutMetaSections: getTitleWithoutMeta(rawTitle), - }; - }); - - assertUniqueness( - songsMeta.map(({ metaSections }) => metaSections[SongMeta.ID]), - ); - - const dateAsString = getTodayAsDateString(); - const updatedSongsInventory = songsMeta.map(({ filename, metaSections }) => { - const id = metaSections[SongMeta.ID]; - const contentHash = metaSections[SongMeta.CONTENT_HASH]; - const currentInventory = songsAsHashMap[id]; - - if (!currentInventory) { - return { - id, - filename, - changes: [ - { - date: dateAsString, - contentHash, - }, - ], - } as SongsInventoryEntry; - } - - const { changes } = currentInventory; - const { - filename: lastChangeFilename, - contentHash: lastChangedContentHash, - date: lastChangedDate, - } = last(changes) as SongInventoryChange; - - if (isEqual(lastChangedContentHash, contentHash)) { - return currentInventory; - } - - const isFilenameChanged = !isEqual(lastChangeFilename, filename); - return { - id, - filename, - changes: [ - ...(isEqual(lastChangedDate, dateAsString) - ? reject(changes, { date: lastChangedDate }) - : changes), - { - date: dateAsString, - contentHash, - ...(isFilenameChanged ? { filename, isFilenameChanged } : {}), - }, - ], - } as SongsInventoryEntry; - }); - - assertUniqueness(Object.keys(updatedSongsInventory).map((id) => id)); - - const nextSongs = updatedSongsInventory.sort((a, b) => - a.filename.localeCompare(b.filename), - ); - fs.writeFileSync( - inventoryChangelogFilename, - JSON.stringify( - isEqual(songs, nextSongs) - ? currentInventory - : { - updatedOn: dateAsString, - songs: nextSongs, - }, - ), - ); -}; diff --git a/src/manifestGenerator.ts b/src/manifestGenerator.ts new file mode 100644 index 0000000..a8949cf --- /dev/null +++ b/src/manifestGenerator.ts @@ -0,0 +1,26 @@ +import dotenv from 'dotenv'; +import { assertUniqueness } from './core'; +import { Song, SongManifest } from './types'; + +dotenv.config(); + +export const generateManifest = async ( + deployableSongs: Array<{ + song: Song; + fileName: string; + fileAsText: string; + }>, +) => { + assertUniqueness(deployableSongs.map(({ song }) => song.id)); + + return deployableSongs + .map( + ({ song: { id, contentHash }, fileName }) => + ({ + id, + fileName, + contentHash, + } as SongManifest), + ) + .sort((a, b) => a.fileName.localeCompare(b.fileName)); +}; diff --git a/src/migratorRunner.ts b/src/migratorRunner.ts index 4d99958..7e0b04e 100644 --- a/src/migratorRunner.ts +++ b/src/migratorRunner.ts @@ -8,47 +8,229 @@ import { Config, convertSongToProPresenter7, } from './proPresenter7SongConverter'; +import { generateManifest } from './manifestGenerator'; import { EMPTY_STRING, PRO_EXTENSION, TXT_EXTENSION } from './constants'; +import { + getClosestVersionedDir, + parseDateFromVersionedDir, + getVersionedDir, + logFileWithLinkInConsole, + logProcessingFile, +} from './core'; +import { DeployableSong, SongManifest, SongsInventoryManifest } from './types'; +import assert from 'node:assert'; +import { isEmpty, isEqual } from 'lodash'; + +const MANIFEST_FILE = 'manifest.json'; + +// --- +// Generate the songs from the given array to the new deployment directory +const generateAndWriteSongsToDeploymentDir = ( + songsToGenerate: DeployableSong[], + dir: string, + config: Config, +) => { + songsToGenerate.forEach(({ fileName, song }) => { + const songInProPresenter7Format = convertSongToProPresenter7(song, config); + + const songFileName = fileName.replace(TXT_EXTENSION, EMPTY_STRING); + const pp7ExportedFile = path.join(dir, `${songFileName}${PRO_EXTENSION}`); + + fs.writeFileSync( + pp7ExportedFile, + Buffer.from(Presentation.encode(songInProPresenter7Format).finish()), + ); + + console.log( + `Successfully generated "${pp7ExportedFile}" in the "${dir}" directory.`, + ); + }); +}; + +const getPreviousInventoryManifest = ( + deploymentDate: Date, + allPreviousDeploymentDirs: { + deploymentDirDate: Date; + deploymentDir: string; + }[], + baseOutDir: string, +) => { + const previousDeploymentDirDate = getClosestVersionedDir( + deploymentDate, + allPreviousDeploymentDirs.map(({ deploymentDirDate }) => deploymentDirDate), + ); + + const previousDeploymentManifestFilePath = path.join( + baseOutDir, + getVersionedDir(previousDeploymentDirDate), + MANIFEST_FILE, + ); + + console.log( + `Previous deployment manifest file path: ${previousDeploymentManifestFilePath}.`, + ); + + const previousInventoryManifest = JSON.parse( + fs.readFileSync(previousDeploymentManifestFilePath).toString(), + ) as SongsInventoryManifest; + + logProcessingFile( + previousDeploymentManifestFilePath, + 'Previous manifest inspection', + ); + logFileWithLinkInConsole(previousDeploymentManifestFilePath); + + return previousInventoryManifest; +}; + +const getManifestHashMap = (songManifestArray: SongManifest[]) => + songManifestArray.reduce( + (accumulator, entry) => ({ + ...accumulator, + [entry.id]: entry, + }), + {} as { + [id: string]: SongManifest; + }, + ); + +const getSongDiffs = ( + currentManifest: SongsInventoryManifest, + previousManifest: SongsInventoryManifest, +) => { + const previousManifestHashMap = getManifestHashMap( + previousManifest.inventory, + ); + const currentManifestHashMap = getManifestHashMap(currentManifest.inventory); + + const newOrUpdatedSongs = currentManifest.inventory.filter( + ({ id, fileName, contentHash }) => + // Is new song + !previousManifestHashMap[id] || + // Is an existing-updated song + !isEqual(previousManifestHashMap[id]?.contentHash, contentHash), + ); + + const toBeRemovedFileNames = previousManifest.inventory + .filter( + ({ id, fileName, contentHash }) => + // File-name has changed + !isEqual(currentManifestHashMap[id]?.fileName, fileName), + ) + .map(({ id }) => previousManifestHashMap[id]?.fileName); + + return { + newOrUpdatedSongs, + toBeRemovedFileNames, + }; +}; /** * Removes all the files from the out directory */ export const migrateSongsToPP7Format = async ({ sourceDir, - outDir, - clearOutputDirFirst, + baseOutDir, config, }: { sourceDir: string; - outDir: string; - clearOutputDirFirst?: boolean; + baseOutDir: string; config: Config; }) => { - if (clearOutputDirFirst) { - fsExtra.emptydirSync(outDir); - } + const versionedDir = getVersionedDir(); + const deploymentDate = parseDateFromVersionedDir(versionedDir); + const deploymentVersionedDir = `${baseOutDir}/${versionedDir}`; - await fsExtra.ensureDirSync(outDir); + assert( + !fsExtra.pathExistsSync(deploymentVersionedDir), + `The out directory "${deploymentVersionedDir}" does exists already.`, + ); - (await recursive(sourceDir, ['.DS_Store'])).forEach((filePath) => { - const fileAsText = fs.readFileSync(filePath).toString(); - const fileName = path.basename(filePath); + // --- + // Existing deployments + const allPreviousDeploymentDirs = fsExtra + .readdirSync(baseOutDir) + .map((deploymentDir) => ({ + deploymentDir, + deploymentDirDate: parseDateFromVersionedDir(deploymentDir), + })); - console.log(`Processing "${filePath}"...`); + // --- + // Current deployment + const deployableSongs = (await recursive(sourceDir, ['.DS_Store'])).map( + (filePath) => { + const fileAsText = fs.readFileSync(filePath).toString(); + const fileName = path.basename(filePath); + const song = parseSong(fileAsText); - const song = parseSong(fileAsText); - const presentation = convertSongToProPresenter7(song, config); + return { song, fileName, fileAsText } as DeployableSong; + }, + ); - const outFile = `${outDir}/${fileName.replace( - TXT_EXTENSION, - EMPTY_STRING, - )}${PRO_EXTENSION}`; + // --- + // Deployment - fs.writeFileSync( - outFile, - Buffer.from(Presentation.encode(presentation).finish()), + // --- + // Create directory + fsExtra.ensureDirSync(deploymentVersionedDir); + + const currentManifest = { + inventory: await generateManifest(deployableSongs), + updatedOn: versionedDir, + } as SongsInventoryManifest; + + fs.writeFileSync( + path.join(deploymentVersionedDir, MANIFEST_FILE), + JSON.stringify(currentManifest), + ); + + const isAFirstDeployment = isEmpty(allPreviousDeploymentDirs); + + if (isAFirstDeployment) { + console.log( + `No previous deployment found in "${baseOutDir}". Skip incremental deployments by doing a full deployment. Please proceed with a full manual import in PP7.`, ); - console.log(`Successfully generated "${outFile}".`); - }); + generateAndWriteSongsToDeploymentDir( + deployableSongs, + deploymentVersionedDir, + config, + ); + return; + } + + const previousManifest = getPreviousInventoryManifest( + deploymentDate, + allPreviousDeploymentDirs, + baseOutDir, + ); + + const { newOrUpdatedSongs, toBeRemovedFileNames } = getSongDiffs( + currentManifest, + previousManifest, + ); + + if (isEmpty(newOrUpdatedSongs)) { + console.log( + `Skip incremental deployments as no changes have been found between the last two versions.`, + ); + } else { + const partialDeployableSongs = deployableSongs.filter(({ song: { id } }) => + newOrUpdatedSongs + .map(({ id: newOrUpdatedSongId }) => newOrUpdatedSongId) + .includes(id), + ); + + generateAndWriteSongsToDeploymentDir( + partialDeployableSongs, + deploymentVersionedDir, + config, + ); + } + + if (!isEmpty(toBeRemovedFileNames)) { + console.log( + `The following songs have been removed: ${toBeRemovedFileNames}.`, + ); + } }; diff --git a/src/songsParser.spec.ts b/src/songsParser.spec.ts index 0649229..0af7f98 100644 --- a/src/songsParser.spec.ts +++ b/src/songsParser.spec.ts @@ -116,6 +116,9 @@ describe('songsParser', () => { expect(parseSong(SONG_WITH_SUBSECTIONS_MOCK_FILE_CONTENT)) .toMatchInlineSnapshot(` { + "author": "CustomAuthor", + "contentHash": "#customHash", + "id": "customId", "sequence": [ "[v1.1]", "[v1.2]", @@ -287,6 +290,9 @@ describe('songsParser', () => { expect(parseSong(SONG_WITH_SUBSECTIONS_MOCK_FILE_CONTENT)) .toMatchInlineSnapshot(` { + "author": "CustomAuthor", + "contentHash": "#customHash", + "id": "customId", "sequence": [ "[v1.1]", "[v1.2]", diff --git a/src/songsParser.ts b/src/songsParser.ts index a79fd98..f8463af 100644 --- a/src/songsParser.ts +++ b/src/songsParser.ts @@ -20,6 +20,7 @@ export const parseSong = (songContent: string): Song => { const hashMap = {} as Record; const sections = [] as Section[]; + let rawTitle = ''; for ( let sectionIndex = 0; @@ -32,7 +33,8 @@ export const parseSong = (songContent: string): Song => { hashMap[sectionIdentifier] = songSectionContent; if (sectionIdentifier === SongSection.TITLE) { - hashMap[SongSection.TITLE] = first( + rawTitle = songSectionContent; + hashMap[sectionIdentifier] = first( getTitleBySections(songSectionContent), ) as string; } @@ -71,9 +73,10 @@ export const parseSong = (songContent: string): Song => { } const titleContent = hashMap[SongSection.TITLE]; - const metaSectionsFromTitle = getMetaSectionsFromTitle( - titleContent, - ) as Record; + const metaSectionsFromTitle = getMetaSectionsFromTitle(rawTitle) as Record< + SongMeta, + string + >; return pickBy( { diff --git a/src/types.ts b/src/types.ts index 0659454..3c3e89b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -116,21 +116,19 @@ export type Song = { version?: string; }; -export type SongInventoryChange = { - date: string; +export type SongManifest = { + id: string; + fileName: string; contentHash: string; - filename: string; - isFilenameChanged?: boolean; - isDeleted?: boolean; }; -export type SongsInventoryEntry = { - id: string; - filename: string; - changes: SongInventoryChange[]; +export type SongsInventoryManifest = { + updatedOn: string; + inventory: SongManifest[]; }; -export type SongsInventory = { - updatedOn: string; - songs: SongsInventoryEntry[]; +export type DeployableSong = { + song: Song; + fileName: string; + fileAsText: string; };