diff --git a/.env b/.env
index 7bffd96..4aacdf1 100644
--- a/.env
+++ b/.env
@@ -1,2 +1,3 @@
+TZ=Europe/Bucharest
SOURCE_DIR=../bes-lyrics/verified
OUT_DIR=./pp7-songs
diff --git a/.env.remote b/.env.remote
index 29b1234..40c912e 100644
--- a/.env.remote
+++ b/.env.remote
@@ -1,2 +1,3 @@
+TZ=Europe/Bucharest
SOURCE_DIR=../bes-lyrics/verified
-OUT_DIR=/Users/ilucut/WORK/BES/CLOUD DATA/ProPresenter_Generated/ProPresenter_Generated_Version_10
+OUT_DIR=/Users/ilucut/WORK/BES/CLOUD DATA/ProPresenter_Generated
diff --git a/.run/Run migrator.run.xml b/.run/Run migrator.run.xml
deleted file mode 100644
index df0b636..0000000
--- a/.run/Run migrator.run.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-
-
-
-
-
\ No newline at end of file
diff --git a/.run/[Local] Run migrator.run.xml b/.run/[Local] Run migrator.run.xml
new file mode 100644
index 0000000..145f37e
--- /dev/null
+++ b/.run/[Local] Run migrator.run.xml
@@ -0,0 +1,5 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/README.md b/README.md
index d0f472e..3dc6d3a 100644
--- a/README.md
+++ b/README.md
@@ -89,8 +89,7 @@ const CONFIG = {
migrateSongsToPP7Format({
sourceDir: process.env.SOURCE_DIR as string,
- outDir: process.env.OUT_DIR as string,
- clearOutputDirFirst: true,
+ baseOutDir: process.env.OUT_DIR as string,
config: CONFIG,
});
```
diff --git a/package.json b/package.json
index 4729a47..3c8c729 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
"main": "index.ts",
"scripts": {
"test:ci": "NODE_ENV=test jest --runInBand --no-cache",
- "test:watch": "NODE_ENV=test TZ='Europe/Berlin' jest --watch --logHeapUsage",
+ "test:watch": "NODE_ENV=test TZ='Europe/Bucharest' jest --watch --logHeapUsage",
"test": "is-ci-cli test:ci test:watch",
"migrate:local": "ts-node runner.ts",
"migrate:remote": "dotenv -e .env.remote ts-node runner.ts"
diff --git a/runner.ts b/runner.ts
index 715349b..f72bdac 100644
--- a/runner.ts
+++ b/runner.ts
@@ -32,8 +32,7 @@ const CONFIG = {
(async () => {
await migrateSongsToPP7Format({
sourceDir: process.env.SOURCE_DIR as string,
- outDir: process.env.OUT_DIR as string,
- clearOutputDirFirst: true,
+ baseOutDir: process.env.OUT_DIR as string,
config: CONFIG,
});
})();
diff --git a/src/constants.ts b/src/constants.ts
index 788a6c8..033dd41 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -12,6 +12,8 @@ export const SLASH = '/';
export const HASH = '#';
+export const DASH = '-';
+
export const TXT_EXTENSION = '.txt';
export const TEST_FILE = 'TEMPLATE.txt';
diff --git a/src/core.spec.ts b/src/core.spec.ts
index 7c2f7a3..8a8da48 100644
--- a/src/core.spec.ts
+++ b/src/core.spec.ts
@@ -1,10 +1,17 @@
import {
+ parseDateFromVersionedDir,
getMetaSectionsFromTitle,
getSongInSectionTuples,
getTitleWithoutMeta,
+ getVersionedDir,
+ getClosestVersionedDir,
} from './core';
import { SIMPLE_SONG_MOCK_FILE_CONTENT } from '../mocks';
+jest
+ .useFakeTimers()
+ .setSystemTime(new Date('2023-02-19T11:12:13.000Z').getTime());
+
describe('core', () => {
describe('getSongInSectionTuples', () => {
it('should work correctly', () => {
@@ -74,4 +81,38 @@ describe('core', () => {
`);
});
});
+
+ describe('getClosestVersionedDir', () => {
+ it('should work correctly', () => {
+ expect(getVersionedDir()).toMatchInlineSnapshot(`"2023-02-19-12:12:13"`);
+ });
+ });
+
+ describe('parseDateFromVersionedDir', () => {
+ it('should work correctly', () => {
+ expect(
+ parseDateFromVersionedDir('2023-02-19-11:12:13'),
+ ).toMatchInlineSnapshot(`2023-02-19T10:12:13.000Z`);
+
+ expect(
+ parseDateFromVersionedDir('2023-08-08-19:17:25'),
+ ).toMatchInlineSnapshot(`2023-08-08T17:17:25.000Z`);
+ });
+ });
+
+ describe('getClosestVersionedDir', () => {
+ it('should work correctly', () => {
+ expect(
+ getClosestVersionedDir(
+ parseDateFromVersionedDir('2023-26-20-11:12:13'),
+ [
+ parseDateFromVersionedDir('2023-02-20-11:12:13'),
+ parseDateFromVersionedDir('2023-02-19-11:12:13'),
+ parseDateFromVersionedDir('2023-02-23-11:12:13'),
+ parseDateFromVersionedDir('2023-02-24-11:12:13'),
+ ],
+ ),
+ ).toMatchInlineSnapshot(`2023-02-24T10:12:13.000Z`);
+ });
+ });
});
diff --git a/src/core.ts b/src/core.ts
index 2df324f..5e4ffde 100644
--- a/src/core.ts
+++ b/src/core.ts
@@ -13,6 +13,7 @@ import { SequenceChar, SongMeta } from './types';
import {
COLON,
COMMA,
+ DASH,
DOUBLE_LINE_TUPLE,
EMPTY_STRING,
HASH,
@@ -130,18 +131,41 @@ ${tuples
export const convertSequenceToNumber = (sequenceOrderQualifier: string) =>
parseInt(sequenceOrderQualifier) || MISSING_SEQUENCE_NUMBER;
-export const getTodayAsDateString = () => {
- const now = new Date();
-
- return (
- ('0' + now.getDate()).slice(-2) +
- '-' +
- ('0' + (now.getMonth() + 1)).slice(-2) +
- '-' +
- now.getFullYear()
+export const getVersionedDir = (now = new Date()) =>
+ now.getFullYear() +
+ DASH +
+ ('0' + (now.getMonth() + 1)).slice(-2) +
+ DASH +
+ ('0' + now.getDate()).slice(-2) +
+ DASH +
+ ('0' + now.getHours()).slice(-2) +
+ COLON +
+ ('0' + now.getMinutes()).slice(-2) +
+ COLON +
+ ('0' + now.getSeconds()).slice(-2);
+
+export const parseDateFromVersionedDir = (versionFolder: string) => {
+ const [year, month, day, time] = versionFolder.split(DASH);
+ const [hour, minute, second] = time.split(COLON);
+
+ return new Date(
+ parseInt(year),
+ parseInt(month) - 1,
+ parseInt(day),
+ parseInt(hour),
+ parseInt(minute),
+ parseInt(second),
);
};
+export const getClosestVersionedDir = (diffDate: Date, dates: Date[]) =>
+ first(
+ dates.sort((a, b) => {
+ // @ts-ignore
+ return Math.abs(diffDate - a) - Math.abs(diffDate - b); // sort a before b when the distance is smaller
+ }),
+ );
+
export const assertUniqueness = (array: string[]) =>
assert.equal(
size(uniq(array)),
diff --git a/src/index.ts b/src/index.ts
index b137404..f979d07 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,4 +1,3 @@
-export * from './inventorySnapshotRunner';
export * from './migratorRunner';
export * from './proPresenter7SongConverter';
export * from './songsParser';
diff --git a/src/inventorySnapshotRunner.ts b/src/inventorySnapshotRunner.ts
deleted file mode 100644
index a76b0a1..0000000
--- a/src/inventorySnapshotRunner.ts
+++ /dev/null
@@ -1,141 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import dotenv from 'dotenv';
-import recursive from 'recursive-readdir';
-import { isEqual, last, reject } from 'lodash';
-import {
- assertUniqueness,
- getMetaSectionsFromTitle,
- getSongInSectionTuples,
- getTitleWithoutMeta,
- getTodayAsDateString,
- logFileWithLinkInConsole,
- logProcessingFile,
-} from './core';
-import assert from 'node:assert';
-import {
- SongInventoryChange,
- SongMeta,
- SongsInventory,
- SongsInventoryEntry,
-} from './types';
-import { TXT_EXTENSION } from './constants';
-
-dotenv.config();
-
-const inventoryChangelogFilename = path.join(
- __dirname,
- '..',
- 'inventoryChangelog.json',
-);
-
-const snapshotInventory = async (dir: string) => {
- console.log(`"Reprocessing file names from ${dir} directory.."`);
-
- const currentInventory = JSON.parse(
- fs.readFileSync(inventoryChangelogFilename).toString(),
- ) as SongsInventory;
-
- const { songs } = currentInventory;
-
- const songsAsHashMap = songs.reduce(
- (accumulator, entry) => ({
- ...accumulator,
- [entry.id]: entry,
- }),
- {} as {
- [id: string]: SongsInventoryEntry;
- },
- );
-
- const songsMeta = (await recursive(dir))
- .filter((filePath) => path.extname(filePath) === TXT_EXTENSION)
- .map((filePath) => {
- const filename = path.basename(filePath);
- const fileContent = fs.readFileSync(filePath).toString();
-
- logProcessingFile(filename, 'inventory generation');
- logFileWithLinkInConsole(filePath);
-
- const rawTitle = getSongInSectionTuples(fileContent)[1];
-
- assert.ok(
- rawTitle.includes(SongMeta.CONTENT_HASH),
- `The ${SongMeta.CONTENT_HASH} should be defined.`,
- );
-
- return {
- filename,
- metaSections: getMetaSectionsFromTitle(rawTitle),
- titleWithoutMetaSections: getTitleWithoutMeta(rawTitle),
- };
- });
-
- assertUniqueness(
- songsMeta.map(({ metaSections }) => metaSections[SongMeta.ID]),
- );
-
- const dateAsString = getTodayAsDateString();
- const updatedSongsInventory = songsMeta.map(({ filename, metaSections }) => {
- const id = metaSections[SongMeta.ID];
- const contentHash = metaSections[SongMeta.CONTENT_HASH];
- const currentInventory = songsAsHashMap[id];
-
- if (!currentInventory) {
- return {
- id,
- filename,
- changes: [
- {
- date: dateAsString,
- contentHash,
- },
- ],
- } as SongsInventoryEntry;
- }
-
- const { changes } = currentInventory;
- const {
- filename: lastChangeFilename,
- contentHash: lastChangedContentHash,
- date: lastChangedDate,
- } = last(changes) as SongInventoryChange;
-
- if (isEqual(lastChangedContentHash, contentHash)) {
- return currentInventory;
- }
-
- const isFilenameChanged = !isEqual(lastChangeFilename, filename);
- return {
- id,
- filename,
- changes: [
- ...(isEqual(lastChangedDate, dateAsString)
- ? reject(changes, { date: lastChangedDate })
- : changes),
- {
- date: dateAsString,
- contentHash,
- ...(isFilenameChanged ? { filename, isFilenameChanged } : {}),
- },
- ],
- } as SongsInventoryEntry;
- });
-
- assertUniqueness(Object.keys(updatedSongsInventory).map((id) => id));
-
- const nextSongs = updatedSongsInventory.sort((a, b) =>
- a.filename.localeCompare(b.filename),
- );
- fs.writeFileSync(
- inventoryChangelogFilename,
- JSON.stringify(
- isEqual(songs, nextSongs)
- ? currentInventory
- : {
- updatedOn: dateAsString,
- songs: nextSongs,
- },
- ),
- );
-};
diff --git a/src/manifestGenerator.ts b/src/manifestGenerator.ts
new file mode 100644
index 0000000..a8949cf
--- /dev/null
+++ b/src/manifestGenerator.ts
@@ -0,0 +1,26 @@
+import dotenv from 'dotenv';
+import { assertUniqueness } from './core';
+import { Song, SongManifest } from './types';
+
+dotenv.config();
+
+export const generateManifest = async (
+ deployableSongs: Array<{
+ song: Song;
+ fileName: string;
+ fileAsText: string;
+ }>,
+) => {
+ assertUniqueness(deployableSongs.map(({ song }) => song.id));
+
+ return deployableSongs
+ .map(
+ ({ song: { id, contentHash }, fileName }) =>
+ ({
+ id,
+ fileName,
+ contentHash,
+ } as SongManifest),
+ )
+ .sort((a, b) => a.fileName.localeCompare(b.fileName));
+};
diff --git a/src/migratorRunner.ts b/src/migratorRunner.ts
index 4d99958..7e0b04e 100644
--- a/src/migratorRunner.ts
+++ b/src/migratorRunner.ts
@@ -8,47 +8,229 @@ import {
Config,
convertSongToProPresenter7,
} from './proPresenter7SongConverter';
+import { generateManifest } from './manifestGenerator';
import { EMPTY_STRING, PRO_EXTENSION, TXT_EXTENSION } from './constants';
+import {
+ getClosestVersionedDir,
+ parseDateFromVersionedDir,
+ getVersionedDir,
+ logFileWithLinkInConsole,
+ logProcessingFile,
+} from './core';
+import { DeployableSong, SongManifest, SongsInventoryManifest } from './types';
+import assert from 'node:assert';
+import { isEmpty, isEqual } from 'lodash';
+
+const MANIFEST_FILE = 'manifest.json';
+
+// ---
+// Generate the songs from the given array to the new deployment directory
+const generateAndWriteSongsToDeploymentDir = (
+ songsToGenerate: DeployableSong[],
+ dir: string,
+ config: Config,
+) => {
+ songsToGenerate.forEach(({ fileName, song }) => {
+ const songInProPresenter7Format = convertSongToProPresenter7(song, config);
+
+ const songFileName = fileName.replace(TXT_EXTENSION, EMPTY_STRING);
+ const pp7ExportedFile = path.join(dir, `${songFileName}${PRO_EXTENSION}`);
+
+ fs.writeFileSync(
+ pp7ExportedFile,
+ Buffer.from(Presentation.encode(songInProPresenter7Format).finish()),
+ );
+
+ console.log(
+ `Successfully generated "${pp7ExportedFile}" in the "${dir}" directory.`,
+ );
+ });
+};
+
+const getPreviousInventoryManifest = (
+ deploymentDate: Date,
+ allPreviousDeploymentDirs: {
+ deploymentDirDate: Date;
+ deploymentDir: string;
+ }[],
+ baseOutDir: string,
+) => {
+ const previousDeploymentDirDate = getClosestVersionedDir(
+ deploymentDate,
+ allPreviousDeploymentDirs.map(({ deploymentDirDate }) => deploymentDirDate),
+ );
+
+ const previousDeploymentManifestFilePath = path.join(
+ baseOutDir,
+ getVersionedDir(previousDeploymentDirDate),
+ MANIFEST_FILE,
+ );
+
+ console.log(
+ `Previous deployment manifest file path: ${previousDeploymentManifestFilePath}.`,
+ );
+
+ const previousInventoryManifest = JSON.parse(
+ fs.readFileSync(previousDeploymentManifestFilePath).toString(),
+ ) as SongsInventoryManifest;
+
+ logProcessingFile(
+ previousDeploymentManifestFilePath,
+ 'Previous manifest inspection',
+ );
+ logFileWithLinkInConsole(previousDeploymentManifestFilePath);
+
+ return previousInventoryManifest;
+};
+
+const getManifestHashMap = (songManifestArray: SongManifest[]) =>
+ songManifestArray.reduce(
+ (accumulator, entry) => ({
+ ...accumulator,
+ [entry.id]: entry,
+ }),
+ {} as {
+ [id: string]: SongManifest;
+ },
+ );
+
+const getSongDiffs = (
+ currentManifest: SongsInventoryManifest,
+ previousManifest: SongsInventoryManifest,
+) => {
+ const previousManifestHashMap = getManifestHashMap(
+ previousManifest.inventory,
+ );
+ const currentManifestHashMap = getManifestHashMap(currentManifest.inventory);
+
+ const newOrUpdatedSongs = currentManifest.inventory.filter(
+ ({ id, fileName, contentHash }) =>
+ // Is new song
+ !previousManifestHashMap[id] ||
+ // Is an existing-updated song
+ !isEqual(previousManifestHashMap[id]?.contentHash, contentHash),
+ );
+
+ const toBeRemovedFileNames = previousManifest.inventory
+ .filter(
+ ({ id, fileName, contentHash }) =>
+ // File-name has changed
+ !isEqual(currentManifestHashMap[id]?.fileName, fileName),
+ )
+ .map(({ id }) => previousManifestHashMap[id]?.fileName);
+
+ return {
+ newOrUpdatedSongs,
+ toBeRemovedFileNames,
+ };
+};
/**
* Removes all the files from the out directory
*/
export const migrateSongsToPP7Format = async ({
sourceDir,
- outDir,
- clearOutputDirFirst,
+ baseOutDir,
config,
}: {
sourceDir: string;
- outDir: string;
- clearOutputDirFirst?: boolean;
+ baseOutDir: string;
config: Config;
}) => {
- if (clearOutputDirFirst) {
- fsExtra.emptydirSync(outDir);
- }
+ const versionedDir = getVersionedDir();
+ const deploymentDate = parseDateFromVersionedDir(versionedDir);
+ const deploymentVersionedDir = `${baseOutDir}/${versionedDir}`;
- await fsExtra.ensureDirSync(outDir);
+ assert(
+ !fsExtra.pathExistsSync(deploymentVersionedDir),
+ `The out directory "${deploymentVersionedDir}" does exists already.`,
+ );
- (await recursive(sourceDir, ['.DS_Store'])).forEach((filePath) => {
- const fileAsText = fs.readFileSync(filePath).toString();
- const fileName = path.basename(filePath);
+ // ---
+ // Existing deployments
+ const allPreviousDeploymentDirs = fsExtra
+ .readdirSync(baseOutDir)
+ .map((deploymentDir) => ({
+ deploymentDir,
+ deploymentDirDate: parseDateFromVersionedDir(deploymentDir),
+ }));
- console.log(`Processing "${filePath}"...`);
+ // ---
+ // Current deployment
+ const deployableSongs = (await recursive(sourceDir, ['.DS_Store'])).map(
+ (filePath) => {
+ const fileAsText = fs.readFileSync(filePath).toString();
+ const fileName = path.basename(filePath);
+ const song = parseSong(fileAsText);
- const song = parseSong(fileAsText);
- const presentation = convertSongToProPresenter7(song, config);
+ return { song, fileName, fileAsText } as DeployableSong;
+ },
+ );
- const outFile = `${outDir}/${fileName.replace(
- TXT_EXTENSION,
- EMPTY_STRING,
- )}${PRO_EXTENSION}`;
+ // ---
+ // Deployment
- fs.writeFileSync(
- outFile,
- Buffer.from(Presentation.encode(presentation).finish()),
+ // ---
+ // Create directory
+ fsExtra.ensureDirSync(deploymentVersionedDir);
+
+ const currentManifest = {
+ inventory: await generateManifest(deployableSongs),
+ updatedOn: versionedDir,
+ } as SongsInventoryManifest;
+
+ fs.writeFileSync(
+ path.join(deploymentVersionedDir, MANIFEST_FILE),
+ JSON.stringify(currentManifest),
+ );
+
+ const isAFirstDeployment = isEmpty(allPreviousDeploymentDirs);
+
+ if (isAFirstDeployment) {
+ console.log(
+ `No previous deployment found in "${baseOutDir}". Skip incremental deployments by doing a full deployment. Please proceed with a full manual import in PP7.`,
);
- console.log(`Successfully generated "${outFile}".`);
- });
+ generateAndWriteSongsToDeploymentDir(
+ deployableSongs,
+ deploymentVersionedDir,
+ config,
+ );
+ return;
+ }
+
+ const previousManifest = getPreviousInventoryManifest(
+ deploymentDate,
+ allPreviousDeploymentDirs,
+ baseOutDir,
+ );
+
+ const { newOrUpdatedSongs, toBeRemovedFileNames } = getSongDiffs(
+ currentManifest,
+ previousManifest,
+ );
+
+ if (isEmpty(newOrUpdatedSongs)) {
+ console.log(
+ `Skip incremental deployments as no changes have been found between the last two versions.`,
+ );
+ } else {
+ const partialDeployableSongs = deployableSongs.filter(({ song: { id } }) =>
+ newOrUpdatedSongs
+ .map(({ id: newOrUpdatedSongId }) => newOrUpdatedSongId)
+ .includes(id),
+ );
+
+ generateAndWriteSongsToDeploymentDir(
+ partialDeployableSongs,
+ deploymentVersionedDir,
+ config,
+ );
+ }
+
+ if (!isEmpty(toBeRemovedFileNames)) {
+ console.log(
+ `The following songs have been removed: ${toBeRemovedFileNames}.`,
+ );
+ }
};
diff --git a/src/songsParser.spec.ts b/src/songsParser.spec.ts
index 0649229..0af7f98 100644
--- a/src/songsParser.spec.ts
+++ b/src/songsParser.spec.ts
@@ -116,6 +116,9 @@ describe('songsParser', () => {
expect(parseSong(SONG_WITH_SUBSECTIONS_MOCK_FILE_CONTENT))
.toMatchInlineSnapshot(`
{
+ "author": "CustomAuthor",
+ "contentHash": "#customHash",
+ "id": "customId",
"sequence": [
"[v1.1]",
"[v1.2]",
@@ -287,6 +290,9 @@ describe('songsParser', () => {
expect(parseSong(SONG_WITH_SUBSECTIONS_MOCK_FILE_CONTENT))
.toMatchInlineSnapshot(`
{
+ "author": "CustomAuthor",
+ "contentHash": "#customHash",
+ "id": "customId",
"sequence": [
"[v1.1]",
"[v1.2]",
diff --git a/src/songsParser.ts b/src/songsParser.ts
index a79fd98..f8463af 100644
--- a/src/songsParser.ts
+++ b/src/songsParser.ts
@@ -20,6 +20,7 @@ export const parseSong = (songContent: string): Song => {
const hashMap = {} as Record;
const sections = [] as Section[];
+ let rawTitle = '';
for (
let sectionIndex = 0;
@@ -32,7 +33,8 @@ export const parseSong = (songContent: string): Song => {
hashMap[sectionIdentifier] = songSectionContent;
if (sectionIdentifier === SongSection.TITLE) {
- hashMap[SongSection.TITLE] = first(
+ rawTitle = songSectionContent;
+ hashMap[sectionIdentifier] = first(
getTitleBySections(songSectionContent),
) as string;
}
@@ -71,9 +73,10 @@ export const parseSong = (songContent: string): Song => {
}
const titleContent = hashMap[SongSection.TITLE];
- const metaSectionsFromTitle = getMetaSectionsFromTitle(
- titleContent,
- ) as Record;
+ const metaSectionsFromTitle = getMetaSectionsFromTitle(rawTitle) as Record<
+ SongMeta,
+ string
+ >;
return pickBy(
{
diff --git a/src/types.ts b/src/types.ts
index 0659454..3c3e89b 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -116,21 +116,19 @@ export type Song = {
version?: string;
};
-export type SongInventoryChange = {
- date: string;
+export type SongManifest = {
+ id: string;
+ fileName: string;
contentHash: string;
- filename: string;
- isFilenameChanged?: boolean;
- isDeleted?: boolean;
};
-export type SongsInventoryEntry = {
- id: string;
- filename: string;
- changes: SongInventoryChange[];
+export type SongsInventoryManifest = {
+ updatedOn: string;
+ inventory: SongManifest[];
};
-export type SongsInventory = {
- updatedOn: string;
- songs: SongsInventoryEntry[];
+export type DeployableSong = {
+ song: Song;
+ fileName: string;
+ fileAsText: string;
};