feat: SKFP-354 migration from persona

Ferlab-Ste-Justine · Jul 5, 2024 · 4ba171b · 4ba171b
1 parent ab41eb5
commit 4ba171b
Show file tree

Hide file tree

Showing 7 changed files with 133 additions and 57 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -37,7 +37,8 @@
         "node-pg-migrate": "^6.0.0",
         "pg": "^8.7.1",
         "sequelize": "^6.28.2",
-        "uuidv4": "^6.2.13"
+        "uuidv4": "^6.2.13",
+        "validator": "^13.12.0"
     },
     "devDependencies": {
         "@types/cors": "^2.8.12",

diff --git a/src/db/models/User.ts b/src/db/models/User.ts
@@ -98,14 +98,14 @@ UserModel.init(
         first_name: {
             type: DataTypes.CITEXT,
             validate: {
-                len: [1, 35],
+                len: [1, 70],
                 is: NAME_REGEX,
             },
         },
         last_name: {
             type: DataTypes.CITEXT,
             validate: {
-                len: [1, 35],
+                len: [1, 70],
                 is: NAME_REGEX,
             },
         },

diff --git a/src/external/persona.ts b/src/external/persona.ts
@@ -1,5 +1,7 @@
+/* eslint-disable complexity */
 /* eslint-disable no-console */
 import { parseString } from '@fast-csv/parse';
+import validator from 'validator';
 
 import { personaURL } from '../config/env';
 import UserModel from '../db/models/User';
@@ -68,7 +70,7 @@ export const readCsv = (csvContent: string): Promise<Persona[]> =>
         parseString(csvContent, { headers: true })
             .on('error', reject)
             .on('data', (row) => {
-                const obj = convertEmptyToNull(row);
+                const obj = convertToPersona(row);
                 if (obj) data.push(obj);
             })
             .on('end', () => {
@@ -96,30 +98,46 @@ export const getUserList = async (accessToken: string): Promise<string> => {
     throw new Error(`Error during call to Persona with status [${response.status}]`);
 };
 
-export const convertEmptyToNull = (p: any): Persona => ({
+export const convertToPersona = (p: any): Persona => ({
     id: p.id,
     isPublic: p.isPublic === 'true',
     isActive: p.isActive === 'true',
     title: p.title,
-    firstName: p.firstName === '' ? null : p.firstName,
-    lastName: p.lastName === '' ? null : p.lastName,
+    firstName: p.firstName === '' ? null : (p.firstName as string).trim(),
+    lastName: p.lastName === '' ? null : (p.lastName as string).trim(),
     role: p.role === '' ? [] : p.role.split(','),
-    loginEmail: p.loginEmail === '' ? null : p.loginEmail,
-    institution: p.institution === '' ? null : p.institution,
+    loginEmail:
+        p.loginEmail === ''
+            ? null
+            : validator.isURL((p.loginEmail as string).trim())
+            ? (p.loginEmail as string).trim()
+            : null,
+    institution: p.institution === '' ? null : (p.institution as string).trim(),
     department: p.department,
     jobTitle: p.jobTitle,
     addressLine1: p.addressLine1,
     addressLine2: p.addressLine2,
     city: p.city,
-    state: p.state === '' ? null : p.state,
-    country: p.country === '' ? null : p.country,
+    state: p.state === '' ? null : (p.state as string).trim(),
+    country: p.country === '' ? null : (p.country as string).trim(),
     phone: p.phone,
-    institutionalEmail: p.institutionalEmail === '' ? null : p.institutionalEmail,
-    eraCommonsID: p.eraCommonsID === '' ? null : p.eraCommonsID,
-    website: p.website === '' ? null : p.website,
+    institutionalEmail:
+        p.institutionalEmail === ''
+            ? null
+            : validator.isURL((p.institutionalEmail as string).trim())
+            ? (p.institutionalEmail as string).trim()
+            : null,
+    eraCommonsID: p.eraCommonsID === '' ? null : (p.eraCommonsID as string).trim(),
+    website:
+        p.website === '' ? null : validator.isURL((p.website as string).trim()) ? (p.website as string).trim() : null,
     twitter: p.twitter,
     orchid: p.orchid,
-    linkedin: p.linkedin === '' ? null : LINKEDIN_REGEX.test(p.linkedin) ? p.linkedin : null,
+    linkedin:
+        p.linkedin === ''
+            ? null
+            : LINKEDIN_REGEX.test((p.linkedin as string).trim())
+            ? (p.linkedin as string).trim()
+            : null,
     googleScholarId: p.googleScholarId,
     github: p.github,
     facebook: p.facebook,
@@ -128,45 +146,17 @@ export const convertEmptyToNull = (p: any): Persona => ({
     acceptedKfOptIn: p.acceptedKfOptIn === 'true',
     acceptedDatasetSub: p.acceptedDatasetSub === 'true',
     interests: p.interests === '' ? [] : p.interests.split(','),
-    egoId: p.egoId === '' ? null : p.egoId,
+    egoId: p.egoId === '' ? null : (p.egoId as string).trim(),
     story: p.story,
     bio: p.bio,
 });
 
 export const createOrUpdate = async (payload: Persona): Promise<string> => {
     const existing = await UserModel.findAll({ where: { keycloak_id: payload.egoId } });
-    if (existing.length === 0) {
-        await UserModel.create({
-            keycloak_id: payload.egoId,
-            first_name: payload.firstName,
-            last_name: payload.lastName,
-            era_commons_id: payload.eraCommonsID,
-            email: payload.loginEmail,
-            linkedin: payload.linkedin,
-            external_individual_email: payload.institutionalEmail,
-            roles: payload.role,
-            affiliation: payload.institution,
-            consent_date: new Date('2021-11-22'),
-            accepted_terms: payload.acceptedTerms,
-            completed_registration: payload.acceptedTerms && payload.isActive,
-            deleted: !payload.isActive,
-            newsletter_email: payload.loginEmail,
-            newsletter_subscription_status: payload.acceptedKfOptIn
-                ? SubscriptionStatus.SUBSCRIBED
-                : SubscriptionStatus.UNSUBSCRIBED,
-            newsletter_dataset_subscription_status: payload.acceptedDatasetSub
-                ? SubscriptionStatus.SUBSCRIBED
-                : SubscriptionStatus.UNSUBSCRIBED,
-            location_country: payload.country,
-            location_state: payload.state,
-            website: payload.website,
-            areas_of_interest: payload.interests,
-            is_public: payload.isPublic,
-        });
-        return 'created';
-    } else if (existing.length === 1) {
-        await UserModel.update(
-            {
+    try {
+        if (existing.length === 0) {
+            await UserModel.create({
+                keycloak_id: payload.egoId,
                 first_name: payload.firstName,
                 last_name: payload.lastName,
                 era_commons_id: payload.eraCommonsID,
@@ -179,7 +169,7 @@ export const createOrUpdate = async (payload: Persona): Promise<string> => {
                 accepted_terms: payload.acceptedTerms,
                 completed_registration: payload.acceptedTerms && payload.isActive,
                 deleted: !payload.isActive,
-                newsletter_email: payload.loginEmail,
+                newsletter_email: payload.acceptedKfOptIn || payload.acceptedDatasetSub ? payload.loginEmail : null,
                 newsletter_subscription_status: payload.acceptedKfOptIn
                     ? SubscriptionStatus.SUBSCRIBED
                     : SubscriptionStatus.UNSUBSCRIBED,
@@ -191,11 +181,45 @@ export const createOrUpdate = async (payload: Persona): Promise<string> => {
                 website: payload.website,
                 areas_of_interest: payload.interests,
                 is_public: payload.isPublic,
-            },
-            { where: { keycloak_id: payload.egoId } },
-        );
-        return 'updated';
-    } else {
-        console.warn(`Duplicate user in users-api: ${payload.egoId}`);
+            });
+            return 'created';
+        } else if (existing.length === 1) {
+            await UserModel.update(
+                {
+                    first_name: payload.firstName,
+                    last_name: payload.lastName,
+                    era_commons_id: payload.eraCommonsID,
+                    email: payload.loginEmail,
+                    linkedin: payload.linkedin,
+                    external_individual_email: payload.institutionalEmail,
+                    roles: payload.role,
+                    affiliation: payload.institution,
+                    consent_date: new Date('2021-11-22'),
+                    accepted_terms: payload.acceptedTerms,
+                    completed_registration: payload.acceptedTerms && payload.isActive,
+                    deleted: !payload.isActive,
+                    newsletter_email: payload.acceptedKfOptIn || payload.acceptedDatasetSub ? payload.loginEmail : null,
+                    newsletter_subscription_status: payload.acceptedKfOptIn
+                        ? SubscriptionStatus.SUBSCRIBED
+                        : SubscriptionStatus.UNSUBSCRIBED,
+                    newsletter_dataset_subscription_status: payload.acceptedDatasetSub
+                        ? SubscriptionStatus.SUBSCRIBED
+                        : SubscriptionStatus.UNSUBSCRIBED,
+                    location_country: payload.country,
+                    location_state: payload.state,
+                    website: payload.website,
+                    areas_of_interest: payload.interests,
+                    is_public: payload.isPublic,
+                    updated_date: new Date(),
+                },
+                { where: { keycloak_id: payload.egoId } },
+            );
+            return 'updated';
+        } else {
+            console.warn(`Duplicate user in users-api: ${payload.egoId}`);
+        }
+    } catch (e) {
+        console.error(`Error: ${e.message} - Ignoring [${payload.egoId}]`);
+        return 'ignored';
     }
 };
diff --git a/src/routes/admin.ts b/src/routes/admin.ts
@@ -40,6 +40,7 @@ adminRouter.post('/doMigrationFromPersona', async (req, res, next) => {
                 res.status(StatusCodes.OK).send({
                     created: result.filter((s) => s === 'created').length,
                     updated: result.filter((s) => s === 'updated').length,
+                    ignored: result.filter((s) => s === 'ignored').length,
                 });
             } else {
                 res.status(StatusCodes.BAD_REQUEST).send('Duplicate persona, migration aborted');

diff --git a/src/utils/constants.ts b/src/utils/constants.ts
@@ -1,4 +1,4 @@
 export const UUID_VERSION = 4;
-export const NAME_REGEX = /^[a-zà-ÿ ,.'-_]+$/iu;
+export const NAME_REGEX = /^[\u0027-\u0029\u002F-\u0039\u0040\u0061-\u007A\u00C0-\uFFFF ,.'\-_]+$/iu; // see regex.test.ts to understand the regex
 export const LINKEDIN_REGEX = /^(http(s)?:\/\/)?([\w]+\.)?linkedin\.com\/(pub|in|profile)\/([-a-zA-Z0-9]+)\/*/iu;
 export const MAX_LENGTH_PER_ROLE = 100;
diff --git a/src/utils/regex.test.ts b/src/utils/regex.test.ts
@@ -0,0 +1,49 @@
+import { NAME_REGEX } from './constants';
+
+describe('NAME_REGEX', () => {
+    it('should allow alphanumeric lower and uppercase', () => {
+        expect(NAME_REGEX.test('ThisIsAlphanumeric123')).toBeTruthy();
+    });
+    it('should allow acutes', () => {
+        expect(NAME_REGEX.test('éèîôàùêïöÀÈÉÎÔÙÊÏÖÜ')).toBeTruthy();
+    });
+    it(`should allow / @ ( ) ' - _ , . space`, () => {
+        expect(NAME_REGEX.test(`/ @ ( ) ' - _ , . `)).toBeTruthy();
+    });
+    it(`should allow non latin characters`, () => {
+        expect(NAME_REGEX.test(`abæcdöef`)).toBeTruthy();
+        expect(NAME_REGEX.test(`правда`)).toBeTruthy();
+        expect(NAME_REGEX.test(`ยจฆฟคฏข`)).toBeTruthy();
+        expect(NAME_REGEX.test(`도메인`)).toBeTruthy();
+        expect(NAME_REGEX.test(`ドメイン名例`)).toBeTruthy();
+        expect(NAME_REGEX.test(`MajiでKoiする5秒前`)).toBeTruthy();
+        expect(NAME_REGEX.test(`Krstanović Bezsažna Kołodziejczak Štepec`)).toBeTruthy();
+        expect(NAME_REGEX.test(`הדר כהן`)).toBeTruthy();
+        expect(NAME_REGEX.test(`Đỗ Ngọc Tuấn`)).toBeTruthy();
+        expect(NAME_REGEX.test(`HALİL İBRAHİM`)).toBeTruthy();
+        expect(NAME_REGEX.test(`博 林 석주 박 益颖 麦 宇鴻 台師大 藍`)).toBeTruthy();
+    });
+    it(`should not allow ; : { } ^ ! " # $ % & * + < = > ? [ ] \\ | ~`, () => {
+        expect(NAME_REGEX.test('a;a')).toBeFalsy();
+        expect(NAME_REGEX.test('a:a')).toBeFalsy();
+        expect(NAME_REGEX.test('a{a')).toBeFalsy();
+        expect(NAME_REGEX.test('a}a')).toBeFalsy();
+        expect(NAME_REGEX.test('a^a')).toBeFalsy();
+        expect(NAME_REGEX.test('a!a')).toBeFalsy();
+        expect(NAME_REGEX.test('a"a')).toBeFalsy();
+        expect(NAME_REGEX.test('a#a')).toBeFalsy();
+        expect(NAME_REGEX.test('a$a')).toBeFalsy();
+        expect(NAME_REGEX.test('a%a')).toBeFalsy();
+        expect(NAME_REGEX.test('a&a')).toBeFalsy();
+        expect(NAME_REGEX.test('a*a')).toBeFalsy();
+        expect(NAME_REGEX.test('a+a')).toBeFalsy();
+        expect(NAME_REGEX.test('a<a')).toBeFalsy();
+        expect(NAME_REGEX.test('a=a')).toBeFalsy();
+        expect(NAME_REGEX.test('a>a')).toBeFalsy();
+        expect(NAME_REGEX.test('a?a')).toBeFalsy();
+        expect(NAME_REGEX.test('a[a')).toBeFalsy();
+        expect(NAME_REGEX.test('a]a')).toBeFalsy();
+        expect(NAME_REGEX.test('a\\a')).toBeFalsy();
+        expect(NAME_REGEX.test('a~a')).toBeFalsy();
+    });
+});