diff --git a/migrations/1719943513629_add-kf-columns.sql b/migrations/1719943513629_add-kf-columns.sql new file mode 100644 index 0000000..531aaa1 --- /dev/null +++ b/migrations/1719943513629_add-kf-columns.sql @@ -0,0 +1,28 @@ +-- Up Migration +ALTER TABLE users + ADD COLUMN newsletter_dataset_subscription_status TEXT; +ALTER TABLE users + ADD COLUMN location_country CITEXT; +ALTER TABLE users + ADD COLUMN location_state CITEXT; +ALTER TABLE users + ADD COLUMN website TEXT; +ALTER TABLE users + ADD COLUMN areas_of_interest CITEXT[]; +ALTER TABLE users + ADD COLUMN is_public BOOLEAN NOT NULL DEFAULT false; +UPDATE users SET is_public = true; + +-- Down Migration +ALTER TABLE users + DROP COLUMN newsletter_dataset_subscription_status; +ALTER TABLE users + DROP COLUMN location_country; +ALTER TABLE users + DROP COLUMN location_state; +ALTER TABLE users + DROP COLUMN website; +ALTER TABLE users + DROP COLUMN areas_of_interest; +ALTER TABLE users + DROP COLUMN is_public; diff --git a/migrations/1719944573861_remove-unused-nih-ned-id.sql b/migrations/1719944573861_remove-unused-nih-ned-id.sql new file mode 100644 index 0000000..0e5bac2 --- /dev/null +++ b/migrations/1719944573861_remove-unused-nih-ned-id.sql @@ -0,0 +1,5 @@ +-- Up Migration +ALTER TABLE users DROP COLUMN nih_ned_id; + +-- Down Migration +ALTER TABLE users ADD COLUMN nih_ned_id VARCHAR(255); diff --git a/package-lock.json b/package-lock.json index 679c9a2..15e24fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "ISC", "dependencies": { "@aws-sdk/client-s3": "^3.312.0", + "@fast-csv/parse": "^5.0.0", "@types/pg-format": "^1.0.2", "@types/validator": "^13.7.1", "cors": "^2.8.5", @@ -24,7 +25,8 @@ "node-pg-migrate": "^6.0.0", "pg": "^8.7.1", "sequelize": "^6.28.2", - "uuidv4": "^6.2.13" + "uuidv4": "^6.2.13", + "validator": "^13.12.0" }, "devDependencies": { "@types/cors": "^2.8.12", @@ -54,9 +56,6 @@ "engines": { "node": ">=18.8.0", "npm": ">=8.1.0" - }, - "peerDependencies": { - "braces": "3.0.3" } }, "node_modules/@ampproject/remapping": { @@ -1590,6 +1589,19 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fast-csv/parse": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.0.tgz", + "integrity": "sha512-ecF8tCm3jVxeRjEB6VPzmA+1wGaJ5JgaUX2uesOXdXD6qQp0B3EdshOIed4yT1Xlj/F2f8v4zHSo0Oi31L697g==", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "node_modules/@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -7631,6 +7643,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -7643,12 +7665,22 @@ "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", "dev": true }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", "dev": true }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -7667,6 +7699,11 @@ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "dev": true }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -7685,6 +7722,11 @@ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "dev": true }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -11592,6 +11634,19 @@ "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", "dev": true }, + "@fast-csv/parse": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.0.tgz", + "integrity": "sha512-ecF8tCm3jVxeRjEB6VPzmA+1wGaJ5JgaUX2uesOXdXD6qQp0B3EdshOIed4yT1Xlj/F2f8v4zHSo0Oi31L697g==", + "requires": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -13134,7 +13189,8 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true + "dev": true, + "requires": {} }, "acorn-walk": { "version": "8.3.3", @@ -13827,7 +13883,8 @@ "version": "1.5.3", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", - "dev": true + "dev": true, + "requires": {} }, "deep-is": { "version": "0.1.4", @@ -14238,7 +14295,8 @@ "version": "8.10.0", "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.10.0.tgz", "integrity": "sha512-SM8AMJdeQqRYT9O9zguiruQZaN7+z+E4eAP9oiLNGKMtomwaB1E9dcgUD6ZAn/eQAb52USbvezbiljfZUhbJcg==", - "dev": true + "dev": true, + "requires": {} }, "eslint-plugin-prettier": { "version": "4.2.1", @@ -14253,7 +14311,8 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/eslint-plugin-simple-import-sort/-/eslint-plugin-simple-import-sort-7.0.0.tgz", "integrity": "sha512-U3vEDB5zhYPNfxT5TYR7u01dboFZp+HNpnGhkDB2g/2E4wZ/g1Q9Ton8UwCLfRV9yAKyYqDh62oHOamvkFxsvw==", - "dev": true + "dev": true, + "requires": {} }, "eslint-scope": { "version": "5.1.1", @@ -15703,7 +15762,8 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true + "dev": true, + "requires": {} }, "jest-regex-util": { "version": "29.6.3", @@ -16227,6 +16287,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -16239,12 +16309,22 @@ "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", "dev": true }, + "lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, "lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", "dev": true }, + "lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, "lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -16263,6 +16343,11 @@ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "dev": true }, + "lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -16281,6 +16366,11 @@ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "dev": true }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" + }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -16888,7 +16978,8 @@ "pg-pool": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==" + "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", + "requires": {} }, "pg-protocol": { "version": "1.6.1", diff --git a/package.json b/package.json index 099198d..6229c38 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.312.0", + "@fast-csv/parse": "^5.0.0", "@types/pg-format": "^1.0.2", "@types/validator": "^13.7.1", "cors": "^2.8.5", @@ -36,7 +37,8 @@ "node-pg-migrate": "^6.0.0", "pg": "^8.7.1", "sequelize": "^6.28.2", - "uuidv4": "^6.2.13" + "uuidv4": "^6.2.13", + "validator": "^13.12.0" }, "devDependencies": { "@types/cors": "^2.8.12", diff --git a/src/config/env.ts b/src/config/env.ts index 010b5d6..929d2b4 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -20,3 +20,5 @@ export const adminRoleName = process.env.ADMIN_ROLE_NAME || 'admin'; export const smartsheetId = process.env.SMARTSHEET_ID; export const smartsheetToken = process.env.SMARTSHEET_TOKEN; + +export const personaURL = process.env.PERSONA_URL || 'not supported'; diff --git a/src/db/dal/user.ts b/src/db/dal/user.ts index 6127998..8b69860 100644 --- a/src/db/dal/user.ts +++ b/src/db/dal/user.ts @@ -24,9 +24,9 @@ const sanitizeInputPayload = (payload: IUserInput) => { creation_date, email, era_commons_id, - nih_ned_id, newsletter_email, newsletter_subscription_status, + newsletter_dataset_subscription_status, ...rest } = payload; @@ -112,6 +112,7 @@ export const searchUsers = async ({ where: { [Op.and]: { completed_registration: true, + is_public: true, deleted: false, ...matchClauses, [Op.and]: andClauses, @@ -225,7 +226,6 @@ export const deleteUser = async (keycloak_id: string): Promise => { email: null, affiliation: null, public_email: null, - nih_ned_id: null, era_commons_id: null, first_name: null, last_name: null, diff --git a/src/db/models/User.ts b/src/db/models/User.ts index bd46c70..c17ab42 100644 --- a/src/db/models/User.ts +++ b/src/db/models/User.ts @@ -10,7 +10,6 @@ interface IUserAttributes { first_name?: string; last_name?: string; era_commons_id?: string; - nih_ned_id?: string; email?: string; linkedin?: string; public_email?: string; @@ -34,6 +33,12 @@ interface IUserAttributes { locale?: string; newsletter_email?: string; newsletter_subscription_status?: SubscriptionStatus; + newsletter_dataset_subscription_status?: SubscriptionStatus; + location_country?: string; + location_state?: string; + website?: string; + areas_of_interest?: string[]; + is_public?: boolean; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -57,6 +62,11 @@ class UserModel extends Model implements IUserAttri public public_email?: string; public external_individual_email?: string; public linkedin?: string; + public location_country?: string; + public location_state?: string; + public website?: string; + public areas_of_interest?: string[]; + public is_public?: boolean; } UserModel.init( @@ -88,31 +98,25 @@ UserModel.init( first_name: { type: DataTypes.CITEXT, validate: { - len: [1, 35], + len: [1, 70], is: NAME_REGEX, }, }, last_name: { type: DataTypes.CITEXT, validate: { - len: [1, 35], + len: [1, 70], is: NAME_REGEX, }, }, era_commons_id: { type: DataTypes.STRING, }, - nih_ned_id: { - type: DataTypes.STRING, - validate: { - isAlpha: true, - }, - }, commercial_use_reason: { type: DataTypes.STRING, allowNull: true, validate: { - validate: (value: string) => value === '' || NAME_REGEX.test(value), + validate: (value: string) => value === '' || NAME_REGEX.test(value.trim()), }, }, email: { @@ -149,7 +153,7 @@ UserModel.init( type: DataTypes.CITEXT, allowNull: true, validate: { - validate: (value: string) => value === '' || NAME_REGEX.test(value), + validate: (value: string) => value === '' || NAME_REGEX.test(value.trim()), }, }, public_email: { @@ -234,6 +238,44 @@ UserModel.init( isAlpha: true, }, }, + newsletter_dataset_subscription_status: { + type: DataTypes.ENUM( + SubscriptionStatus.SUBSCRIBED, + SubscriptionStatus.UNSUBSCRIBED, + SubscriptionStatus.FAILED, + ), + allowNull: true, + validate: { + isAlpha: true, + }, + }, + location_country: { + type: DataTypes.CITEXT, + allowNull: true, + validate: { + validate: (value: string) => value === '' || NAME_REGEX.test(value.trim()), + }, + }, + location_state: { + type: DataTypes.CITEXT, + allowNull: true, + validate: { + validate: (value: string) => value === '' || NAME_REGEX.test(value.trim()), + }, + }, + website: { + type: DataTypes.TEXT, + allowNull: true, + validate: { isUrl: true }, + }, + areas_of_interest: DataTypes.ARRAY(DataTypes.CITEXT), + is_public: { + type: DataTypes.BOOLEAN, + defaultValue: false, + validate: { + isBoolean: true, + }, + }, }, { sequelize: sequelizeConnection, diff --git a/src/external/persona.test.ts b/src/external/persona.test.ts new file mode 100644 index 0000000..4240ddf --- /dev/null +++ b/src/external/persona.test.ts @@ -0,0 +1,197 @@ +import { convertToPersona, Persona, validateUniqPersonas } from './persona'; + +describe('validateUniqPersonas', () => { + const p: Persona = { egoId: '1' } as Persona; + it('should return true if no duplicate', () => { + const personas = [p, { ...p, egoId: '2' }, { ...p, egoId: '3' }]; + + expect(validateUniqPersonas(personas)).toBeTruthy(); + }); + + it('should return false if at least 1 duplicate', () => { + const personas = [p, { ...p, egoId: '2' }, { ...p, egoId: '2' }]; + + expect(validateUniqPersonas(personas)).toBeFalsy(); + }); +}); + +describe('convertToPersona', () => { + const emptyInput = { + id: '', + isPublic: '', + isActive: '', + title: '', + firstName: '', + lastName: '', + role: '', + loginEmail: '', + institution: '', + department: '', + jobTitle: '', + addressLine1: '', + addressLine2: '', + city: '', + state: '', + country: '', + phone: '', + institutionalEmail: '', + eraCommonsID: '', + website: '', + twitter: '', + orchid: '', + linkedin: '', + googleScholarId: '', + github: '', + facebook: '', + acceptedTerms: '', + acceptedNihOptIn: '', + acceptedKfOptIn: '', + acceptedDatasetSub: '', + interests: '', + egoId: '', + story: '', + bio: '', + }; + + it('should transform boolean', () => { + const input = { + ...emptyInput, + isPublic: 'true', + isActive: 'true', + acceptedTerms: 'false', + acceptedNihOptIn: '0', + acceptedKfOptIn: '1', + acceptedDatasetSub: '', + }; + + const persona = convertToPersona(input); + + expect(persona.isPublic).toBeTruthy(); + expect(persona.isActive).toBeTruthy(); + expect(persona.acceptedTerms).toBeFalsy(); + expect(persona.acceptedNihOptIn).toBeFalsy(); + expect(persona.acceptedKfOptIn).toBeFalsy(); + expect(persona.acceptedDatasetSub).toBeFalsy(); + }); + + it('should replace empty string by null', () => { + const persona = convertToPersona(emptyInput); + + expect(persona.firstName).toBeNull(); + expect(persona.lastName).toBeNull(); + expect(persona.loginEmail).toBeNull(); + expect(persona.institution).toBeNull(); + expect(persona.state).toBeNull(); + expect(persona.country).toBeNull(); + expect(persona.institutionalEmail).toBeNull(); + expect(persona.eraCommonsID).toBeNull(); + expect(persona.website).toBeNull(); + expect(persona.linkedin).toBeNull(); + expect(persona.egoId).toBeNull(); + }); + + it('should split arrays of string', () => { + const input = { + ...emptyInput, + role: 'a,b', + interests: 'c,d', + }; + + const personaEmpty = convertToPersona(emptyInput); + const persona = convertToPersona(input); + + expect(personaEmpty.role).toEqual([]); + expect(personaEmpty.interests).toEqual([]); + expect(persona.role).toEqual(['a', 'b']); + expect(persona.interests).toEqual(['c', 'd']); + }); + + it('should trim', () => { + const input = { + ...emptyInput, + firstName: ' firstName ', + lastName: ' lastName ', + loginEmail: ' login@email.com ', + institution: ' institution ', + state: ' state ', + country: ' country ', + institutionalEmail: ' institutional@email.com ', + eraCommonsID: ' eraCommonsID ', + website: ' http://www.website.org ', + linkedin: ' http://www.linkedin.com/in/test ', + egoId: ' egoId ', + }; + + const persona = convertToPersona(input); + + expect(persona.firstName).toBe('firstName'); + expect(persona.lastName).toBe('lastName'); + expect(persona.loginEmail).toBe('login@email.com'); + expect(persona.institution).toBe('institution'); + expect(persona.state).toBe('state'); + expect(persona.country).toBe('country'); + expect(persona.institutionalEmail).toBe('institutional@email.com'); + expect(persona.eraCommonsID).toBe('eraCommonsID'); + expect(persona.website).toBe('http://www.website.org'); + expect(persona.linkedin).toBe('http://www.linkedin.com/in/test'); + expect(persona.egoId).toBe('egoId'); + }); + + it('should validate URL', () => { + const inputValid = { + ...emptyInput, + website: 'http://www.website.org', + }; + + const inputInvalid = { + ...emptyInput, + website: 'invalid_url', + }; + + const personaValidUrl = convertToPersona(inputValid); + const personaInvalidUrl = convertToPersona(inputInvalid); + + expect(personaValidUrl.website).toBe('http://www.website.org'); + expect(personaInvalidUrl.website).toBeNull(); + }); + + it('should validate emails', () => { + const inputValid = { + ...emptyInput, + loginEmail: 'login@email.com', + institutionalEmail: 'institutional@email.com', + }; + + const inputInvalid = { + ...emptyInput, + loginEmail: 'invalid_login_email', + institutionalEmail: 'invalid_institutional_email', + }; + + const personaValidEmails = convertToPersona(inputValid); + const personaInvalidEmails = convertToPersona(inputInvalid); + + expect(personaValidEmails.loginEmail).toBe('login@email.com'); + expect(personaValidEmails.institutionalEmail).toBe('institutional@email.com'); + expect(personaInvalidEmails.loginEmail).toBeNull(); + expect(personaInvalidEmails.institutionalEmail).toBeNull(); + }); + + it('should validate linkedin link', () => { + const inputValid = { + ...emptyInput, + linkedin: 'http://www.linkedin.com/in/test', + }; + + const inputInvalid = { + ...emptyInput, + linkedin: 'invalid_linkedin', + }; + + const personaValidLinkedin = convertToPersona(inputValid); + const personaInvalidLinkedin = convertToPersona(inputInvalid); + + expect(personaValidLinkedin.linkedin).toBe('http://www.linkedin.com/in/test'); + expect(personaInvalidLinkedin.linkedin).toBeNull(); + }); +}); diff --git a/src/external/persona.ts b/src/external/persona.ts new file mode 100644 index 0000000..5346ef5 --- /dev/null +++ b/src/external/persona.ts @@ -0,0 +1,212 @@ +/* eslint-disable no-console */ +import { parseString } from '@fast-csv/parse'; +import validator from 'validator'; + +import { personaURL } from '../config/env'; +import UserModel from '../db/models/User'; +import { LINKEDIN_REGEX } from '../utils/constants'; +import { SubscriptionStatus } from '../utils/newsletter'; + +export type Persona = { + id: string; + isPublic: boolean; + isActive: boolean; + title: string; + firstName: string; + lastName: string; + role: string[]; + loginEmail: string; + institution: string; + department: string; + jobTitle: string; + addressLine1: string; + addressLine2: string; + city: string; + state: string; + country: string; + phone: string; + institutionalEmail: string; + eraCommonsID: string; + website: string; + twitter: string; + orchid: string; + linkedin: string; + googleScholarId: string; + github: string; + facebook: string; + acceptedTerms: boolean; + acceptedNihOptIn: boolean; + acceptedKfOptIn: boolean; + acceptedDatasetSub: boolean; + interests: string[]; + egoId: string; + story: string; + bio: string; +}; + +export const validateUniqPersonas = (personas: Persona[]): boolean => { + const ids = personas.map((p) => p.egoId); + const uniqIds = new Set(ids); + return ids.length === uniqIds.size; +}; + +export const readCsv = (csvContent: string): Promise => + new Promise((resolve, reject) => { + const data = []; + + parseString(csvContent, { headers: true }) + .on('error', reject) + .on('data', (row) => { + const obj = convertToPersona(row); + if (obj) data.push(obj); + }) + .on('end', () => { + resolve(data); + }); + }); + +export const getUserList = async (accessToken: string): Promise => { + const uri = `${personaURL}/userlist`; + + const response = await fetch(encodeURI(uri), { + method: 'get', + headers: { + Authorization: accessToken, + 'Content-Type': 'text/csv', + }, + }); + + const body = await response.text(); + + if (response.status === 200) { + return body; + } + + throw new Error(`Error during call to Persona with status [${response.status}]`); +}; + +export const convertToPersona = (p: any): Persona => ({ + id: p.id, + isPublic: p.isPublic === 'true', + isActive: p.isActive === 'true', + title: p.title, + firstName: p.firstName === '' ? null : (p.firstName as string).trim(), + lastName: p.lastName === '' ? null : (p.lastName as string).trim(), + role: p.role === '' ? [] : p.role.split(','), + loginEmail: ((email) => { + if (email === '') return null; + const trimEmail = email.trim(); + return validator.isEmail(trimEmail) ? trimEmail : null; + })(p.loginEmail), + institution: p.institution === '' ? null : (p.institution as string).trim(), + department: p.department, + jobTitle: p.jobTitle, + addressLine1: p.addressLine1, + addressLine2: p.addressLine2, + city: p.city, + state: p.state === '' ? null : (p.state as string).trim(), + country: p.country === '' ? null : (p.country as string).trim(), + phone: p.phone, + institutionalEmail: ((email) => { + if (email === '') return null; + const trimEmail = email.trim(); + return validator.isEmail(trimEmail) ? trimEmail : null; + })(p.institutionalEmail), + eraCommonsID: p.eraCommonsID === '' ? null : (p.eraCommonsID as string).trim(), + website: ((website) => { + if (website === '') return null; + const trimWebsite = website.trim(); + return validator.isURL(trimWebsite) ? trimWebsite : null; + })(p.website), + twitter: p.twitter, + orchid: p.orchid, + linkedin: ((linkedin) => { + if (linkedin === '') return null; + const trimLinkedin = linkedin.trim(); + return LINKEDIN_REGEX.test(trimLinkedin) ? trimLinkedin : null; + })(p.linkedin), + googleScholarId: p.googleScholarId, + github: p.github, + facebook: p.facebook, + acceptedTerms: p.acceptedTerms === 'true', + acceptedNihOptIn: p.acceptedNihOptIn === 'true', + acceptedKfOptIn: p.acceptedKfOptIn === 'true', + acceptedDatasetSub: p.acceptedDatasetSub === 'true', + interests: p.interests === '' ? [] : p.interests.split(','), + egoId: p.egoId === '' ? null : (p.egoId as string).trim(), + story: p.story, + bio: p.bio, +}); + +export const createOrUpdate = async (payload: Persona): Promise => { + const existing = await UserModel.findAll({ where: { keycloak_id: payload.egoId } }); + try { + if (existing.length === 0) { + await UserModel.create({ + keycloak_id: payload.egoId, + first_name: payload.firstName, + last_name: payload.lastName, + era_commons_id: payload.eraCommonsID, + email: payload.loginEmail, + linkedin: payload.linkedin, + external_individual_email: payload.institutionalEmail, + roles: payload.role, + affiliation: payload.institution, + consent_date: new Date('2021-11-22'), + accepted_terms: payload.acceptedTerms, + completed_registration: payload.acceptedTerms && payload.isActive, + deleted: !payload.isActive, + newsletter_email: payload.acceptedKfOptIn || payload.acceptedDatasetSub ? payload.loginEmail : null, + newsletter_subscription_status: payload.acceptedKfOptIn + ? SubscriptionStatus.SUBSCRIBED + : SubscriptionStatus.UNSUBSCRIBED, + newsletter_dataset_subscription_status: payload.acceptedDatasetSub + ? SubscriptionStatus.SUBSCRIBED + : SubscriptionStatus.UNSUBSCRIBED, + location_country: payload.country, + location_state: payload.state, + website: payload.website, + areas_of_interest: payload.interests, + is_public: payload.isPublic, + }); + return 'created'; + } else if (existing.length === 1) { + await UserModel.update( + { + first_name: payload.firstName, + last_name: payload.lastName, + era_commons_id: payload.eraCommonsID, + email: payload.loginEmail, + linkedin: payload.linkedin, + external_individual_email: payload.institutionalEmail, + roles: payload.role, + affiliation: payload.institution, + consent_date: new Date('2021-11-22'), + accepted_terms: payload.acceptedTerms, + completed_registration: payload.acceptedTerms && payload.isActive, + deleted: !payload.isActive, + newsletter_email: payload.acceptedKfOptIn || payload.acceptedDatasetSub ? payload.loginEmail : null, + newsletter_subscription_status: payload.acceptedKfOptIn + ? SubscriptionStatus.SUBSCRIBED + : SubscriptionStatus.UNSUBSCRIBED, + newsletter_dataset_subscription_status: payload.acceptedDatasetSub + ? SubscriptionStatus.SUBSCRIBED + : SubscriptionStatus.UNSUBSCRIBED, + location_country: payload.country, + location_state: payload.state, + website: payload.website, + areas_of_interest: payload.interests, + is_public: payload.isPublic, + updated_date: new Date(), + }, + { where: { keycloak_id: payload.egoId } }, + ); + return 'updated'; + } else { + console.warn(`Duplicate user in users-api: ${payload.egoId}`); + } + } catch (e) { + console.error(`Error: ${e.message} - Ignoring [${payload.egoId}]`); + return 'ignored'; + } +}; diff --git a/src/routes/admin.ts b/src/routes/admin.ts index d49f73d..899c9e2 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -1,7 +1,10 @@ import { Router } from 'express'; import { StatusCodes } from 'http-status-codes'; +import { keycloakRealm } from '../config/env'; +import Realm from '../config/realm'; import { deleteUser, resetAllConsents } from '../db/dal/user'; +import { createOrUpdate, getUserList, readCsv, validateUniqPersonas } from '../external/persona'; // Handles requests made to /admin const adminRouter = Router(); @@ -25,4 +28,25 @@ adminRouter.put('/resetAllConsents', async (req, res, next) => { } }); +adminRouter.post('/doMigrationFromPersona', async (req, res, next) => { + try { + if (keycloakRealm !== Realm.KF) { + return res.status(StatusCodes.BAD_REQUEST).send('Not available for this project'); + } + const csvContent: string = await getUserList(req.headers.authorization); + const personas = await readCsv(csvContent); + if (validateUniqPersonas(personas)) { + const result = await Promise.all(personas.map((p) => createOrUpdate(p))); + return res.status(StatusCodes.OK).send({ + created: result.filter((s) => s === 'created').length, + updated: result.filter((s) => s === 'updated').length, + ignored: result.filter((s) => s === 'ignored').length, + }); + } + return res.status(StatusCodes.BAD_REQUEST).send('Duplicate persona, migration aborted'); + } catch (e) { + next(e); + } +}); + export default adminRouter; diff --git a/src/tests/userValidator.test.ts b/src/tests/userValidator.test.ts index 446a338..df56bb0 100644 --- a/src/tests/userValidator.test.ts +++ b/src/tests/userValidator.test.ts @@ -74,7 +74,6 @@ describe('User Validator', () => { }; const userWithEraCommonsId = { ...inputUser, era_commons_id: 'era_commons_id' }; - const userWithNihNedId = { ...inputUser, nih_ned_id: 'nih_ned_id' }; const userWithExternalIds = { ...inputUser, external_individual_fullname: 'external_individual_fullname', @@ -82,7 +81,6 @@ describe('User Validator', () => { }; expect(getUserValidator(Realm.INCLUDE)(userWithEraCommonsId)).toBeTruthy(); - expect(getUserValidator(Realm.INCLUDE)(userWithNihNedId)).toBeTruthy(); expect(getUserValidator(Realm.INCLUDE)(userWithExternalIds)).toBeTruthy(); }); diff --git a/src/utils/constants.ts b/src/utils/constants.ts index bf2f2fb..23273e3 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -1,4 +1,4 @@ export const UUID_VERSION = 4; -export const NAME_REGEX = /^[a-zà-ÿ ,.'-_]+$/iu; +export const NAME_REGEX = /^[\u0027-\u0029\u002F-\u0039\u0040\u0061-\u007A\u00C0-\uFFFF ,.'\-_]+$/iu; // see regex.test.ts to understand the regex export const LINKEDIN_REGEX = /^(http(s)?:\/\/)?([\w]+\.)?linkedin\.com\/(pub|in|profile)\/([-a-zA-Z0-9]+)\/*/iu; export const MAX_LENGTH_PER_ROLE = 100; diff --git a/src/utils/errors.ts b/src/utils/errors.ts index 9c38e05..762f72f 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -20,14 +20,6 @@ export const globalErrorHandler = (err: unknown, _req: Request, res: Response, _ res.status(err.status).json({ error: err.message, }); - } else if (err instanceof ValidationError) { - const error = { - name: 'Invalid data', - errors: err.errors.map((error) => error.message.replace('%s', error.path)), - }; - res.status(StatusCodes.UNPROCESSABLE_ENTITY).json({ - error, - }); } else if (err instanceof Error || err instanceof BaseError) { res.status(StatusCodes.INTERNAL_SERVER_ERROR).json({ error: getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR), diff --git a/src/utils/regex.test.ts b/src/utils/regex.test.ts new file mode 100644 index 0000000..a455472 --- /dev/null +++ b/src/utils/regex.test.ts @@ -0,0 +1,49 @@ +import { NAME_REGEX } from './constants'; + +describe('NAME_REGEX', () => { + it('should allow alphanumeric lower and uppercase', () => { + expect(NAME_REGEX.test('ThisIsAlphanumeric123')).toBeTruthy(); + }); + it('should allow acutes', () => { + expect(NAME_REGEX.test('éèîôàùêïöÀÈÉÎÔÙÊÏÖÜ')).toBeTruthy(); + }); + it(`should allow / @ ( ) ' - _ , . space`, () => { + expect(NAME_REGEX.test(`/ @ ( ) ' - _ , . `)).toBeTruthy(); + }); + it(`should allow non latin characters`, () => { + expect(NAME_REGEX.test(`abæcdöef`)).toBeTruthy(); + expect(NAME_REGEX.test(`правда`)).toBeTruthy(); + expect(NAME_REGEX.test(`ยจฆฟคฏข`)).toBeTruthy(); + expect(NAME_REGEX.test(`도메인`)).toBeTruthy(); + expect(NAME_REGEX.test(`ドメイン名例`)).toBeTruthy(); + expect(NAME_REGEX.test(`MajiでKoiする5秒前`)).toBeTruthy(); + expect(NAME_REGEX.test(`Krstanović Bezsažna Kołodziejczak Štepec`)).toBeTruthy(); + expect(NAME_REGEX.test(`הדר כהן`)).toBeTruthy(); + expect(NAME_REGEX.test(`Đỗ Ngọc Tuấn`)).toBeTruthy(); + expect(NAME_REGEX.test(`HALİL İBRAHİM`)).toBeTruthy(); + expect(NAME_REGEX.test(`博 林 석주 박 益颖 麦 宇鴻 台師大 藍`)).toBeTruthy(); + }); + it(`should not allow ; : { } ^ ! " # $ % & * + < = > ? [ ] \\ | ~`, () => { + expect(NAME_REGEX.test('a;a')).toBeFalsy(); + expect(NAME_REGEX.test('a:a')).toBeFalsy(); + expect(NAME_REGEX.test('a{a')).toBeFalsy(); + expect(NAME_REGEX.test('a}a')).toBeFalsy(); + expect(NAME_REGEX.test('a^a')).toBeFalsy(); + expect(NAME_REGEX.test('a!a')).toBeFalsy(); + expect(NAME_REGEX.test('a"a')).toBeFalsy(); + expect(NAME_REGEX.test('a#a')).toBeFalsy(); + expect(NAME_REGEX.test('a$a')).toBeFalsy(); + expect(NAME_REGEX.test('a%a')).toBeFalsy(); + expect(NAME_REGEX.test('a&a')).toBeFalsy(); + expect(NAME_REGEX.test('a*a')).toBeFalsy(); + expect(NAME_REGEX.test('a+a')).toBeFalsy(); + expect(NAME_REGEX.test('aa')).toBeFalsy(); + expect(NAME_REGEX.test('a?a')).toBeFalsy(); + expect(NAME_REGEX.test('a[a')).toBeFalsy(); + expect(NAME_REGEX.test('a]a')).toBeFalsy(); + expect(NAME_REGEX.test('a\\a')).toBeFalsy(); + expect(NAME_REGEX.test('a~a')).toBeFalsy(); + }); +}); diff --git a/src/utils/userValidator.ts b/src/utils/userValidator.ts index 8af670b..be6c181 100644 --- a/src/utils/userValidator.ts +++ b/src/utils/userValidator.ts @@ -6,9 +6,7 @@ export interface UserValidator { } const includeUserValidator = (payload: IUserInput): boolean => - (payload.era_commons_id || - payload.nih_ned_id || - (payload.external_individual_fullname && payload.external_individual_email)) && + (payload.era_commons_id || (payload.external_individual_fullname && payload.external_individual_email)) && payload.roles?.length && payload.portal_usages?.length > 0;