fix: fix db setup in tests (#458)

* feat: initially add seed and migration files

* feat: place seeds and migrations in folders

* feat: remove not used inserts

* fix: fix adoption code test

* fix: remove init mariadb script from gh ci

* feat: add node env check

* fix: use migration both in tests and in docker compose

* refactor: use sql connection as object

* feat: separate dev and test dbs

* fix: rename migration script

* fix: fix docker compose file

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
         ports:
           - 3306:3306
         env:
-          MARIADB_DATABASE: directus
+          MARIADB_DATABASE: directus-test
           MARIADB_USER: directus
           MARIADB_PASSWORD: password
           MARIADB_RANDOM_ROOT_PASSWORD: 1
@@ -41,10 +41,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Init MariaDB
-        run: |
-          mysql -u directus -ppassword --database=directus --protocol=tcp < config/init.sql
-
       - uses: actions/setup-node@v3
         with:
           node-version: 18.x

config/create-dbs.sql

@@ -0,0 +1,5 @@
+CREATE DATABASE IF NOT EXISTS directus;
+GRANT ALL PRIVILEGES ON directus.* to 'directus'@'%';
+
+CREATE DATABASE IF NOT EXISTS `directus-test`;
+GRANT ALL PRIVILEGES ON `directus-test`.* to 'directus'@'%';

config/default.cjs

@@ -13,7 +13,13 @@ module.exports = {
 	},
 	db: {
 		type: 'mysql',
-		connection: 'mysql://directus:password@localhost:3306/directus',
+		connection: {
+			host: 'localhost',
+			user: 'directus',
+			password: 'password',
+			database: 'directus',
+			port: 3306,
+		},
 	},
 	admin: {
 		key: '',

config/test.cjs

@@ -4,6 +4,12 @@ module.exports = {
 			tls: false,
 		},
 	},
+	db: {
+		connection: {
+			database: 'directus-test',
+			multipleStatements: true,
+		},
+	},
 	admin: {
 		key: 'admin',
 	},

docker-compose.yml

@@ -19,4 +19,5 @@ services:
     ports:
       - "3306:3306"
     volumes:
-      - ./config/init.sql:/docker-entrypoint-initdb.d/init.sql
+      - ./config/create-dbs.sql:/docker-entrypoint-initdb.d/01-create-dbs.sql
+      - ./migrations/create-tables.js.sql:/docker-entrypoint-initdb.d/02-create-tables.sql

knexfile.js

@@ -18,6 +18,9 @@ export default _.merge({}, ...[ 'development', 'production', 'staging', 'test' ]
 				propagateCreateError: false,
 			},
 			acquireConnectionTimeout: 10000,
+			seeds: {
+				directory: `./seeds/${environment}`,
+			},
 		},
 	};
 }));

migrations/create-tables.js

@@ -0,0 +1,11 @@
+import fs from 'node:fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+export const up = async (db) => {
+	const __filename = fileURLToPath(import.meta.url);
+	const query = fs.readFileSync(path.join(__filename + '.sql'), 'utf8');
+	await db.schema.raw(query);
+};
+
+export const down = () => {};

config/init.sql → migrations/create-tables.js.sql

@@ -1,6 +1,3 @@
-CREATE DATABASE IF NOT EXISTS directus;
-USE directus;
-
 CREATE TABLE IF NOT EXISTS adopted_probes (
   id INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
   user_created CHAR(36),
@@ -11,16 +8,16 @@ CREATE TABLE IF NOT EXISTS adopted_probes (
   ip VARCHAR(255) NOT NULL,
   uuid VARCHAR(255),
   lastSyncDate DATE NOT NULL,
-	isCustomCity TINYINT DEFAULT 0,
-	tags LONGTEXT,
+  isCustomCity TINYINT DEFAULT 0,
+  tags LONGTEXT,
   status VARCHAR(255) NOT NULL,
   version VARCHAR(255) NOT NULL,
   country VARCHAR(255) NOT NULL,
   city VARCHAR(255),
   state VARCHAR(255),
-	latitude FLOAT(10, 5),
-	longitude FLOAT(10, 5),
-	asn INTEGER NOT NULL,
+  latitude FLOAT(10, 5),
+  longitude FLOAT(10, 5),
+  asn INTEGER NOT NULL,
   network VARCHAR(255) NOT NULL,
   countryOfCustomCity VARCHAR(255)
 );
@@ -37,45 +34,3 @@ CREATE TABLE IF NOT EXISTS directus_notifications (
   subject VARCHAR(255),
   message TEXT
 );
-
-INSERT IGNORE INTO adopted_probes (
-	userId,
-	lastSyncDate,
-	ip,
-	uuid,
-	isCustomCity,
-	tags,
-	status,
-	version,
-	country,
-	city,
-	latitude,
-	longitude,
-	network,
-	asn,
-	countryOfCustomCity
-) VALUES (
-	'89da69bd-a236-4ab7-9c5d-b5f52ce09959',
-	CURRENT_DATE,
-	'51.158.22.211',
-	'c77f021d-23ff-440a-aa96-35e82c73e731',
-	1,
-	'["mytag1"]',
-	'ready',
-	'0.26.0',
-	'FR',
-	'Marseille',
-	'43.29695',
-	'5.38107',
-	'SCALEWAY S.A.S.',
-	12876,
-	'FR'
-);
-
-INSERT IGNORE INTO directus_users (
-	id,
-	github
-) VALUES (
-	'89da69bd-a236-4ab7-9c5d-b5f52ce09959',
-	'jimaek'
-);

seeds/test/index.js

@@ -0,0 +1,6 @@
+export const seed = async (db) => {
+	await db('directus_users').insert({
+		id: '89da69bd-a236-4ab7-9c5d-b5f52ce09959',
+		github: 'jimaek',
+	});
+};

src/lib/adopted-probes.ts

@@ -276,8 +276,8 @@ export class AdoptedProbes {
 
 	private async sendNotification (adoptedProbe: AdoptedProbe, connectedProbe: Probe) {
 		await this.sql.raw(`
-			INSERT INTO directus_notifications (recipient, subject, message) SELECT :recipient, :subject, :message
-			WHERE NOT EXISTS (SELECT 1 FROM directus_notifications WHERE recipient = :recipient AND message = :message AND DATE(timestamp) = CURRENT_DATE)
+			INSERT INTO ${NOTIFICATIONS_TABLE} (recipient, subject, message) SELECT :recipient, :subject, :message
+			WHERE NOT EXISTS (SELECT 1 FROM ${NOTIFICATIONS_TABLE} WHERE recipient = :recipient AND message = :message AND DATE(timestamp) = CURRENT_DATE)
 		`, {
 			recipient: adoptedProbe.userId,
 			subject: 'Adopted probe country change',

test/setup.ts

@@ -1,7 +1,10 @@
+import config from 'config';
+import Bluebird from 'bluebird';
 import chai from 'chai';
 import nock from 'nock';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
+import type { Knex } from 'knex';
 
 import { populateMemList } from '../src/lib/geoip/whitelist.js';
 import {
@@ -10,24 +13,41 @@ import {
 	populateIpRangeList,
 	populateNockCitiesList,
 } from './utils/populate-static-files.js';
-
 import chaiOas from './plugins/oas/index.js';
 import { getRedisClient, initRedis } from '../src/lib/redis/client.js';
-import { client } from '../src/lib/sql/client.js';
-import { USERS_TABLE } from '../src/lib/adopted-probes.js';
+import { client as sql } from '../src/lib/sql/client.js';
+
+const dbConfig = config.get<{ connection: { database: string, host: string } }>('db');
+
+if (!dbConfig.connection.database.endsWith('-test') && dbConfig.connection.host !== 'localhost') {
+	throw new Error(`Database name for test env needs to end with "-test" or the host must be "localhost". Got "${dbConfig.connection.database}"@"${dbConfig.connection.host}".`);
+}
 
 before(async () => {
 	chai.use(await chaiOas({ specPath: path.join(fileURLToPath(new URL('.', import.meta.url)), '../public/v1/spec.yaml') }));
+
 	await initRedis();
 	const redis = getRedisClient();
 	await redis.flushDb();
-	await client(USERS_TABLE).insert({ id: '89da69bd-a236-4ab7-9c5d-b5f52ce09959', github: 'jimaek' }).onConflict().ignore();
+
+	await dropAllTables(sql);
+	await sql.migrate.latest();
+	await sql.seed.run();
 
 	nock.disableNetConnect();
 	nock.enableNetConnect('127.0.0.1');
+
 	await populateIpList();
 	await populateDomainList();
 	await populateIpRangeList();
 	await populateMemList();
 	await populateNockCitiesList();
 });
+
+const dropAllTables = async (sql: Knex) => {
+	const allTables = (await sql('information_schema.tables')
+		.whereRaw(`table_schema = database()`)
+		.select(`table_name as table`)
+	).map(({ table }: { table: string }) => table);
+	await Bluebird.map(allTables, table => sql.schema.raw(`drop table \`${table}\``));
+};

test/tests/integration/adoption-code.test.ts

@@ -1,18 +1,21 @@
+import type { Server } from 'node:http';
 import nock from 'nock';
 import { expect } from 'chai';
 import * as sinon from 'sinon';
 import type { Socket } from 'socket.io-client';
-import request from 'supertest';
+import request, { type SuperTest, type Test } from 'supertest';
 import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../utils/server.js';
 import nockGeoIpProviders from '../../utils/nock-geo-ip.js';
 
-let probe: Socket;
-const app = await getTestServer();
-const requestAgent = request(app);
-const adoptionCodeStub = sinon.stub();
-
 describe('Adoption code', () => {
+	let app: Server;
+	let requestAgent: SuperTest<Test>;
+	let probe: Socket;
+	const adoptionCodeStub = sinon.stub();
+
 	before(async () => {
+		app = await getTestServer();
+		requestAgent = request(app);
 		nockGeoIpProviders();
 
 		probe = await addFakeProbe({