From d4e3c4d6a237cd7b6f6d86b6ae9f71e2b7e8f556 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 10 Jun 2024 17:32:30 +0200 Subject: [PATCH 1/3] bumping versions --- client/package.json | 14 +++++++------- server/package.json | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/client/package.json b/client/package.json index e31dd915..4ec63ac4 100644 --- a/client/package.json +++ b/client/package.json @@ -14,12 +14,12 @@ "dependencies": { "@creativebulma/bulma-tooltip": "*", "@femessage/log-viewer": "*", - "@fortawesome/fontawesome-svg-core": "~6.4.2", - "@fortawesome/free-brands-svg-icons": "~6.4.2", - "@fortawesome/free-regular-svg-icons": "~6.4.2", - "@fortawesome/free-solid-svg-icons": "~6.4.2", + "@fortawesome/fontawesome-svg-core": "~6.5.2", + "@fortawesome/free-brands-svg-icons": "~6.5.2", + "@fortawesome/free-regular-svg-icons": "~6.5.2", + "@fortawesome/free-solid-svg-icons": "~6.5.2", "@fortawesome/vue-fontawesome": "2.0.10", - "axios": "~1.6.8", + "axios": "~1.7.2", "brace": "~0.11.1", "bulma": "0.9.4", "bulma-calendar": "6.1.19", @@ -28,7 +28,7 @@ "bulma-quickview": "*", "bulmaswatch": "0.8.1", "copy-to-clipboard": "~3.3.3", - "core-js": "~3.36.1", + "core-js": "~3.37.1", "es6-promise": "~4.2.8", "highlight.js": "9.11.0", "jsonwebtoken": "^9.0.2", @@ -54,7 +54,7 @@ "babel-eslint": "~10.1.0", "eslint": "~6.8.0", "eslint-plugin-vue": "~6.2.2", - "nodemon": "~3.0.2", + "nodemon": "~3.1.3", "sass": "~1.49.11", "sass-loader": "10.1.1", "vue-template-compiler": "~2.6.11" diff --git a/server/package.json b/server/package.json index b8f05c7d..fea12980 100644 --- a/server/package.json +++ b/server/package.json @@ -20,18 +20,18 @@ "@outlinewiki/passport-azure-ad-oauth2": "~0.1.0", "ajv": "~6.12.6", "ajv-error-parser": "~1.0.7", - "axios": "~1.6.8", + "axios": "~1.7.2", "bcrypt": "~5.1.0", "bluebird": "~3.7.2", "cert-info": "~1.5.1", "cheerio": "~1.0.0-rc.12", "connect-history-api-fallback": "~2.0.0", - "core-js": "~3.36.1", + "core-js": "~3.37.1", "cors": "~2.8.5", "cron-parser": "~4.9.0", - "dayjs": "1.11.10", + "dayjs": "1.11.11", "express": "~4.19.2", - "express-session": "~1.18.0", + "cookie-session": "~2.1.0", "fs-extra": "~11.2.0", "ip": "2.0.1", "json-bigint": "~1.0.0", @@ -39,33 +39,33 @@ "lodash": "~4.17.21", "modern-passport-http": "~0.3.0", "moment": "~2.30.1", - "mongodb": "~6.5.0", + "mongodb": "~6.7.0", "mssql": "~10.0.2", "multer": "~1.4.5-lts.1", - "mysql2": "~3.9.4", + "mysql2": "~3.10.0", "node-cache": "~5.1.2", - "node-jq": "~4.3.0", + "node-jq": "~4.4.0", "nodemailer": "~6.9.8", "openid-client": "^5.6.5", "passport": "~0.7.0", "passport-jwt": "~4.0.1", - "pg": "~8.11.3", + "pg": "~8.12.0", "read-last-lines": "~1.8.0", - "swagger-ui-express": "~5.0.0", + "swagger-ui-express": "~5.0.1", "thenby": "~1.3.4", "winston": "~3.13.0", "winston-daily-rotate-file": "~5.0.0", "winston-syslog": "~2.7.0", - "yaml": "~2.3.4" + "yaml": "~2.4.5" }, "devDependencies": { - "@babel/cli": "~7.23.4", - "@babel/core": "7.23.7", - "@babel/eslint-parser": "7.23.3", - "@babel/node": "~7.22.19", + "@babel/cli": "~7.24.7", + "@babel/core": "7.24.7", + "@babel/eslint-parser": "7.24.7", + "@babel/node": "~7.24.7", "dotenv": "~16.4.1", "eslint": "~8.56.0", - "nodemon": "~3.0.3", + "nodemon": "~3.1.3", "npm-run-all": "*", "rifraf": "~2.0.3" }, From b17fa1b5acc6c9d9c36d21a575f0f593ce491d77 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 10 Jun 2024 17:32:51 +0200 Subject: [PATCH 2/3] wait for mysql --- server/src/configure.js | 244 ++++++++++++++++++++------------------- server/src/init/index.js | 140 ++++++++++++---------- 2 files changed, 207 insertions(+), 177 deletions(-) diff --git a/server/src/configure.js b/server/src/configure.js index 86624a32..148a97ff 100644 --- a/server/src/configure.js +++ b/server/src/configure.js @@ -6,7 +6,8 @@ if (process.env.NODE_ENV !== 'production' || process.env.FORCE_DOTENV==1 || proc } // express is the base http server for nodejs const express = require('express'); -const session = require('express-session'); +const session = require('cookie-session'); + // cors is a middleware to allow cross origin resource sharing // some routes/apis we will allow coming from other ip's/sources // for some internal apis we will not allow cors, all requests can only come @@ -29,121 +30,128 @@ const appConfig = require('../config/app.config') module.exports = app => { // first time run init - require('./init/') - - // passport - app.use(session({ - secret: 'AnsibleForms', - resave: false, - saveUninitialized: true - })); - app.use(passport.initialize()); - app.use(passport.session()); - - // we use 3 authentications/authorization strategies - // - basic : with username and password to get jwt tokens - // - azure-ad-oauth2 : microsoft login - // - jwt : to use the jwt tokens - // passport (the auth lib used) is smart, if basic authentication headers are detected - // then the basic authentication strategy kicks and the basic login procedure starts - require('./auth/auth_basic'); - require('./auth/auth_jwt'); - const auth_azuread = require('./auth/auth_azuread'); - auth_azuread.initialize() - - const auth_oidc = require('./auth/auth_oidc'); - auth_oidc.initialize() - - app.use(bodyParser.json({limit: '50mb'})); - app.use(bodyParser.urlencoded({limit: '50mb', extended: true})); - - // import api routes - const awxRoutes = require('./routes/awx.routes') - const jobRoutes = require('./routes/job.routes') - const queryRoutes = require('./routes/query.routes') - const expressionRoutes = require('./routes/expression.routes') - const userRoutes = require('./routes/user.routes') - const groupRoutes = require('./routes/group.routes') - const ldapRoutes = require('./routes/ldap.routes') - const azureadRoutes = require('./routes/azuread.routes') - const oidcRoutes = require('./routes/oidc.routes') - const settingsRoutes = require('./routes/settings.routes') - const credentialRoutes = require('./routes/credential.routes') - const loginRoutes = require('./routes/login.routes') - const schemaRoutes = require('./routes/schema.routes') - const tokenRoutes = require('./routes/token.routes') - const configRoutes = require('./routes/config.routes') - const versionRoutes = require('./routes/version.routes') - const lockRoutes = require('./routes/lock.routes') - const profileRoutes = require('./routes/profile.routes') - const sshRoutes = require('./routes/ssh.routes') - const logRoutes = require('./routes/log.routes') - const knownhostsRoutes = require('./routes/knownhosts.routes') - const helpRoutes = require('./routes/help.routes') - const installRoutes = require('./routes/install.routes') - const repositoryRoutes = require('./routes/repository.routes') - - // mysql2 has a bug that can throw an uncaught exception if the mysql server crashes (not enough mem for example) - // also git commands can chain child processes and cause issues - process.on('uncaughtException', function(err) { - // handle the error safely - console.error("An uncaught exception happened, ignore... ",err) -}) - - // using json web tokens as middleware - // the jwtauthentication strategy from passport (/auth/auth.js) - // is used as a middleware. Every route will check the token for validity - const authobj = passport.authenticate('jwt', { session: false }) - - // api routes for browser only (no cors) - const swaggerOptions = { - customSiteTitle: "Ansibleforms Swagger UI", - customfavIcon: `${appConfig.baseUrl}favicon.svg`, - customCssUrl: `${appConfig.baseUrl}assets/css/swagger.css`, - docExpansion:"none" - } - // change basePath dynamically - swaggerDocument.basePath = `${appConfig.baseUrl}api/v1` - app.use(`${appConfig.baseUrl}api-docs`, swaggerUi.serve, swaggerUi.setup(swaggerDocument,swaggerOptions)); - app.use(`${appConfig.baseUrl}api/v1/schema`, schemaRoutes) - - // api routes for querying - app.use(`${appConfig.baseUrl}api/v1/query`,cors(), authobj, queryRoutes) - app.use(`${appConfig.baseUrl}api/v1/expression`,cors(), authobj, expressionRoutes) - - // api route for version - app.use(`${appConfig.baseUrl}api/v1/version`,cors(), versionRoutes) - app.use(`${appConfig.baseUrl}api/v1/install`,cors(), installRoutes) - - app.use(`${appConfig.baseUrl}api/v1/lock`,cors(),authobj, lockRoutes) - app.use(`${appConfig.baseUrl}api/v1/help`,cors(),authobj, helpRoutes) - - // api route for profile - app.use(`${appConfig.baseUrl}api/v1/profile`,cors(), authobj, profileRoutes) - - // api routes for authorization - app.use(`${appConfig.baseUrl}api/v1/auth`,cors(), loginRoutes) - app.use(`${appConfig.baseUrl}api/v1/token`,cors(), tokenRoutes) - - // api routes for automation actions - - // app.use(`${appConfig.baseUrl}api/v1/multistep`,cors(), authobj, multistepRoutes) - - // api routes for admin management - app.use(`${appConfig.baseUrl}api/v1/job`,cors(), authobj, jobRoutes) - app.use(`${appConfig.baseUrl}api/v1/user`,cors(), authobj, checkAdminMiddleware, userRoutes) - app.use(`${appConfig.baseUrl}api/v1/group`,cors(), authobj, checkAdminMiddleware, groupRoutes) - app.use(`${appConfig.baseUrl}api/v1/ldap`,cors(), authobj, checkAdminMiddleware, ldapRoutes) - app.use(`${appConfig.baseUrl}api/v1/azuread`,cors(), authobj, checkAdminMiddleware, azureadRoutes) - app.use(`${appConfig.baseUrl}api/v1/oidc`,cors(), authobj, checkAdminMiddleware, oidcRoutes) - app.use(`${appConfig.baseUrl}api/v1/settings`,cors(), authobj, checkAdminMiddleware, settingsRoutes) - app.use(`${appConfig.baseUrl}api/v1/credential`,cors(), authobj, checkAdminMiddleware, credentialRoutes) - app.use(`${appConfig.baseUrl}api/v1/sshkey`,cors(), authobj, checkAdminMiddleware, sshRoutes) - app.use(`${appConfig.baseUrl}api/v1/awx`,cors(), authobj, checkAdminMiddleware, awxRoutes) - app.use(`${appConfig.baseUrl}api/v1/log`,cors(), authobj, checkAdminMiddleware, logRoutes) - app.use(`${appConfig.baseUrl}api/v1/repository`,cors(), authobj, checkAdminMiddleware, repositoryRoutes) - app.use(`${appConfig.baseUrl}api/v1/knownhosts`,cors(), authobj, checkAdminMiddleware, knownhostsRoutes) - - // routes for form config (extra middleware in the routes itself) - app.use(`${appConfig.baseUrl}api/v1/config`,cors(), authobj, configRoutes) + // from now on, it's async => we wait for mysql to be ready + const init = require('./init/') + init().then(()=>{ + + // passport + app.use(session({ + secret: 'AnsibleForms', + resave: false, + saveUninitialized: true + })); + app.use(passport.initialize()); + app.use(passport.session()); + + // we use 4 authentications/authorization strategies + // - basic : with username and password to get jwt tokens + // - azure-ad-oauth2 : microsoft login + // - oidc : open id connect + // - jwt : to use the jwt tokens + // passport (the auth lib used) is smart, if basic authentication headers are detected + // then the basic authentication strategy kicks and the basic login procedure starts + require('./auth/auth_basic'); + require('./auth/auth_jwt'); + const auth_azuread = require('./auth/auth_azuread'); + auth_azuread.initialize() + + const auth_oidc = require('./auth/auth_oidc'); + auth_oidc.initialize() + + app.use(bodyParser.json({limit: '50mb'})); + app.use(bodyParser.urlencoded({limit: '50mb', extended: true})); + + // import api routes + const awxRoutes = require('./routes/awx.routes') + const jobRoutes = require('./routes/job.routes') + const queryRoutes = require('./routes/query.routes') + const expressionRoutes = require('./routes/expression.routes') + const userRoutes = require('./routes/user.routes') + const groupRoutes = require('./routes/group.routes') + const ldapRoutes = require('./routes/ldap.routes') + const azureadRoutes = require('./routes/azuread.routes') + const oidcRoutes = require('./routes/oidc.routes') + const settingsRoutes = require('./routes/settings.routes') + const credentialRoutes = require('./routes/credential.routes') + const loginRoutes = require('./routes/login.routes') + const schemaRoutes = require('./routes/schema.routes') + const tokenRoutes = require('./routes/token.routes') + const configRoutes = require('./routes/config.routes') + const versionRoutes = require('./routes/version.routes') + const lockRoutes = require('./routes/lock.routes') + const profileRoutes = require('./routes/profile.routes') + const sshRoutes = require('./routes/ssh.routes') + const logRoutes = require('./routes/log.routes') + const knownhostsRoutes = require('./routes/knownhosts.routes') + const helpRoutes = require('./routes/help.routes') + const installRoutes = require('./routes/install.routes') + const repositoryRoutes = require('./routes/repository.routes') + + // mysql2 has a bug that can throw an uncaught exception if the mysql server crashes (not enough mem for example) + // also git commands can chain child processes and cause issues + process.on('uncaughtException', function(err) { + // handle the error safely + console.error("An uncaught exception happened, ignore... ",err) + }) + + // using json web tokens as middleware + // the jwtauthentication strategy from passport (/auth/auth.js) + // is used as a middleware. Every route will check the token for validity + const authobj = passport.authenticate('jwt', { session: false }) + + // api routes for browser only (no cors) + const swaggerOptions = { + customSiteTitle: "Ansibleforms Swagger UI", + customfavIcon: `${appConfig.baseUrl}favicon.svg`, + customCssUrl: `${appConfig.baseUrl}assets/css/swagger.css`, + docExpansion:"none" + } + // change basePath dynamically + swaggerDocument.basePath = `${appConfig.baseUrl}api/v1` + app.use(`${appConfig.baseUrl}api-docs`, swaggerUi.serve, swaggerUi.setup(swaggerDocument,swaggerOptions)); + app.use(`${appConfig.baseUrl}api/v1/schema`, schemaRoutes) + + // api routes for querying + app.use(`${appConfig.baseUrl}api/v1/query`,cors(), authobj, queryRoutes) + app.use(`${appConfig.baseUrl}api/v1/expression`,cors(), authobj, expressionRoutes) + + // api route for version + app.use(`${appConfig.baseUrl}api/v1/version`,cors(), versionRoutes) + app.use(`${appConfig.baseUrl}api/v1/install`,cors(), installRoutes) + + app.use(`${appConfig.baseUrl}api/v1/lock`,cors(),authobj, lockRoutes) + app.use(`${appConfig.baseUrl}api/v1/help`,cors(),authobj, helpRoutes) + + // api route for profile + app.use(`${appConfig.baseUrl}api/v1/profile`,cors(), authobj, profileRoutes) + + // api routes for authorization + app.use(`${appConfig.baseUrl}api/v1/auth`,cors(), loginRoutes) + app.use(`${appConfig.baseUrl}api/v1/token`,cors(), tokenRoutes) + + // api routes for automation actions + + // app.use(`${appConfig.baseUrl}api/v1/multistep`,cors(), authobj, multistepRoutes) + + // api routes for admin management + app.use(`${appConfig.baseUrl}api/v1/job`,cors(), authobj, jobRoutes) + app.use(`${appConfig.baseUrl}api/v1/user`,cors(), authobj, checkAdminMiddleware, userRoutes) + app.use(`${appConfig.baseUrl}api/v1/group`,cors(), authobj, checkAdminMiddleware, groupRoutes) + app.use(`${appConfig.baseUrl}api/v1/ldap`,cors(), authobj, checkAdminMiddleware, ldapRoutes) + app.use(`${appConfig.baseUrl}api/v1/azuread`,cors(), authobj, checkAdminMiddleware, azureadRoutes) + app.use(`${appConfig.baseUrl}api/v1/oidc`,cors(), authobj, checkAdminMiddleware, oidcRoutes) + app.use(`${appConfig.baseUrl}api/v1/settings`,cors(), authobj, checkAdminMiddleware, settingsRoutes) + app.use(`${appConfig.baseUrl}api/v1/credential`,cors(), authobj, checkAdminMiddleware, credentialRoutes) + app.use(`${appConfig.baseUrl}api/v1/sshkey`,cors(), authobj, checkAdminMiddleware, sshRoutes) + app.use(`${appConfig.baseUrl}api/v1/awx`,cors(), authobj, checkAdminMiddleware, awxRoutes) + app.use(`${appConfig.baseUrl}api/v1/log`,cors(), authobj, checkAdminMiddleware, logRoutes) + app.use(`${appConfig.baseUrl}api/v1/repository`,cors(), authobj, checkAdminMiddleware, repositoryRoutes) + app.use(`${appConfig.baseUrl}api/v1/knownhosts`,cors(), authobj, checkAdminMiddleware, knownhostsRoutes) + + // routes for form config (extra middleware in the routes itself) + app.use(`${appConfig.baseUrl}api/v1/config`,cors(), authobj, configRoutes) + + }) + + } diff --git a/server/src/init/index.js b/server/src/init/index.js index 2b8e3dcc..d0427c48 100644 --- a/server/src/init/index.js +++ b/server/src/init/index.js @@ -1,71 +1,93 @@ -const logger=require("../lib/logger"); -var Ssh = require('../models/ssh.model'); -var Form = require('../models/form.model'); -var Job = require('../models/job.model'); -const mysql=require("../models/db.model"); -const Repository = require('../models/repository.model'); -const parser = require("cron-parser") -const dayjs = require("dayjs") +async function init(){ -Ssh.generate(false) - .catch((err)=>{ - logger.warning("Failed to generate ssh keys : " + err) - }) -Form.initBackupFolder() + const logger=require("../lib/logger"); + var Ssh = require('../models/ssh.model'); + var Form = require('../models/form.model'); + var Job = require('../models/job.model'); + const mysql=require("../models/db.model"); + const Repository = require('../models/repository.model'); + const parser = require("cron-parser") + const dayjs = require("dayjs") -// this is at startup, abandon all running jobs, pointless to not do it. -Job.abandon(true) -.then((changed)=>{ - logger.warning(`Abandoned ${changed} jobs`) -}) -.catch((err)=>{ - logger.error("Failed to abandon jobs : " + err) -}) + // this is at startup, don't start the app until mysql is ready + // rewrite with await + console.log("Waiting for mysql to start") + async function sleep(millis) { + return new Promise(resolve => setTimeout(resolve, millis)); + } + var MYSQL_IS_READY = false + while(!MYSQL_IS_READY){ + try{ + await mysql.do("SELECT 1") + MYSQL_IS_READY = true + }catch(e){ + console.log("Mysql not ready yet") + await sleep(5000) + } + } -logger.info("Initializing hourly abandoned jobs timer") -// this is hourly, abandon running jobs older than a day. -setInterval(()=>{ - Job.abandon() - .then((changed)=>{ - logger.warning(`Abandoned ${changed} jobs`) - }) + Ssh.generate(false) .catch((err)=>{ - logger.error("Failed to abandon jobs : " + err) + logger.warning("Failed to generate ssh keys : " + err) }) -},3600000) + Form.initBackupFolder() -logger.info("Pulling repositories") -mysql.do("SELECT name FROM AnsibleForms.`repositories` WHERE rebase_on_start=1") -.then((repositories)=>{ - repositories.map((repo)=>{ - logger.info("Pulling " + repo.name) - Repository.clone(repo.name).catch((e)=>{}) + Job.abandon(true) + .then((changed)=>{ + logger.warning(`Abandoned ${changed} jobs`) + }) + .catch((err)=>{ + logger.error("Failed to abandon jobs : " + err) }) -}) -.catch((e)=>{}) -logger.info("Initializing repository cron schedules") -// this is hourly, abandon running jobs older than a day. -setInterval(()=>{ - mysql.do("SELECT name,cron FROM AnsibleForms.`repositories` WHERE status<>'running' AND cron<>''",undefined,true) + logger.info("Initializing hourly abandoned jobs timer") + // this is hourly, abandon running jobs older than a day. + setInterval(()=>{ + Job.abandon() + .then((changed)=>{ + logger.warning(`Abandoned ${changed} jobs`) + }) + .catch((err)=>{ + logger.error("Failed to abandon jobs : " + err) + }) + },3600000) + + logger.info("Pulling repositories") + mysql.do("SELECT name FROM AnsibleForms.`repositories` WHERE rebase_on_start=1") .then((repositories)=>{ repositories.map((repo)=>{ - try{ - const interval = parser.parseExpression(repo.cron) - const next = interval.next().toDate() - const date = dayjs(next) - const now = dayjs() - const minutes = date.diff(now,'m') - if(minutes==0){ - Repository.pull(repo.name) - }else{ - // logger.debug(`Not time yet, ${minutes} minutes to go`) - } - }catch(e){ - logger.error(`Failed to parse cron schedule ${repo.cron}`) - } - + logger.info("Pulling " + repo.name) + Repository.clone(repo.name).catch((e)=>{}) }) }) - .catch((e)=>{}) -},56000) // run every 55 second, should hit 0 minutes once \ No newline at end of file + .catch((e)=>{}) + + logger.info("Initializing repository cron schedules") + // this is hourly, abandon running jobs older than a day. + setInterval(()=>{ + mysql.do("SELECT name,cron FROM AnsibleForms.`repositories` WHERE status<>'running' AND cron<>''",undefined,true) + .then((repositories)=>{ + repositories.map((repo)=>{ + try{ + const interval = parser.parseExpression(repo.cron) + const next = interval.next().toDate() + const date = dayjs(next) + const now = dayjs() + const minutes = date.diff(now,'m') + if(minutes==0){ + Repository.pull(repo.name) + }else{ + // logger.debug(`Not time yet, ${minutes} minutes to go`) + } + }catch(e){ + logger.error(`Failed to parse cron schedule ${repo.cron}`) + } + + }) + }) + .catch((e)=>{}) + },56000) // run every 55 second, should hit 0 minutes once + +} + +module.exports = init \ No newline at end of file From 6de8e62e9fa1b30035b3365647eec3d6a6f859ac Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 10 Jun 2024 17:34:10 +0200 Subject: [PATCH 3/3] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b56f6c81..87c7e805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ytt implementation to template yaml files (credits mdaugs) - Vault credentials, pass a vault password to ansible playbook. - OIDC authentication (credits mdaugs) +- Ansibleforms will now wait for mysql to be ready before initializing ### Changed @@ -24,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - radio button errors - some issue with the designer when a field without name was added - multistep was always successfull (tx to mdaugs) +- using cookie session instead express session ## [5.0.1] - 2024-04-10