From 931764900717800b42119dfd7cde99b817f4667c Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 30 Nov 2023 14:12:53 +0100 Subject: [PATCH 01/50] add jobid to extravars --- client/src/components/Form.vue | 36 ++++++++------ docs/_data/help.yaml | 87 ++++++++++++++++++++------------- server/schema/forms_schema.json | 3 -- server/src/models/job.model.js | 16 ++++-- 4 files changed, 86 insertions(+), 56 deletions(-) diff --git a/client/src/components/Form.vue b/client/src/components/Form.vue index 18c26dd4..72028947 100644 --- a/client/src/components/Form.vue +++ b/client/src/components/Form.vue @@ -318,6 +318,7 @@ v-bind="field.attrs" :required="field.required" :type="field.type" + :readonly="field.hide" :placeholder="field.placeholder" @keydown="(field.keydown)?evaluateDynamicFields(field.name):null" @change="evaluateDynamicFields(field.name)"> @@ -722,7 +723,7 @@ this.clip(url,true) }, // to execute form events - doAction(a){ + doAction(a,jobid){ var ref=this const action=Object.keys(a)[0] const value=a[action] @@ -745,7 +746,7 @@ if(action=="load"){ setTimeout(()=>{ ref.reloadForm() - ref.$router.replace({name:"Form",query:{form:form}}).catch(err => {}); + ref.$router.replace({name:"Form",query:{form:form,"__previous_jobid__":jobid}}).catch(err => {}); },wait*1000) } if(action=="reload"){ @@ -1746,22 +1747,22 @@ // final result if(ref.currentForm.onFinish){ ref.currentForm.onFinish.forEach((action, i) => { - ref.doAction(action); + ref.doAction(action,id); }); } if(this.jobResult.status=="success" && ref.currentForm.onSuccess){ ref.currentForm.onSuccess.forEach((action, i) => { - ref.doAction(action); + ref.doAction(action,id); }); } if(this.jobResult.status=="error" && ref.currentForm.onFailure){ ref.currentForm.onFailure.forEach((action, i) => { - ref.doAction(action); + ref.doAction(action,id); }); } if(this.jobResult.status=="warning" && ref.currentForm.onAbort){ ref.currentForm.onAbort.forEach((action, i) => { - ref.doAction(action); + ref.doAction(action,id); }); } this.abortTriggered=false @@ -1808,15 +1809,18 @@ field = [].concat(field ?? []); // force to array } if(field.length>0){ // not emtpy - if(typeof field[0]==="object"){ // array of objects, analyse first object - keys = Object.keys(field[0]) // get properties - if(keys.length>0){ - key = (keys.includes(column))?column:keys[0] // get column, fall back to first - field=field.map((item) => ((item)?((item[key]==null)?null:(item[key]??item)):undefined)) // flatten array - }else{ - field=(!keepArray)?undefined:field // force undefined if we don't want arrays - } - } // no else, array is already flattened + if(column!="*"){ + if(typeof field[0]==="object"){ // array of objects, analyse first object + keys = Object.keys(field[0]) // get properties + if(keys.length>0){ + key = (keys.includes(column))?column:keys[0] // get column, fall back to first + field=field.map((item) => ((item)?((item[key]==null)?null:(item[key]??item)):undefined)) // flatten array + }else{ + field=(!keepArray)?undefined:field // force undefined if we don't want arrays + } + } // no else, array is already flattened + } + field=(!wasArray || !keepArray)?field[0]:field // if it wasn't an array, we take first again }else{ field=(!keepArray)?undefined:field // force undefined if we don't want arrays @@ -2007,7 +2011,7 @@ this.jobResult.data.output = "" if(ref.currentForm.onSubmit){ ref.currentForm.onSubmit.forEach((action, i) => { - ref.doAction(action); + ref.doAction(action,jobid); }); } // wait for 2 seconds, and get the output of the job diff --git a/docs/_data/help.yaml b/docs/_data/help.yaml index 4483ef17..6ed0bf0d 100644 --- a/docs/_data/help.yaml +++ b/docs/_data/help.yaml @@ -1206,11 +1206,11 @@ version: 3.1.0 description: | Once a job is launched, Ansible Forms has a few status events you can act on. - * **onSubmit** : Triggered the moment the submit button is pressed - * **onSuccess** : Triggered when the job is finished succesfully - * **onFailure** : Triggered when the job was not succesful - * **onAbort** : Triggered when the job was aborted - * **onFinish** : Triggered when the job is finished, regardless the status + * **onSubmit** : Triggered the moment the submit button is pressed + * **onSuccess** : Triggered when the job is finished succesfully + * **onFailure** : Triggered when the job was not succesful + * **onAbort** : Triggered when the job was aborted + * **onFinish** : Triggered when the job is finished, regardless the status Follow the link above to learn about the possible job status actions examples: @@ -1230,11 +1230,11 @@ version: 3.1.0 description: | Once a job is launched, Ansible Forms has a few status events you can act on. - * **onSubmit** : Triggered the moment the submit button is pressed - * **onSuccess** : Triggered when the job is finished succesfully - * **onFailure** : Triggered when the job was not succesful - * **onAbort** : Triggered when the job was aborted - * **onFinish** : Triggered when the job is finished, regardless the status + * **onSubmit** : Triggered the moment the submit button is pressed + * **onSuccess** : Triggered when the job is finished succesfully + * **onFailure** : Triggered when the job was not succesful + * **onAbort** : Triggered when the job was aborted + * **onFinish** : Triggered when the job is finished, regardless the status Follow the link above to learn about the possible job status actions examples: @@ -1253,11 +1253,11 @@ version: 3.1.0 description: | Once a job is launched, Ansible Forms has a few status events you can act on. - * **onSubmit** : Triggered the moment the submit button is pressed - * **onSuccess** : Triggered when the job is finished succesfully - * **onFailure** : Triggered when the job was not succesful - * **onAbort** : Triggered when the job was aborted - * **onFinish** : Triggered when the job is finished, regardless the status + * **onSubmit** : Triggered the moment the submit button is pressed + * **onSuccess** : Triggered when the job is finished succesfully + * **onFailure** : Triggered when the job was not succesful + * **onAbort** : Triggered when the job was aborted + * **onFinish** : Triggered when the job is finished, regardless the status Follow the link above to learn about the possible job status actions examples: @@ -1276,11 +1276,11 @@ version: 3.1.0 description: | Once a job is launched, Ansible Forms has a few status events you can act on. - * **onSubmit** : Triggered the moment the submit button is pressed - * **onSuccess** : Triggered when the job is finished succesfully - * **onFailure** : Triggered when the job was not succesful - * **onAbort** : Triggered when the job was aborted - * **onFinish** : Triggered when the job is finished, regardless the status + * **onSubmit** : Triggered the moment the submit button is pressed + * **onSuccess** : Triggered when the job is finished succesfully + * **onFailure** : Triggered when the job was not succesful + * **onAbort** : Triggered when the job was aborted + * **onFinish** : Triggered when the job is finished, regardless the status Follow the link above to learn about the possible job status actions examples: @@ -1299,11 +1299,11 @@ version: 3.1.0 description: | Once a job is launched, Ansible Forms has a few status events you can act on. - * **onSubmit** : Triggered the moment the submit button is pressed - * **onSuccess** : Triggered when the job is finished succesfully - * **onFailure** : Triggered when the job was not succesful - * **onAbort** : Triggered when the job was aborted - * **onFinish** : Triggered when the job is finished, regardless the status + * **onSubmit** : Triggered the moment the submit button is pressed + * **onSuccess** : Triggered when the job is finished succesfully + * **onFailure** : Triggered when the job was not succesful + * **onAbort** : Triggered when the job was aborted + * **onFinish** : Triggered when the job is finished, regardless the status Follow the link above to learn about the possible job status actions examples: @@ -2577,7 +2577,9 @@ description: | When selecting from an `enum`-field (dropdown), either the first column or the column defined in `valueColumn` is sent as an extravar. - If you want the full selected record, use the property. + If you want the full selected record, use the property. + + **New in v4.0.20** : valueColumn: "*" has the same behavior, and will output all columns with_types: enum - name: expression type: string @@ -2836,6 +2838,8 @@ When you select an item in an `enum` field, by default the first column is used to send as extravar. By setting this property, you can change it to another field. This setting is ignored when `outputObject` is true. + + **new in v4.0.20** : setting valueColumn to "*" will export all columns examples: - name: Export name code: | @@ -2847,8 +2851,7 @@ query: SELECT name, username FROM users columns: - name - valueColumn: - - username + valueColumn: username - name: placeholderColumn type: string group: data @@ -2860,7 +2863,9 @@ by default the first column is used as a value. By setting this property you change the column to be used. **NOTE** : this property is somewhat out-of-date since the introduction of the dot-notation. - See examples below. + See examples below. + + **New in v4.0.20** : setting placeholderColumn to "*" will output the entire record, instead of a single column. examples: - name: Export name code: | @@ -3237,7 +3242,9 @@ description: | When selecting from an `enum` field (dropdown), either the first column or the column defined in `valueColumn` is sent as an extravar. - If you want the full selected record, use the property. + If you want the full selected record, use the property. + + **new in v4.0.20** : you can also use valueColumn: "*" to export all columns with_types: enum - name: columns type: array @@ -3314,6 +3321,8 @@ When you select an item in an `enum` field, by default the first column is used to send as extravar. By setting this property, you can change it to another field. This setting is ignored when `outputObject` is true. + + **new in v4.0.20** : setting value "*" will export all columns. examples: - name: Export name code: | @@ -3325,8 +3334,7 @@ query: SELECT name, username FROM users columns: - name - valueColumn: - - username + valueColumn: username - name: previewColumn type: string default: first column @@ -3526,7 +3534,12 @@ allowed: number_of_seconds,name_of_form short: Load a form description: | - This action allows you to load another form + This action allows you to load another form. + + **new in v4.0.20** : the previous jobid is sent as \_\_previous\_\_jobid\_\_. + You must make sure you have this field in your form, it will be populated with the previous jobid. + + This is particularly interesting to grab information from the previous job (for example, load output, ...) examples: - name: Clear and hide on submit language: YAML @@ -3971,6 +3984,7 @@ The power of this concept lies in the client web-application that is re-evaluating fields every 100ms. With current processors and the chromium engine, this should be a very seamless experience. **Note**: When you reference another enum field, you reference the selected values, NOT the full dropdown list. Use the `placeholderColumn`-property or a dot-notation like `$(city.name)` + **New in v4.0.20** : setting placeholderColumn to "*" will output the entire record, instead of a single column. language: yaml code: | - type: enum @@ -4046,6 +4060,7 @@ If the placeholder is pointing to an expression field, then either the full object is returned or you can have an advanced placeholder reference like `$(myarray[0].name)` where you can create javascript-like references. **Note**: Important to know is that the placeholder is replaced BEFORE the evaluation of the expression. If you expect the result to be a string, then you must wrap it with quotes!. + **New in v4.0.20** : setting placeholderColumn to "*" will output the entire record, instead of a single column. language: yaml code: | - name: field1 @@ -4129,6 +4144,12 @@ runLocal: true expression: "{vc_cred: 'vcenter',ad_cred: 'ad',veeam_cred:'$(veeam_server)'}" # note : in the expression you can use placeholders to make them dynamic + - name: Jobid + short: Pass the current jobid + description: | + Ansible Forms automatically sends the current jobid in the extravars. + You don't need to do anything. + It is sent as `__jobid__`. - name: Userinfo short: Pass the current user description: | diff --git a/server/schema/forms_schema.json b/server/schema/forms_schema.json index 27551412..fcdae69e 100644 --- a/server/schema/forms_schema.json +++ b/server/schema/forms_schema.json @@ -566,7 +566,6 @@ "editable": {"not":{}}, "pctColumns": {"not":{}}, "filterColumns": {"not":{}}, - "hide": {"not":{}}, "allowDelete": {"not":{}}, "allowInsert": {"not":{}}, "deleteMarker": {"not":{}}, @@ -737,8 +736,6 @@ "minLength": {"not":{}}, "maxLength": {"not":{}}, "editable": {"not":{}}, - "pctColumns": {"not":{}}, - "filterColumns": {"not":{}}, "expression": {"not":{}}, "runLocal": {"not":{}}, "keydown": {"not":{}}, diff --git a/server/src/models/job.model.js b/server/src/models/job.model.js index 57aa628f..05afea2c 100644 --- a/server/src/models/job.model.js +++ b/server/src/models/job.model.js @@ -327,6 +327,7 @@ Job.launch = async function(form,formObj,user,creds,extravars,parentId=null,next } logger.debug(`Job id ${jobid} is created`) + extravars["__jobid__"]=jobid // job created - return to client if(next)next({id:jobid}) @@ -425,6 +426,7 @@ Job.continue = async function(form,user,creds,extravars,jobid,next) { } pushForminfoToExtravars(formObj,extravars,creds) + extravars["__jobid__"]=jobid // console.log(formObj) // we have form and we have access @@ -1048,8 +1050,13 @@ Awx.launchTemplate = async function (template,ev,invent,tags,limit,check,diff,ve counter=0 } // get existing credentials in the template, and then add the external ones. - var awxCredentialList=await Awx.findCredentialsByTemplate(template.id) - logger.notice(`Found ${awxCredentialList.length} existing creds`) + var awxCredentialList=[] + try{ + awxCredentialList=await Awx.findCredentialsByTemplate(template.id) + logger.notice(`Found ${awxCredentialList.length} existing creds`) + }catch(e){ + logger.warning("No credentials available... could be workflow template") + } // add external ones for(let i=0;i Date: Thu, 30 Nov 2023 14:13:45 +0100 Subject: [PATCH 02/50] better error handling in functions --- server/src/functions/default.js | 43 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/server/src/functions/default.js b/server/src/functions/default.js index 503b21dc..90f01865 100644 --- a/server/src/functions/default.js +++ b/server/src/functions/default.js @@ -34,8 +34,9 @@ exports.fnGetNumberedName=function(names,pattern,value,fillgap=false){ var re=new RegExp("[^\#]*(\#+)[^\#]*") var patternmatch=re.exec(pattern) if(!names || !Array.isArray(names)){ - logger.warning("fnGetNumberedName, No input or no array") - return value + const e = "[fnGetNumberedName] No input or no array" + logger.warning(e) + throw(new Error(e)) } if(patternmatch && patternmatch.length==2){ nrsequence=patternmatch[1] @@ -72,11 +73,13 @@ exports.fnGetNumberedName=function(names,pattern,value,fillgap=false){ var tmp = pattern.replace(nrsequence,nr.toString().padStart(nrsequence.length,"0")) return tmp }else{ - logger.warning("fnGetNumberedName, no pattern matches found in the list") + const e = "fnGetNumberedName, no pattern matches found in the list" + logger.warning(e) return value } }else{ - logger.warning("fnGetNumberedName, no pattern found, use ### for numbers") + const e = "fnGetNumberedName, no pattern found, use ### for numbers" + logger.warning(e) return value } } @@ -101,6 +104,7 @@ exports.fnJq=async function(input,jqe){ result=await jq.run(jqDef+jqe, input, { input:"json",output:"json" }) }catch(e){ logger.error("Error in fnJq : ",e) + throw(e) } return result } @@ -110,12 +114,14 @@ exports.fnCopy=function(input){ exports.fnSort=function(input,sort){ let result=input if(!Array.isArray(input)){ - logger.warning("Warning in fnSort : input is not an array") - return result + const e = "Warning in fnSort : input is not an array" + logger.warning(e) + throw(new Error(e)) } if(!sort){ - logger.warning("Warning in fnSort : sort list not provided") - return result + const e = "Warning in fnSort : sort list not provided" + logger.warning(e) + throw(new Error(e)) } logger.debug("[fnSort] sorting result") // force sort to array @@ -149,7 +155,8 @@ exports.fnSort=function(input,sort){ try{ result.sort(s) }catch(e){ - logger.error("[fnSort] error in fnSort") + logger.error("[fnSort] error in fnSort : ",e) + throw(e) } return result } @@ -164,6 +171,7 @@ exports.fnReadJsonFile = async function(path,jqe=null) { } }catch(e){ logger.error("Error in fnReadJsonFile : ",e) + throw(e) } return result }; @@ -178,6 +186,7 @@ exports.fnReadYamlFile = async function(path,jqe=null) { } }catch(e){ logger.error("Error in fnReadYamlFile : ",e) + throw(e) } return result }; @@ -186,8 +195,10 @@ exports.fnCredentials = async function(name,fallbackname=""){ if(name){ try{ result = await credentialModel.findByName(name,fallbackname) + // console.log(result) }catch(e){ - logger.error(e) + logger.error("Error getting credentials",e) + throw(e) } } return result @@ -195,19 +206,16 @@ exports.fnCredentials = async function(name,fallbackname=""){ exports.fnRestBasic = async function(action,url,body,credential=null,jqe=null,sort=null,hasBigInt=false){ var headers={} if(credential){ - try{ - restCreds = await exports.fnCredentials(credential) - }catch(e){ - logger.error(e) - } + restCreds = await exports.fnCredentials(credential) headers.Authorization="Basic " + Buffer.from(restCreds.user + ':' + restCreds.password).toString('base64') } return await exports.fnRestAdvanced(action,url,body,headers,jqe,sort,hasBigInt) } exports.fnRestAdvanced = async function(action,url,body,headers={},jqe=null,sort=null,hasBigInt=false){ if(!action || !url){ - logger.warning("[fnRest] No action or url defined") - return undefined + const e = "[fnRest] No action or url defined" + logger.warning(e) + throw(new Error(e)) } const httpsAgent = new https.Agent({ rejectUnauthorized: false, @@ -242,6 +250,7 @@ exports.fnRestAdvanced = async function(action,url,body,headers={},jqe=null,sort if(sort)result=exports.fnSort(result,sort) }catch(e){ logger.error("Error in fnRestAdvanced : ",e) + throw(e) } return result From bbe46d1c0b1af53473bab8f68a1eefc63b7d7c1a Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 30 Nov 2023 14:14:16 +0100 Subject: [PATCH 03/50] version bump --- CHANGELOG.md | 11 +++++++++++ client/package.json | 2 +- server/package.json | 2 +- server/src/swagger.json | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dbb7f88..0edc9694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- in remote expression functions, we throw errors so they show up in the form. +- added valueColumn "\*" and placeholderColumn "\*", to return all column, this also means that valueColumn "\*" results in the same as outputObject: true. +- jobid is passed now as extravar and passed to nextform, incase an action exists +- you can now hide a text field + +### Fixed + +- awx workflow template failed with 404 + ## [4.0.19] - 2023-11-22 ### Fixed diff --git a/client/package.json b/client/package.json index fd7e27c5..e16daf60 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "ansible_forms_vue", - "version": "4.0.19", + "version": "4.0.20", "private": true, "scripts": { "serve": "vue-cli-service serve", diff --git a/server/package.json b/server/package.json index e83a22c6..895c224e 100644 --- a/server/package.json +++ b/server/package.json @@ -1,6 +1,6 @@ { "name": "ansible_forms", - "version": "4.0.19", + "version": "4.0.20", "repository": { "type": "git", "url": "git://github.com/ansibleguy76/ansibleforms.git" diff --git a/server/src/swagger.json b/server/src/swagger.json index bdaf19ed..84fd29ed 100644 --- a/server/src/swagger.json +++ b/server/src/swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "info": { "description": "This is the swagger interface for AnsibleForms.\r\nUse the `/auth/login` api with basic authentication to obtain a JWT token.\r\nThen use the access token, prefixed with the word '**Bearer**' to use all other api's.\r\nNote that the access token is limited in time. You can then either login again and get a new set of tokens or use the `/token` api and the refresh token to obtain a new set (preferred).", - "version": "4.0.19", + "version": "4.0.20", "title": "AnsibleForms", "contact": { "email": "info@ansibleforms.com" From 2ab0a25565f8b9f9295b9c3c00915d13155b3a58 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Fri, 1 Dec 2023 09:06:04 +0100 Subject: [PATCH 04/50] outputobject with values --- server/schema/forms_schema.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/schema/forms_schema.json b/server/schema/forms_schema.json index fcdae69e..f89ff738 100644 --- a/server/schema/forms_schema.json +++ b/server/schema/forms_schema.json @@ -747,8 +747,7 @@ "readonlyColumns": {"not":{}}, "isHtml": {"not":{}}, "dbConfig": {"not":{}}, - "hide": {"not":{}}, - "outputObject": {"not":{}} + "hide": {"not":{}} }, "required": ["values"] }, From e73123057420d23914cd3cb8261edf108c95350c Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 12:48:55 +0100 Subject: [PATCH 05/50] remove git type --- client/src/components/Form.vue | 60 +------- docs/_data/help.yaml | 254 ++++++--------------------------- server/src/configure.js | 3 - server/src/models/job.model.js | 53 ------- 4 files changed, 45 insertions(+), 325 deletions(-) diff --git a/client/src/components/Form.vue b/client/src/components/Form.vue index 72028947..c21f703a 100644 --- a/client/src/components/Form.vue +++ b/client/src/components/Form.vue @@ -520,7 +520,7 @@ visibility:{}, // holds which fields are visiable or not canSubmit:false, // flag must be true to enable submit - allows to finish background queries - has a watchdog in case not possible validationsLoaded:false, // ready flag if field validations are ready, we don't show the form otherwise - pretasksFinished:false, // ready flag for form pre tasks (git => git pull) + pretasksFinished:false, // ready flag for form pre tasks (future use - use to be gitpulls) loadMessage:"Loading form...", timeout:undefined, // determines how long we should show the result of run showHelp:false, // flag to show/hide form help @@ -2032,11 +2032,6 @@ this.pauseJsonOutput=false; } }, - pullGit(repo){ - var postdata={} - postdata.gitRepo = repo; - return axios.post("/api/v1/git/pull",postdata,TokenStorage.getAuthentication()) - }, initForm(){ var ref=this this.pretasksFinished=false @@ -2167,56 +2162,11 @@ this.findVariableDependencies() this.findVariableDependentOf() - // set as ready - // if our type is git, we need to first pull git - if(!["git","multistep"].includes(this.currentForm.type)){ - this.pretasksFinished=true - // start dynamic field loop (= infinite) - this.startDynamicFieldsLoop() - }else{ - var gitpulls=[] + // for future use, run something before the form starts + this.pretasksFinished=true + // start dynamic field loop (= infinite) + this.startDynamicFieldsLoop() - if(this.currentForm.type=="git"){ - gitpulls.push(this.pullGit(this.currentForm.repo)) - }else if(this.currentForm.type=="multistep"){ - this.currentForm.steps.forEach((step, i) => { - if(step.type=="git") - gitpulls.push(this.pullGit(step.repo)) - }); - } - // wait for all git pulls - Promise.all(gitpulls) - .then(results=>{ - var status=true - results.forEach((result, i) => { - if(result){ - - if(result.data.status=="error"){ - ref.$toast.error(result.data.message) - status=false - } - - }else{ - ref.$toast.error("Failed to pull from git") - status=false - } - - }); - if(status){ - if(gitpulls.length>0) - ref.$toast.success("Git was pulled") - } - ref.pretasksFinished=true - // start dynamic field loop (= infinite) - ref.startDynamicFieldsLoop() - }) - .catch(function(err){ - ref.$toast.error("Failed to pull from git " + err.toString()) - ref.pretasksFinished=true - // start dynamic field loop (= infinite) - ref.startDynamicFieldsLoop() - }) - } }, }, diff --git a/docs/_data/help.yaml b/docs/_data/help.yaml index 6ed0bf0d..a6c69652 100644 --- a/docs/_data/help.yaml +++ b/docs/_data/help.yaml @@ -328,8 +328,7 @@ default: "%PERSISTENT_FOLDER%/repositories" version: 3.0.0 description: | - Ansible Forms target a git repository. The extravars will be save to a local repository and a git push is triggered. - This path is the root path for your local repositories. + This path is the root path for your local repositories, allowing you to integrate data with git repositories. - name: ENCRYPTION_SECRET type: string allowed: a strong encryption string @@ -815,59 +814,24 @@ groupname1: has-background-info-light groupname2: has-background-success-light fields: # list of field objects - - name: git - description: | - Updates a file in a repository. - The repository must be added in the settings. - examples: - - name: Git Form - code: | - - name: Update repo - roles: # reference to roles defined earlier - - vmware - description: Updates a repo in git - image: https://picsum.photos/64 # an image url (can be base64) - categories: # reference to categories defined earlier - - Demo - - Vmware/Linux # Add to a subcategory using the `/` separator - tileClass: has-background-danger # a bulma background color style (https://bulma.io) - type: git # type is awx, ansible, git or multistep - repo: - dir: myrepodir - file: myrepofile.yaml - push: git push origin main - pull: git pull origin main - fieldGroupClasses: # give fieldgroups a custom background - groupname1: has-background-info-light - groupname2: has-background-success-light - fields: # list of field objects - name: multistep description: | Executes multiple jobs sequentially. - Each step can be a different type (ansible, awx, git). + Each step can be a different type (ansible, awx). examples: - name: Multistep form code: | - name: Create VM roles: # reference to roles defined earlier - vmware - description: Updates a repo in git + description: Create and start a vm image: https://picsum.photos/64 # an image url (can be base64) categories: # reference to categories defined earlier - Demo - Vmware tileClass: has-background-danger # a bulma background color style (https://bulma.io) - type: multistep # type is awx, ansible, git or multistep + type: multistep # type is awx, ansible or multistep steps: - - type: git - name: Sync repository - repo: - dir: myrepodir - file: myrepofile.yaml - push: git push origin main - pull: git pull origin main - key: repoInfo # a specific key in the extravars - continue: true # continue even this step fails - type: awx name: Create vm template: Create VM @@ -886,9 +850,7 @@ fields: # list of field objects short: Form type description: | - Our form eventually sends extravars to a target. That target can be an ansible-playbook, an awx-template a git repository or a multistep (which is a list or combination of multiple playbooks/templates/repositories, executed sequentially). - - *Note* : In case of git, the repository is pulled before the form is loaded. + Our form eventually sends extravars to a target. That target can be an ansible-playbook, an awx-template or a multistep (which is a list or combination of multiple playbooks/templates, executed sequentially). examples: - name: Run AWX Template Create VM code: | @@ -900,30 +862,17 @@ name: Delete vm type: ansible template: delete_vm.yaml - - name: Update git repo - code: | - name: Update sourcecontrol - type: git - repo: - dir: my_project - file: test.yaml - push: git push origin main - pull: git pull origin main - name: Run multistep job code: | name: Trigger awx job type: multistep steps: - - name: Update git - type: git - repo: - dir: vmware_inventory - file: virtual_machines.yaml - push: git push origin main - pull: git pull origin main - - name: Update vms + - name: Create vm type: awx - template: Create VM + template: create_vm + - name: Start vm + type: awx + template: start_vm - name: template type: string with_types: awx @@ -1030,24 +979,6 @@ short: Pass ansible_user and ansible_password to ansible playbook using stored credentials description: | Ansible allows to pass default ansible credentials in the form of 2 extravars ansible_user and ansible_password - - name: repo - type: object - with_types: git - group: basic - allowed: a repo object - objectLink: forms/form/repo - docsObjectLink: "#repository-object" - short: Repository - description: | - This attribute is only used when the form type is `git`. - examples: - - name: Push to git - code: | - repo: - dir: vmware_inventory - file: virtual_machines.yaml - push: git push origin main - pull: git pull origin main - name: steps type: array with_types: multistep @@ -1368,14 +1299,9 @@ description: | Targets an Awx Template. The AWX Connection token must be set in the settings. - - name: git - description: | - Updates a file in a repository. - The repository must be added in the settings. short: Step type description: | - Every step can be an ansible-playbook, an awx-template or a git repository. - *Note* : In case of git, the repository is pulled before the form is loaded. + Every step can be an ansible-playbook or an awx-template examples: - name: Run AWX Template Helloworld code: | @@ -1387,15 +1313,6 @@ name: Delete vm type: ansible template: delete_vm.yaml - - name: Update git repo - code: | - name: Update sourcecontrol - type: git - repo: - dir: my_project - file: test.yaml - push: git push origin main - pull: git pull origin main - name: template type: string group: basic @@ -1448,23 +1365,6 @@ - name: __playbook__ type: text default: helloworld.yaml # this will overwrite the playbook name - - name: repo - type: object - allowed: a repo object - group: basic - with_types: git - objectLink: forms/form/repo - short: Repository - description: | - This attribute is only used when the form type is `git`. - examples: - - name: Push to git - code: | - repo: - dir: vmware_inventory - file: virtual_machines.yaml - push: git push origin main - pull: git pull origin main - name: inventory type: string group: workflow @@ -1553,71 +1453,41 @@ examples: - name: Resize volume code: | - # This example might need a little explanation : - # - # When the operator opens the form : - # - Phase 1 : the git repo (from step 1) is pulled and the local copy is now up-to-date - # - Phase 2 : the form loads the local file volume.yaml - # - Phase 3 : the operator sets a new size - # - Phase 4 : the expression 'new_data' merges the new size in the original data - # at this moment the extravars contains 2 parts: - # original : the contents of volume.yaml - # new_data : the contents of volume.yaml with updated size info - # - Phase 5 : the operator submits the form - # - Phase 6 : step 1 is executed - # the extravar 'new_data' is send as step 1 (using the 'key' attribute !) - # the 'new_data' is saved as volume.yaml (overwrite repo data) - # the git repo is pushed and updated - # - Phase 7 : step 2 is executed - # the awx template 'volume resize' is triggered - # the awx templates project is set to reload from git on run - # awx pulls from git and is now aware of the new inventory data (new volume size) - # the tempate is executed - # ansibles idempotency sees the size change and resizes the volume name: Resize volume type: multistep steps: - - name: Push to git - type: git - repo: - dir: storage - file: volume.yaml - push: git push origin main - pull: git pull origin main - key: new_data - - name: Resize volume + - name: Create datastore type: awx - template: volume resize - key: new_data + template: create_datastore + key: datastore # to create the datastore, we only send the datastore info + - name: Create vm + type: awx + template: create_vm fields: - - name: original - type: expression - expression: fn.fnReadYamlFile('/root/ansible_forms/server/persistent/repositories/storage/volume.yaml') - label: original yaml from git (pull) - - name: size + - name: datastore + type: text + model: datastore.name + - name: datastore_size type: number - label: new size - required: true - - name: new_data - label: new yaml for git (push) - type: expression - expression: (()=>{ return {...$(original),...{'size':$(size)}} })() - runLocal: true - - # the expression above is vanilla ES6 javascript. using the 'spread' operator (... the 3 dots), - # the new size is merged into the original data - # https://atomizedobjects.com/blog/javascript/how-to-merge-two-objects-in-javascript/ + model: datastore.size + - name: vm + type: text + model: vm.name + - name: vm_size + type: number + model: vm.size # the extravars of the above example will be somewhat like this : - original: - name: volumex + datastore: + name: ds1 size: 123 - server_id: 5 - new_data: # the step is referencing this key, only this is send as extravars to the step. - name: volumex - size: 500 # the new size is merged - server_id: 5 + vm: + name: vm1 + size: 100 + # Step 1 will not receive the full set of extravars, due to the 'key: datastore' + # it will receive only 'name' (ds1) and 'size' (123) (the contents of key 'datastore') + # Step 2 has no key set, and will receive the full extravars # Extra : Every step can reference its own key and thus send a piece of the extravars - name: ifExtraVar type: extravar reference @@ -3354,52 +3224,6 @@ type: mysql query: SELECT id,name FROM users previewColumn: name - - name: Repository - itemName: attribute - link: forms/form/repo - icon: fab,git - description: | - One of the form destination types is `git`. This allows the form to update a file in a git repository. - To define the git connection, a repo-object attribute is required. - items: - - name: dir - type: string - required: true - allowed: a subfolder in the repositories folder - short: Directory - description: | - Ansible Forms can hold multiple local copies of repositories that can be pulled and push using forms. - Each copy goes in a subfolder of the repositories folder. Use this attribute to define the subfolder name. - - name: file - type: string - required: true - allowed: a repository filename - short: File - description: | - This property hold the filename of a yaml file that can be updated with the forms extravars. - - name: push - type: string - required: true - allowed: a git push command - short: Push command - description: | - When the form is submitted the git-step will push the extravars (or subset) using this git command. - - name: pull - type: string - required: true - allowed: a git pull command - short: Pull command - description: | - Before the form is loaded the repository is pulled using this command. - examples: - - name: Pull from main branch - language: YAML - code: | - repo: - dir: vmware_inventory - file: virtual_machines.yaml - push: git push origin main - pull: git pull origin main - name: Approval point itemName: attribute link: forms/form/approval @@ -3637,7 +3461,7 @@ .filterBy({property1:'value1'},{property2:'value2'}, ...}} // filters the array by property value (* wildcards allowed) .regexBy({property1:'regex1'},{property2:'regex2'}, ...}} // filters the array by property matched against regex .distinctBy('property1','property2', ...) // will make the array entries unique by property - .selectAttr('property1','property2') // only selects a certain property + .selectAttr({prop1:'property1',prop2:'property2'}) // only selects a certain property, and you can relabel them .sortBy('property1','-property2', ...) // will order the array. To have descending add a "-" (minus) before the property mylist: @@ -3667,7 +3491,7 @@ fnArray.from($(mylist)) // take data from another field 'mylist' .regexBy({name:'.*man$'}) // name must end with 'man' .filterBy({has_ability:true}) // must have abilities - .selectAttr('name','ability') // only take properties name and ability + .selectAttr({name:'name',ability:'ability'}) // only take properties name and ability .sortBy('-name') // sort by name descending /* @@ -3850,6 +3674,7 @@ // - jq-expression : an optional jq-expression (https://jsplay.org) // - sort-object : a sort object to sort the result // - hasBigInt : a boolean indicating if it should convert Int64 to string + // - tokenPrefix : a prefix, defaults to 'Bearer' (v5.0.0) - name: rest api with secured token authentication code: | fn.fnRestJwtSecure( @@ -3872,6 +3697,7 @@ // - jq-expression : an optional jq-expression (https://jsplay.org) // - sort-object : a sort object to sort the result // - hasBigInt : a boolean indicating if it should convert Int64 to string + // - tokenPrefix : a prefix, defaults to 'Bearer' (v5.0.0) - name: rest api with custom headers code: | fn.fnRestAdvanced( diff --git a/server/src/configure.js b/server/src/configure.js index 7fe44a05..b719c7c9 100644 --- a/server/src/configure.js +++ b/server/src/configure.js @@ -59,7 +59,6 @@ module.exports = app => { // import api routes const awxRoutes = require('./routes/awx.routes') const jobRoutes = require('./routes/job.routes') - const gitRoutes = require('./routes/git.routes') const queryRoutes = require('./routes/query.routes') const expressionRoutes = require('./routes/expression.routes') const userRoutes = require('./routes/user.routes') @@ -117,8 +116,6 @@ module.exports = app => { // api routes for automation actions - // app.use('/api/v1/ansible',cors(), authobj, ansibleRoutes) - app.use('/api/v1/git',cors(), authobj, gitRoutes) // app.use('/api/v1/multistep',cors(), authobj, multistepRoutes) // api routes for admin management diff --git a/server/src/models/job.model.js b/server/src/models/job.model.js index 05afea2c..85fdea87 100644 --- a/server/src/models/job.model.js +++ b/server/src/models/job.model.js @@ -394,15 +394,6 @@ Job.launch = async function(form,formObj,user,creds,extravars,parentId=null,next notifications ) } - if(jobtype=="git"){ - return await Git.push( - formObj.repo, - extravars, - jobid, - null, - (parentId)?null:formObj.approval // if multistep: no individual approvals checks - ) - } if(jobtype=="multistep"){ return await Multistep.launch( form, @@ -438,9 +429,6 @@ Job.continue = async function(form,user,creds,extravars,jobid,next) { if(jobtype=="awx"){ target=formObj.template } - if(jobtype=="git"){ - target=formObj?.repo?.file - } if(jobtype=="multistep"){ target=formObj.name } @@ -508,9 +496,6 @@ Job.continue = async function(form,user,creds,extravars,jobid,next) { true ) } - if(jobtype=="git"){ - return await Git.push(formObj.repo,extravars,jobid,++counter,formObj.approval,true) - } if(jobtype=="multistep"){ return await Multistep.launch(form,formObj.steps,user,extravars,creds,jobid,++counter,formObj.approval,step,true) } @@ -1367,42 +1352,4 @@ Awx.findInventoryByName = async function (name) { } }; -// git stuff -var Git=function(){}; -Git.push = async function (repo,ev,jobid,counter,approval,approved=false) { - if(!counter){ - counter=0 - }else{ - counter++ - } - if(approval){ - if(!approved){ - await Job.sendApprovalNotification(approval,ev,jobid) - await Job.printJobOutput(`APPROVE [${repo.file}] ${'*'.repeat(69-repo.file.length)}`,"stdout",jobid,++counter) - await Job.update({status:"approve",approval:JSON.stringify(approval),end:moment(Date.now()).format('YYYY-MM-DD HH:mm:ss')},jobid) - return true - }else{ - logger.notice("Continuing awx " + repo.file + " it has been approved") - } - } - // we make a copy to not mutate - var extravars={...ev} - try{ - // save the extravars as file - we do this in sync, should be fast - await Job.printJobOutput(`TASK [Writing YAML to local repo] ${'*'.repeat(72-26)}`,"stdout",jobid,++counter) - var yaml = YAML.stringify(extravars) - fs.writeFileSync(path.join(appConfig.repoPath,repo.dir,repo.file),yaml) - // log the save - await Job.printJobOutput(`ok: [Extravars Yaml file created : ${repo.file}]`,"stdout",jobid,++counter) - await Job.printJobOutput(`TASK [Committing changes] ${'*'.repeat(72-18)}`,"stdout",jobid,++counter) - // start commit - var command = `git commit --allow-empty -am "update from ansibleforms" && ${repo.push}` - var directory = path.join(appConfig.repoPath,repo.dir) - return await Exec.executeCommand({directory:directory,command:command,description:"Pushing to git",task:"Git push"},jobid,counter) - - }catch(e){ - logger.error("error : ",e) - await Job.endJobStatus(jobid,++counter,"stderr","failed",e) - } -}; module.exports= Job; From cf1840e9779fcf0d8ac8d0895ae0c47bb746e161 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:31:21 +0100 Subject: [PATCH 06/50] removed git, better repo control --- server/src/swagger.json | 67 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/server/src/swagger.json b/server/src/swagger.json index 84fd29ed..e27d89f7 100644 --- a/server/src/swagger.json +++ b/server/src/swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "info": { "description": "This is the swagger interface for AnsibleForms.\r\nUse the `/auth/login` api with basic authentication to obtain a JWT token.\r\nThen use the access token, prefixed with the word '**Bearer**' to use all other api's.\r\nNote that the access token is limited in time. You can then either login again and get a new set of tokens or use the `/token` api and the refresh token to obtain a new set (preferred).", - "version": "4.0.20", + "version": "5.0.0", "title": "AnsibleForms", "contact": { "email": "info@ansibleforms.com" @@ -86,6 +86,15 @@ "basicAuth": [] } ], + "parameters": [ + { + "in": "query", + "name": "expiryDays", + "type": "integer", + "required": false, + "description": "Expiry days, only for admins" + } + ], "summary": "Get authorization bearer token", "produces": [ "application/json" @@ -2796,13 +2805,9 @@ "type": "string", "example": "git@github.com:myrepo" }, - "username": { + "command": { "type": "string", - "example": "AnsibleForms" - }, - "email": { - "type": "string", - "example": "info@ansibleforms.com" + "example": "git clone --verbose ..." } } } @@ -2837,6 +2842,54 @@ } } }, + "/repo/pull/{repositoryName}": { + "post": { + "tags": [ + "repositories" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "repositoryName", + "type": "string", + "required": true, + "description": "The repository name" + } + ], + "summary": "Pulls a git repistory", + "produces": [ + "application/json" + ], + "responses": { + "200": { + "description": "successful operation", + "schema": { + "type": "object", + "example": { + "status": "success", + "message": "Repository is pulled", + "data": { + "output": "", + "error": "" + } + } + } + }, + "401": { + "description": "unauthorized", + "schema": { + "type": "string", + "example": "Authorize with a valid Bearer access token" + } + } + } + } + }, "/version": { "get": { "tags": [ From cd0cd0ebff892c39de06315ed89b1cf5b618b3af Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:31:54 +0100 Subject: [PATCH 07/50] improved repo --- server/src/controllers/repo.controller.js | 27 +++- server/src/models/repo.model.js | 176 ++++++++++++++++------ server/src/routes/repo.routes.js | 2 + 3 files changed, 159 insertions(+), 46 deletions(-) diff --git a/server/src/controllers/repo.controller.js b/server/src/controllers/repo.controller.js index 15b3edbb..7c7fdc7a 100644 --- a/server/src/controllers/repo.controller.js +++ b/server/src/controllers/repo.controller.js @@ -20,7 +20,7 @@ exports.create = function(req, res) { if(req.body.constructor === Object && Object.keys(req.body).length === 0){ res.status(400).send({ error:true, message: 'Please provide all required fields' }); }else{ - Repo.create(req.body.uri,req.body.command,req.body.username,req.body.email) + Repo.create(req.body.uri,req.body.command) .then((output)=>{res.json(new RestResult("success","repository created",output,""))}) .catch((err)=>{res.json(new RestResult("error","failed to create repository",null,err.toString()))}) } @@ -44,3 +44,28 @@ exports.delete = function(req, res) { res.json(new RestResult("error","no repository name specified",null,"")); } }; + +exports.pull = async function(req, res) { + + // get the form data + var restResult = new RestResult("success","","","") + var repositoryName = req.params.repositoryName + if(!repositoryName){ + // wrong implementation -> send 400 error + res.json(new RestResult("error","no repository","","name is a required field")); + }else{ + Repo.pull(repositoryName) + .then((out)=>{ + restResult.message = "succesfully pulled repository" + restResult.data.output = out + res.json(restResult); + }) + .catch((err)=>{ + restResult.status = "error" + restResult.message = `error occured pulling repository ${repositoryName}` + restResult.data.error = err.toString() + res.json(restResult); + }) + } + +}; \ No newline at end of file diff --git a/server/src/models/repo.model.js b/server/src/models/repo.model.js index 6aa739ae..e1f47101 100644 --- a/server/src/models/repo.model.js +++ b/server/src/models/repo.model.js @@ -1,10 +1,113 @@ 'use strict'; const exec = require('child_process').exec; +const spawn = require('child_process').spawn; +const execSync = require('child_process').execSync; const logger=require("../lib/logger") const path=require("path") const fs=require("fs") var config=require('../../config/app.config') + +var Exec=function(){} +// tree kill should do the trick, but fails in alpine +// so this piece of code does it +Exec.killChildren = (pid) => { + const children = []; + + try { + const psRes = execSync(`ps -opid="" -oppid="" |grep ${pid}`).toString().trim().split(/\n/); + + (psRes || []).forEach(pidGroup => { + const [actual, parent] = pidGroup.trim().split(/ +/); + + if (parent.toString() === pid.toString()) { + children.push(parseInt(actual, 10)); + } + }); + } catch (e) {} + try { + logger.debug(`Killing process ${pid}`) + process.kill(pid); + children.forEach(childPid => Exec.killChildren(childPid)); + } catch (e) {} +}; + +Exec.maskGitToken =(data)=>{ + if(data){ + try{ + // console.log(data.toString()) + var masked = data.toString().replace(/(http[s]{0,1}):\/\/([^:]+):([^@]+)@(.*)/gm,"$1://$2:*******@$4") + // console.log("==> " + rep) + return masked + }catch(e){ + logger.error("Failed to maskGitToken : ",e) + return data + } + }else{ + return data + } +} + +Exec.executeSilentCommand = (cmd) => { + + return new Promise((resolve,reject)=>{ + var command = cmd.command + var directory = cmd.directory + var description = cmd.description + // execute the procces + logger.debug(`${description}, ${directory} > ${Exec.maskGitToken(command)}`) + try{ + var cmdlist = command.split(' ') + var basecmd = cmdlist[0] + var parameters = cmdlist.slice(1) + var child = spawn(basecmd,parameters,{shell:true,stdio:["ignore","pipe","pipe"],cwd:directory,detached:true}); + var timeout = setTimeout(()=>{ + Exec.killChildren(child.pid) + },60000) + var out=[] + out.push(`Running command : ${Exec.maskGitToken(command)}`) + // add output eventlistener to the process to save output + child.stdout.on('data',function(data){ + // logger.debug(data) + data = Exec.maskGitToken(data) + out.push(data) + }) + // add error eventlistener to the process to save output + child.stderr.on('data',function(data){ + // save the output + // logger.debug(data) + data = Exec.maskGitToken(data) + out.push(data) + }) + // add exit eventlistener to the process to handle status update + child.on('exit',function(data){ + clearTimeout(timeout) + logger.info(description + " finished : " + data) + if(data!=0){ + if(child.signalCode=='SIGTERM'){ + out.push("The command timed out") + } + reject(out.join('\n')) + }else{ + resolve(out.join('\n')) + } + }) + // add error eventlistener to the process; set failed + child.on('error',function(data){ + data = Exec.maskGitToken(data) + logger.error(data) + out.push(data) + reject(out.join('\n')) + }) + + }catch(e){ + reject(e) + } + }) + +} + + const Repo={ } @@ -77,9 +180,9 @@ Repo.findByName = function (name,text){ output.push(directory) } - info.stdout.on('data', function(a){ - // logger.sill(a) - output.push(a) + info.stdout.on('data', function(data){ + data = Exec.maskGitToken(data) + output.push(data) }); info.on('exit',function(code){ @@ -107,10 +210,10 @@ Repo.findByName = function (name,text){ } // run git clone -Repo.create = function (uri,command,username,email) { +Repo.create = function (uri,command) { return new Promise((resolve,reject)=>{ try{ - logger.notice("Creating repository " + (uri)?uri:command) + logger.notice(`Creating repository : ${Exec.maskGitToken((uri)?uri:command)}`) var directory = config.repoPath try{ fs.accessSync(directory) @@ -124,8 +227,8 @@ Repo.create = function (uri,command,username,email) { return; } } - var clone - var cmd + + var cmd,parameters if(uri){ cmd = `git clone --verbose ${uri}` }else if(command){ @@ -134,52 +237,27 @@ Repo.create = function (uri,command,username,email) { reject("No uri or command given") return; } - var repoNameRegex = new RegExp(".*/([^\.]+)\.git.*", "g"); - var match = repoNameRegex.exec(uri); - if(match){ - var repoName = match[1] - logger.notice(`Found repoName in command : ${repoName}`) - cmd = `${cmd} ; cd ${repoName}` - }else{ - logger.error(`No repo name found in uri`) - } - if(username){ - cmd = `${cmd} ; git config user.name "${username}"` - } - if(email){ - cmd = `${cmd}; git config user.email ${email}` + if(!cmd.startsWith('git clone')){ + reject("Not a git clone command") + return; } var hostRegex = new RegExp(".*@([^:]+):.*", "g"); + var match = hostRegex.exec(cmd); - if(match){ + if(match && uri){ var host = match[1] logger.notice(`Found host in command : ${host}; adding it to known_hosts`) cmd = `ssh-keyscan ${host} >> ~/.ssh/known_hosts ; ${cmd}` }else{ - logger.error(`No host found in command`) + logger.warning(`No host found in command`) } - - logger.notice(`Running cmd : ${cmd}`) - var clone = exec(cmd,{cwd:directory}) - var output = [] - clone.stdout.on('data', function(a){ - logger.info(a) - output.push(a) - }); - - clone.on('exit',function(code){ - logger.debug('exit') - if(code==0){ - resolve(`repository created succesfully`) - }else{ - reject(`\ncreating repository failed with code ${code}\n${output.join("\n")}`) - } - }); - - clone.stderr.on('data',function(a){ - output.push(a) - logger.error('stderr:'+a); - }); + Exec.executeSilentCommand({command:cmd,directory:config.repoPath,description:"Cloning repository"}) + .then((out)=>{ + resolve(out) + }).catch((e)=>{ + reject(e) + }) + }catch(e){ logger.error(e) reject(e) @@ -228,4 +306,12 @@ Repo.addKnownHosts = function (hosts) { }) }; +// run a playbook +Repo.pull = function (repo) { + + var command = "git pull --verbose" + var directory = path.join(config.repoPath,repo) + return Exec.executeSilentCommand({directory:directory,command:command,description:"Pulling from git"}) + +}; module.exports= Repo; diff --git a/server/src/routes/repo.routes.js b/server/src/routes/repo.routes.js index ee84a9ab..92410e42 100644 --- a/server/src/routes/repo.routes.js +++ b/server/src/routes/repo.routes.js @@ -10,5 +10,7 @@ router.post('/', repoController.create); router.post('/known_hosts/', repoController.addKnownHosts); // create repo router.delete('/', repoController.delete); +// pull repo +router.post('/pull/:repositoryName', repoController.pull) module.exports = router From 9fcd1571160278aad166083e72b5a9f9de8303b7 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:33:04 +0100 Subject: [PATCH 08/50] add tokenprefix --- server/src/functions/default.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/functions/default.js b/server/src/functions/default.js index 90f01865..33329005 100644 --- a/server/src/functions/default.js +++ b/server/src/functions/default.js @@ -255,18 +255,18 @@ exports.fnRestAdvanced = async function(action,url,body,headers={},jqe=null,sort return result } -exports.fnRestJwt = async function(action,url,body,token,jqe=null,sort=null,hasBigInt=false){ +exports.fnRestJwt = async function(action,url,body,token,jqe=null,sort=null,hasBigInt=false,tokenPrefix="Bearer"){ var headers={} if(token){ - headers.Authorization="Bearer " + token + headers.Authorization=tokenPrefix + " " + token } return await exports.fnRestAdvanced(action,url,body,headers,jqe,sort,hasBigInt) } -exports.fnRestJwtSecure = async function(action,url,body,tokenname,jqe=null,sort=null,hasBigInt=false){ +exports.fnRestJwtSecure = async function(action,url,body,tokenname,jqe=null,sort=null,hasBigInt=false,tokenPrefix="Bearer"){ var headers={} if(tokenname){ var token = await exports.fnCredentials(tokenname) - headers.Authorization="Bearer " + token.password + headers.Authorization=tokenPrefix + " " + token.password } return await exports.fnRestAdvanced(action,url,body,headers,jqe,sort,hasBigInt) } From 7443f2134ebabfb0678e0cfe7c1b2b30ed1302e0 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:34:01 +0100 Subject: [PATCH 09/50] small bugfix --- client/src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main.js b/client/src/main.js index 7018248a..e1429b89 100644 --- a/client/src/main.js +++ b/client/src/main.js @@ -30,7 +30,7 @@ axios.interceptors.response.use( (response) => { return response; }, (error) => { // Return any error which is not due to authentication back to the calling service - if (error.response.status !== 401) { + if (error.response?.status !== 401) { return new Promise((resolve, reject) => { reject(error); }); From cda9d80a8fb2421ae5b73592a95c8985b7b0ea23 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:37:20 +0100 Subject: [PATCH 10/50] ldap improvement --- client/src/views/Ldap.vue | 18 ++++++++++++-- server/src/auth/auth_basic.js | 8 +++++-- server/src/controllers/login.controller.js | 16 +++++++++++-- server/src/models/ldap.model.js | 14 +++++++++++ server/src/models/user.model.js | 28 ++++++++++++---------- 5 files changed, 66 insertions(+), 18 deletions(-) diff --git a/client/src/views/Ldap.vue b/client/src/views/Ldap.vue index b5bb8746..46432c99 100644 --- a/client/src/views/Ldap.vue +++ b/client/src/views/Ldap.vue @@ -16,7 +16,7 @@
- +
@@ -24,6 +24,11 @@ + + + + +
@@ -67,11 +72,17 @@ bind_user_dn:"", search_base:"", username_attribute:"", + groups_attribute:"", enable:true, enable_tls:true, ignore_certs:true, + groups_search_base: "", + group_class: "", + group_member_attribute: "", + group_member_user_attribute: "", cert:"", - ca_bundle:"" + ca_bundle:"", + is_advanced:false } } }, @@ -136,6 +147,9 @@ username_attribute:{ required }, + groups_attribute:{ + required + }, cert:{ requiredIf:requiredIf(function(ldap){ return ldap.enable_tls && !ldap.ignore_certs diff --git a/server/src/auth/auth_basic.js b/server/src/auth/auth_basic.js index 9281c426..7d3c8734 100644 --- a/server/src/auth/auth_basic.js +++ b/server/src/auth/auth_basic.js @@ -4,6 +4,7 @@ const User = require('./../models/user.model'); const authConfig = require('../../config/auth.config.js') const logger=require("../lib/logger"); const Helpers = require('../lib/common'); +const Ldap = require('../models/ldap.model') // create username / password login strategy @@ -41,12 +42,15 @@ passport.use( async (username, password, done) => { // authentication against database first try{ + var ldapConfig = await Ldap.find() var user = await User.checkLdap(username,password) .then((result)=>{ + // logger.debug("Ldap object :") + // logger.debug(JSON.stringify(result)) var user = {} - user.username = result.sAMAccountName + user.username = result[ldapConfig.username_attribute] user.type = 'ldap' - user.groups = User.getGroups(user,result) + user.groups = User.getGroups(user,result,ldapConfig) user.roles = User.getRoles(user.groups,user) logger.info("ldap login for " + user.username) return user diff --git a/server/src/controllers/login.controller.js b/server/src/controllers/login.controller.js index 8db02247..5f666bbe 100644 --- a/server/src/controllers/login.controller.js +++ b/server/src/controllers/login.controller.js @@ -9,13 +9,13 @@ const {inspect}=require("node:util") const Helpers=require('../lib/common') const RestResult = require("../models/restResult.model") -function userToJwt(user){ +function userToJwt(user,expiryDays){ // is something like // {"username":"administrator","type":"local","roles":["public","admin"]} // we create 2 jwt tokens (accesstoken and refresh token) - const token = jwt.sign({user,access:true}, authConfig.secret,{ expiresIn: authConfig.jwtExpiration}); + const token = jwt.sign({user,access:true}, authConfig.secret,{ expiresIn: expiryDays || authConfig.jwtExpiration}); const refreshtoken = jwt.sign({user,refresh:true}, authConfig.secret,{ expiresIn: authConfig.jwtRefreshExpiration}); logger.debug(JSON.stringify(user)) // we store the tokens in the database, to later verify a refresh token action @@ -82,6 +82,10 @@ exports.basic = async function(req, res,next) { //return next(error); } // send the tokens to the requester + // if admin role, you can override the expirydays (for accesstoken only) + if(req.query.expiryDays && user?.roles?.includes("admin") && !isNan(req.query.expiryDays)){ + return res.json(userToJwt(user,`${req.query.expiryDays}D`)) + } return res.json(userToJwt(user)); } ); @@ -124,6 +128,10 @@ exports.basic_ldap = async function(req, res,next) { return next(error); } // send the tokens to the requester + // if admin role, you can override the expirydays (for accesstoken only) + if(req.query.expiryDays && user?.roles?.includes("admin") && !isNan(req.query.expiryDays)){ + return res.json(userToJwt(user,`${req.query.expiryDays}D`)) + } return res.json(userToJwt(user)); } ); @@ -183,6 +191,10 @@ exports.azureadoauth2login = async function(req, res,next) { user.type = 'azuread' user.groups = groups user.roles = User.getRoles(user.groups,user) + // if admin role, you can override the expirydays (for accesstoken only) + if(req.query.expiryDays && user?.roles?.includes("admin") && !isNan(req.query.expiryDays)){ + return res.json(userToJwt(user,`${req.query.expiryDays}D`)) + } res.json(userToJwt(user)) }catch(err){ logger.error(Helpers.getError(err)) diff --git a/server/src/models/ldap.model.js b/server/src/models/ldap.model.js index 26f2b624..4bc7bf65 100644 --- a/server/src/models/ldap.model.js +++ b/server/src/models/ldap.model.js @@ -17,7 +17,13 @@ var Ldap=function(ldap){ this.bind_user_pw = encrypt(ldap.bind_user_pw); this.search_base = ldap.search_base; this.username_attribute = ldap.username_attribute; + this.groups_attribute = ldap.groups_attribute; this.enable = (ldap.enable)?1:0; + this.is_advanced = (ldap.is_advanced)?1:0; + this.groups_search_base = (ldap.is_advanced)?ldap.groups_search_base:"" + this.group_class = (ldap.is_advanced)?ldap.group_class:"" + this.group_member_attribute = (ldap.is_advanced)?ldap.group_member_attribute:"" + this.group_member_user_attribute = (ldap.is_advanced)?ldap.group_member_user_attribute:"" }; Ldap.update = function (record) { logger.info(`Updating ldap ${record.server}`) @@ -64,6 +70,14 @@ Ldap.check = function(ldapConfig){ username: "dummyuser_for_check", // starttls: false } + // new in v4.0.20, add advanced ldap properties + if(ldapConfig.is_advanced){ + if(ldapConfig.groups_search_base){ options.groupsSearchBase = ldapConfig.groups_search_base } + if(ldapConfig.group_class){ options.groupClass = ldapConfig.group_class } + if(ldapConfig.group_member_attribute){ options.groupMemberAttribute = ldapConfig.group_member_attribute } + if(ldapConfig.group_member_user_attribute){ options.groupMemberUserAttribute = ldapConfig.group_member_user_attribute } + } + // console.log(options) // ldap-authentication has bad cert check, so we check first !! if(ldapConfig.enable_tls && !(ldapConfig.ignore_certs==1)){ if(!Helpers.checkCertificate(ldapConfig.cert)){ diff --git a/server/src/models/user.model.js b/server/src/models/user.model.js index 9c8c76a6..861a5f0a 100644 --- a/server/src/models/user.model.js +++ b/server/src/models/user.model.js @@ -120,6 +120,9 @@ User.getRoles = function(groups,user){ forms = Form.load() }catch(e){ logger.error(e) + if(groups.includes('local/admins')){ + roles.push("admin") + } return roles } groups.forEach(function(group){ @@ -140,22 +143,15 @@ User.getRoles = function(groups,user){ return roles } -User.getGroups = function(user,groupObj){ - var groupMatch="" +User.getGroups = function(user,groupObj,ldapConfig={}){ var group="" var groups = [] - var forms=undefined - try{ - forms = Form.load() - }catch(e){ - logger.error(e) - return groups - } + // ldap type - if(user.type=="ldap"){ - if(groupObj.memberOf){ + if(user.type=="ldap" && ldapConfig.groups_attribute){ + if(groupObj[ldapConfig.groups_attribute]){ // get the memberOf field, force to array - var ldapgroups = [].concat(groupObj.memberOf) + var ldapgroups = [].concat(groupObj[ldapConfig.groups_attribute]) //logger.debug(`LDAP Groups = ${ldapgroups}`) // loop ldap groups ldapgroups.forEach(function(v,i,a){ @@ -208,6 +204,14 @@ User.checkLdap = function(username,password){ username: username, // starttls: false } + // new in v4.0.20, add advanced ldap properties + if(ldapConfig.is_advanced){ + if(ldapConfig.groups_search_base){ options.groupsSearchBase = ldapConfig.groups_search_base } + if(ldapConfig.group_class){ options.groupClass = ldapConfig.group_class } + if(ldapConfig.group_member_attribute){ options.groupMemberAttribute = ldapConfig.group_member_attribute } + if(ldapConfig.group_member_user_attribute){ options.groupMemberUserAttribute = ldapConfig.group_member_user_attribute } + } + // console.log(options) // ldap-authentication has bad cert check, so we check first !! if(ldapConfig.enable_tls && !(ldapConfig.ignore_certs==1)){ if(!helpers.checkCertificate(ldapConfig.cert)){ From c226223e23b281781e94fbc78c4b36bd271e5271 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:40:30 +0100 Subject: [PATCH 11/50] allow no forms to be present --- server/src/models/form.model.js | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/server/src/models/form.model.js b/server/src/models/form.model.js index fe33471c..6a909c9a 100644 --- a/server/src/models/form.model.js +++ b/server/src/models/form.model.js @@ -35,23 +35,27 @@ var Form=function(data){ // since version 4.0.3 the backups go under folder => move backups there (should be only once) Form.initBackupFolder=function(){ logger.info("Moving older form backups to new backup folder") - fs.mkdirSync(backupPath, { recursive: true }) - // move old forms.bak.files - var files = fs.readdirSync(formFilePath) - if(files){ - // filter only backup-files and folders - files=files.filter((item)=>item.match(/\.bak\.[0-9]*$/)) - // read files - files.forEach((item, i) => { - try{ - const from = path.join(formFilePath,item) - const to = path.join(backupPath,item) - logger.debug(`moving ${from} -> ${to}`) - fse.moveSync(from,to) - }catch(e){ - logger.error(`failed to move item '${item}'.\n${e}`) - } - }); + try{ + fs.mkdirSync(backupPath, { recursive: true }) + // move old forms.bak.files + var files = fs.readdirSync(formFilePath) + if(files){ + // filter only backup-files and folders + files=files.filter((item)=>item.match(/\.bak\.[0-9]*$/)) + // read files + files.forEach((item, i) => { + try{ + const from = path.join(formFilePath,item) + const to = path.join(backupPath,item) + logger.debug(`moving ${from} -> ${to}`) + fse.moveSync(from,to) + }catch(e){ + logger.error(`failed to move item '${item}'.\n`,e) + } + }); + } + }catch(e){ + logger.error("Failed to init backup folder\n",e) } Form.removeOld(oldBackupDays) } From d56acf489a039cc78d6d673878cbf34e809be393 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:41:08 +0100 Subject: [PATCH 12/50] a hidden check page --- server/src/models/install.model.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/server/src/models/install.model.js b/server/src/models/install.model.js index 1a1d3147..03afe18f 100644 --- a/server/src/models/install.model.js +++ b/server/src/models/install.model.js @@ -124,21 +124,6 @@ const CheckModel = { status : 'OK', label : `Application version = ${version}` }; - var database_version = await this.getDatabaseVersion(databaseName) - summary["Database version"] = { - status : 'OK', - label : `Database version = ${database_version}` - }; - if(database_version == 'Failed' || database_version!=version){ - if(database_version == 'Failed') - database_version = "UNKNOWN" - summary["Database version"].status = "Failed" - summary["Database version"].label = `Database version = ${database_version}` - summary["Application version"].status = "Failed" - summary["Application version"].label = `Application version = ${version} <-> ${database_version}` - } - - try { From a490620e3f974102bdf8b4f642c432e1f5b09040 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Tue, 12 Dec 2023 14:45:55 +0100 Subject: [PATCH 13/50] improved repo control --- client/src/views/Repos.vue | 113 ++++++++++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 21 deletions(-) diff --git a/client/src/views/Repos.vue b/client/src/views/Repos.vue index f462e238..9c584ee7 100644 --- a/client/src/views/Repos.vue +++ b/client/src/views/Repos.vue @@ -1,5 +1,8 @@ @@ -67,6 +71,7 @@ labels:{type:Array}, columns:{type:Array}, filters:{type:Array,default:()=>{return []}}, + icons:{type:Array,default:()=>{return []}}, actions:{type:Array}, identifier:{type:String}, currentItem:{type:[String,Number]}, From 5b8f7bd83907040041b04bd73c6c21f84d248a4d Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:11:19 +0100 Subject: [PATCH 25/50] add BASE_URL --- server/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/index.js b/server/index.js index b5e4687e..dabe89ff 100644 --- a/server/index.js +++ b/server/index.js @@ -16,7 +16,7 @@ app.use(express.static(publicPath, staticConf)) // allow browser history const history = require('connect-history-api-fallback') -app.use('/', history()) +app.use(`${appconfig.baseUrl}`, history()) // choose whether to start https or http server var httpServer From db1f19624082afe64d00aedaf4a48746ceef40f6 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:11:43 +0100 Subject: [PATCH 26/50] add silent mode to mysql query --- server/src/models/db.model.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/server/src/models/db.model.js b/server/src/models/db.model.js index 362ce14d..65fcefa5 100644 --- a/server/src/models/db.model.js +++ b/server/src/models/db.model.js @@ -3,19 +3,15 @@ const logger = require('../lib/logger'); const dbConfig = require('../../config/db.config') const client = require('mysql2'); -// mysql2 has a bug that can throw an uncaught exception if the mysql server crashes (not enough mem for example) -process.on('uncaughtException', function(err) { - // handle the error safely - logger.error("An uncaught exception happened in db.model.js. ",err) -}) - dbConfig.multipleStatements=true delete dbConfig.name // remove unsupported property MySql = {} -MySql.do=function(query,vars){ +MySql.do=function(query,vars,silent=false){ return new Promise((resolve,reject) => { - logger.info("[ansibleforms] running query : " + query) + if(!silent){ + logger.info("[ansibleforms] running query : " + query) + } var conn try{ var conn = client.createConnection(dbConfig) @@ -33,7 +29,9 @@ MySql.do=function(query,vars){ logger.error("[ansibleforms] Query error : " + err) reject(err) }else{ - logger.debug("[ansibleforms] query result : " + JSON.stringify(result)) + if(!silent){ + logger.debug("[ansibleforms] query result : " + JSON.stringify(result)) + } resolve(result) } }) From de7668d18a16fdcaf2d306cc072711b246b4b5eb Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:12:04 +0100 Subject: [PATCH 27/50] fix small bug --- server/src/lib/crypto.js | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/server/src/lib/crypto.js b/server/src/lib/crypto.js index 532262d2..89a3b5ea 100644 --- a/server/src/lib/crypto.js +++ b/server/src/lib/crypto.js @@ -6,24 +6,26 @@ const algorithm = 'aes-256-ctr'; const iv = crypto.randomBytes(16); const encrypt = (text) => { - - const cipher = crypto.createCipheriv(algorithm, appConfig.encryptionSecret, iv); - - const encrypted = Buffer.concat([cipher.update(text), cipher.final()]); - - return encrypted.toString('hex') + "." + iv.toString('hex') - + if(text){ + const cipher = crypto.createCipheriv(algorithm, appConfig.encryptionSecret, iv); + const encrypted = Buffer.concat([cipher.update(text), cipher.final()]); + return encrypted.toString('hex') + "." + iv.toString('hex') + }else{ + return "" + } }; const decrypt = (hash) => { - const tmp = hash.split(".") - const hash2 = {content:tmp[0],iv:tmp[1]} - - const decipher = crypto.createDecipheriv(algorithm, appConfig.encryptionSecret, Buffer.from(hash2.iv, 'hex')); - - const decrpyted = Buffer.concat([decipher.update(Buffer.from(hash2.content, 'hex')), decipher.final()]); + if(hash){ + const tmp = hash.split(".") + const hash2 = {content:tmp[0],iv:tmp[1]} + const decipher = crypto.createDecipheriv(algorithm, appConfig.encryptionSecret, Buffer.from(hash2.iv, 'hex')); + const decrpyted = Buffer.concat([decipher.update(Buffer.from(hash2.content, 'hex')), decipher.final()]); + return decrpyted.toString(); + }else{ + return "" + } - return decrpyted.toString(); }; // promise wrapper for bcrypthash const hashPassword = (pw) => { From a5bc3b0738161336c4bfce50a257eb0d8a2d9dc8 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:12:30 +0100 Subject: [PATCH 28/50] extend error message with prefix --- server/src/lib/common.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/src/lib/common.js b/server/src/lib/common.js index a1db6039..8153f51a 100644 --- a/server/src/lib/common.js +++ b/server/src/lib/common.js @@ -12,20 +12,23 @@ Helpers.checkCertificateBase64=function(cert){ return (Buffer.from(b64, 'base64').toString('base64') === b64) } -Helpers.getError=function(err){ +Helpers.getError=function(err,prefix=""){ + var m = undefined if(err){ if(err.message){ - return err.message + m=err.message }else{ if(typeof err == 'string'){ - return err + m=err }else{ - return 'Could not extract error from err object' + m='Could not extract error from err object' } } - }else{ - return undefined } + if(prefix){ + return `${prefix} : ${m}` + } + return m } Helpers.escapeStringForCommandLine=function(value) { From 3ad6e1ecc71c5c8f9a59a6047a3efbaa88e1f400 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:12:52 +0100 Subject: [PATCH 29/50] add repositories --- server/src/init/index.js | 41 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/server/src/init/index.js b/server/src/init/index.js index 6068a027..2b8e3dcc 100644 --- a/server/src/init/index.js +++ b/server/src/init/index.js @@ -2,7 +2,10 @@ const logger=require("../lib/logger"); var Ssh = require('../models/ssh.model'); var Form = require('../models/form.model'); var Job = require('../models/job.model'); - +const mysql=require("../models/db.model"); +const Repository = require('../models/repository.model'); +const parser = require("cron-parser") +const dayjs = require("dayjs") Ssh.generate(false) .catch((err)=>{ @@ -30,3 +33,39 @@ setInterval(()=>{ logger.error("Failed to abandon jobs : " + err) }) },3600000) + +logger.info("Pulling repositories") +mysql.do("SELECT name FROM AnsibleForms.`repositories` WHERE rebase_on_start=1") +.then((repositories)=>{ + repositories.map((repo)=>{ + logger.info("Pulling " + repo.name) + Repository.clone(repo.name).catch((e)=>{}) + }) +}) +.catch((e)=>{}) + +logger.info("Initializing repository cron schedules") +// this is hourly, abandon running jobs older than a day. +setInterval(()=>{ + mysql.do("SELECT name,cron FROM AnsibleForms.`repositories` WHERE status<>'running' AND cron<>''",undefined,true) + .then((repositories)=>{ + repositories.map((repo)=>{ + try{ + const interval = parser.parseExpression(repo.cron) + const next = interval.next().toDate() + const date = dayjs(next) + const now = dayjs() + const minutes = date.diff(now,'m') + if(minutes==0){ + Repository.pull(repo.name) + }else{ + // logger.debug(`Not time yet, ${minutes} minutes to go`) + } + }catch(e){ + logger.error(`Failed to parse cron schedule ${repo.cron}`) + } + + }) + }) + .catch((e)=>{}) +},56000) // run every 55 second, should hit 0 minutes once \ No newline at end of file From 2f438fc071f0d571ef771afbe4ef638485c83646 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:13:14 +0100 Subject: [PATCH 30/50] add repositories --- server/package.json | 2 +- server/src/controllers/credential.controller.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/package.json b/server/package.json index f2bad255..cc97eb7c 100644 --- a/server/package.json +++ b/server/package.json @@ -56,7 +56,7 @@ "read-last-lines": "~1.8.0", "swagger-ui-express": "~5.0.0", "nodemailer": "~6.9.7", - "nanoid": "~3.3.7" + "cron-parser": "~4.9.0" }, "devDependencies": { "@babel/core": "7.23.2", diff --git a/server/src/controllers/credential.controller.js b/server/src/controllers/credential.controller.js index e8aa4238..003eeb37 100644 --- a/server/src/controllers/credential.controller.js +++ b/server/src/controllers/credential.controller.js @@ -33,7 +33,7 @@ exports.findById = function(req, res) { if(credential.length>0){ res.json(new RestResult("success","found credential",credential[0],"")); }else{ - res.json(new RestResult("error","failed to find credential",null,err.toString())) + res.json(new RestResult("error","failed to find credential",null,"")) } }) .catch((err)=>{ res.json(new RestResult("error","failed to find credential",null,err.toString())) }) From bb60ccb017927765547dd14bdfb5824ecb45dcce Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:21:00 +0100 Subject: [PATCH 31/50] add BASE_URL --- client/src/views/Credentials.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/views/Credentials.vue b/client/src/views/Credentials.vue index 2f9bf59a..ade8fd1b 100644 --- a/client/src/views/Credentials.vue +++ b/client/src/views/Credentials.vue @@ -189,7 +189,7 @@ },newCredential(){ var ref= this; if (!this.$v.credential.$invalid) { - axios.post(`${process.env.BASE_URL}/api/v1/credential/`,this.credential,TokenStorage.getAuthentication()) + axios.post(`${process.env.BASE_URL}api/v1/credential/`,this.credential,TokenStorage.getAuthentication()) .then((result)=>{ if(result.data.status=="error"){ ref.$toast.error(result.data.message + ", " + result.data.data.error); From 7a03850bc73bee1e7af43558513fe106e6f703ab Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Mon, 18 Dec 2023 10:29:24 +0100 Subject: [PATCH 32/50] remove old repo routes --- server/src/configure.js | 2 - server/src/controllers/repo.controller.js | 71 ----------------------- server/src/routes/repo.routes.js | 16 ----- 3 files changed, 89 deletions(-) delete mode 100644 server/src/controllers/repo.controller.js delete mode 100644 server/src/routes/repo.routes.js diff --git a/server/src/configure.js b/server/src/configure.js index 9cdf1e53..337a159a 100644 --- a/server/src/configure.js +++ b/server/src/configure.js @@ -76,7 +76,6 @@ module.exports = app => { const profileRoutes = require('./routes/profile.routes') const sshRoutes = require('./routes/ssh.routes') const logRoutes = require('./routes/log.routes') - const repoRoutes = require('./routes/repo.routes') const knownhostsRoutes = require('./routes/knownhosts.routes') const helpRoutes = require('./routes/help.routes') const installRoutes = require('./routes/install.routes') @@ -139,7 +138,6 @@ module.exports = app => { app.use(`${appConfig.baseUrl}api/v1/sshkey`,cors(), authobj, checkAdminMiddleware, sshRoutes) app.use(`${appConfig.baseUrl}api/v1/awx`,cors(), authobj, checkAdminMiddleware, awxRoutes) app.use(`${appConfig.baseUrl}api/v1/log`,cors(), authobj, checkAdminMiddleware, logRoutes) - app.use(`${appConfig.baseUrl}api/v1/repo`,cors(), authobj, checkAdminMiddleware, repoRoutes) app.use(`${appConfig.baseUrl}api/v1/repository`,cors(), authobj, checkAdminMiddleware, repositoryRoutes) app.use(`${appConfig.baseUrl}api/v1/knownhosts`,cors(), authobj, checkAdminMiddleware, knownhostsRoutes) diff --git a/server/src/controllers/repo.controller.js b/server/src/controllers/repo.controller.js deleted file mode 100644 index 7c7fdc7a..00000000 --- a/server/src/controllers/repo.controller.js +++ /dev/null @@ -1,71 +0,0 @@ -'use strict'; -const Repo = require('../models/repo.model'); -var RestResult = require('../models/restResult.model'); - - -exports.find = function(req, res) { - if(req.query.name){ - Repo.findByName(req.query.name,(req.query.text=="true")) - .then((repo)=>{res.json(new RestResult("success","repository found",repo,""))}) - .catch((err)=>{res.json(new RestResult("error",err,null,err.toString()))}) - }else{ - Repo.findAll() - .then((repos)=>{res.json(new RestResult("success","repositories found",repos,""))}) - .catch((err)=>{res.json(new RestResult("error","failed to find repositories",null,err.toString()))}) - } - -}; -exports.create = function(req, res) { - //handles null error - if(req.body.constructor === Object && Object.keys(req.body).length === 0){ - res.status(400).send({ error:true, message: 'Please provide all required fields' }); - }else{ - Repo.create(req.body.uri,req.body.command) - .then((output)=>{res.json(new RestResult("success","repository created",output,""))}) - .catch((err)=>{res.json(new RestResult("error","failed to create repository",null,err.toString()))}) - } -}; -exports.addKnownHosts = function(req, res) { - //handles null error - if(req.body.constructor === Object && Object.keys(req.body).length === 0){ - res.status(400).send({ error:true, message: 'Please provide all required fields' }); - }else{ - Repo.addKnownHosts(req.body.hosts) - .then((output)=>{res.json(new RestResult("success","hosts added",output,""))}) - .catch((err)=>{res.json(new RestResult("error","failed to add hosts",null,err.toString()))}) - } -}; -exports.delete = function(req, res) { - if(req.query.name){ - Repo.delete( req.query.name) - .then(()=>{res.json(new RestResult("success","repository deleted",null,""))}) - .catch((err)=>{res.json(new RestResult("error","failed to delete repository",null,err.toString()))}) - }else{ - res.json(new RestResult("error","no repository name specified",null,"")); - } -}; - -exports.pull = async function(req, res) { - - // get the form data - var restResult = new RestResult("success","","","") - var repositoryName = req.params.repositoryName - if(!repositoryName){ - // wrong implementation -> send 400 error - res.json(new RestResult("error","no repository","","name is a required field")); - }else{ - Repo.pull(repositoryName) - .then((out)=>{ - restResult.message = "succesfully pulled repository" - restResult.data.output = out - res.json(restResult); - }) - .catch((err)=>{ - restResult.status = "error" - restResult.message = `error occured pulling repository ${repositoryName}` - restResult.data.error = err.toString() - res.json(restResult); - }) - } - -}; \ No newline at end of file diff --git a/server/src/routes/repo.routes.js b/server/src/routes/repo.routes.js deleted file mode 100644 index 92410e42..00000000 --- a/server/src/routes/repo.routes.js +++ /dev/null @@ -1,16 +0,0 @@ -const express = require('express') -const router = express.Router() -const repoController = require('../controllers/repo.controller'); - -// get repos -router.get('/', repoController.find); -// create repo -router.post('/', repoController.create); -// create repo -router.post('/known_hosts/', repoController.addKnownHosts); -// create repo -router.delete('/', repoController.delete); -// pull repo -router.post('/pull/:repositoryName', repoController.pull) - -module.exports = router From 1522c86250dcb4da495b5957be7c1ed3859e0d00 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 21 Dec 2023 16:43:58 +0100 Subject: [PATCH 33/50] fix postgres issue --- server/src/lib/postgres.js | 2 +- server/src/models/credential.model.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/lib/postgres.js b/server/src/lib/postgres.js index 778301d1..c41fa0ae 100644 --- a/server/src/lib/postgres.js +++ b/server/src/lib/postgres.js @@ -16,7 +16,7 @@ Postgres.query=function(connection_name,query){ database: creds.db_name||creds.user, port: creds.port, }; - + // console.log(config) return config }) .then((config)=>{ diff --git a/server/src/models/credential.model.js b/server/src/models/credential.model.js index 88bd8d0b..ffc0e9ff 100644 --- a/server/src/models/credential.model.js +++ b/server/src/models/credential.model.js @@ -85,7 +85,7 @@ Credential.findByName = async function (name,fallbackName="") { if (cred === undefined) { var result - var sql = "SELECT host,port,db_name,name,user,password,secure,db_type FROM AnsibleForms.`credentials` WHERE name REGEXP ?" + var sql = "SELECT host,port,db_name,name,user,password,secure,db_type,is_database FROM AnsibleForms.`credentials` WHERE name REGEXP ?" var res = await mysql.do(sql,name) if(res.length>0){ result = res[0] From 4c0eb60554af67710b9d8bf19ebb5410bb70b665 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 21 Dec 2023 16:44:37 +0100 Subject: [PATCH 34/50] prep versions --- CHANGELOG.md | 1 + client/package.json | 6 +++--- server/package.json | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f43574d..455f45cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - awx workflow template failed with 404 - ldap usernameattribute not used +- fixed database query issue for postgres ## [4.0.19] - 2023-11-22 diff --git a/client/package.json b/client/package.json index 3815eded..010d7a08 100644 --- a/client/package.json +++ b/client/package.json @@ -12,9 +12,9 @@ "lint": "vue-cli-service lint" }, "dependencies": { - "core-js": "~3.33.2", + "core-js": "~3.34.0", "vue": "~2.7.15", - "axios": "~1.6.0", + "axios": "~1.6.2", "es6-promise": "~4.2.8", "vuelidate": "~0.7.7", "vue-router": "~3.5.4", @@ -54,7 +54,7 @@ "eslint": "~6.8.0", "eslint-plugin-vue": "~6.2.2", "vue-template-compiler": "~2.6.11", - "nodemon": "~3.0.1", + "nodemon": "~3.0.2", "sass" :"~1.49.11", "sass-loader":"10.1.1" }, diff --git a/server/package.json b/server/package.json index cc97eb7c..8f0872a7 100644 --- a/server/package.json +++ b/server/package.json @@ -17,8 +17,8 @@ "clean": "rimraf dist" }, "dependencies": { - "core-js": "~3.33.2", - "axios": "~1.6.0", + "core-js": "~3.34.0", + "axios": "~1.6.2", "cors": "~2.8.5", "bcrypt": "~5.1.0", "cheerio": "~1.0.0-rc.12", @@ -29,11 +29,11 @@ "winston": "~3.11.0", "winston-syslog": "~2.7.0", "winston-daily-rotate-file": "~4.7.1", - "passport": "~0.6.0", + "passport": "~0.7.0", "modern-passport-http": "~0.3.0", "passport-jwt": "~4.0.1", "@outlinewiki/passport-azure-ad-oauth2": "~0.1.0", - "mysql2": "~3.6.2", + "mysql2": "~3.6.5", "ajv": "~6.12.6", "ajv-error-parser": "~1.0.7", "lodash": "~4.17.21", @@ -44,30 +44,30 @@ "cert-info":"~1.5.1", "thenby":"~1.3.4", "mssql":"~10.0.1", - "mongodb":"~6.2.0", + "mongodb":"~6.3.0", "pg":"~8.11.3", "ip":"1.1.8", "dayjs": "1.11.10", - "fs-extra":"~11.1.1", + "fs-extra":"~11.2.0", "node-cache":"~5.1.2", "json-bigint":"~1.0.0", "ldap-authentication": "~3.0.3", - "ldapjs": "~3.0.5", + "ldapjs": "~3.0.7", "read-last-lines": "~1.8.0", "swagger-ui-express": "~5.0.0", "nodemailer": "~6.9.7", "cron-parser": "~4.9.0" }, "devDependencies": { - "@babel/core": "7.23.2", - "@babel/cli": "~7.23.0", + "@babel/core": "7.23.6", + "@babel/cli": "~7.23.4", "@babel/node": "~7.22.19", - "@babel/eslint-parser": "7.22.15", - "nodemon": "~3.0.1", + "@babel/eslint-parser": "7.23.3", + "nodemon": "~3.0.2", "rifraf": "~2.0.3", "npm-run-all": "*", "dotenv": "~16.3.1", - "eslint": "~8.53.0" + "eslint": "~8.56.0" }, "eslintConfig": { "root": true, From 67ac6b2e83e167f9f99db132fe1a505dee64b129 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 21 Dec 2023 16:44:55 +0100 Subject: [PATCH 35/50] typo --- server/src/swagger.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/swagger.json b/server/src/swagger.json index 123a1dd2..c2ade4a7 100644 --- a/server/src/swagger.json +++ b/server/src/swagger.json @@ -35,8 +35,8 @@ "description": "Operations on ldap configuration" }, { - "name": "Azure AD config", - "description": "Operations on azure ad configuration" + "name": "MS Entra Id config", + "description": "Operations on MS Entra Id (aka azure AD) configuration" }, { "name": "credentials", @@ -2803,7 +2803,7 @@ } } }, - "/repository{repositoryName}": { + "/repository/{repositoryName}": { "get": { "tags": [ "repositories" From d051d2353a298259729d948160af6efc5db0a3e3 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 21 Dec 2023 16:45:10 +0100 Subject: [PATCH 36/50] typo --- server/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/index.js b/server/index.js index dabe89ff..72b016cd 100644 --- a/server/index.js +++ b/server/index.js @@ -16,7 +16,7 @@ app.use(express.static(publicPath, staticConf)) // allow browser history const history = require('connect-history-api-fallback') -app.use(`${appconfig.baseUrl}`, history()) +app.use(`${appConfig.baseUrl}`, history()) // choose whether to start https or http server var httpServer From 1e4235125b4163fd28e7a20dffefdbd0b0699354 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Thu, 21 Dec 2023 16:45:21 +0100 Subject: [PATCH 37/50] add repo --- client/src/views/Schema.vue | 2 +- docs/installation.md | 1 + docs/introduction.md | 5 +++-- server/src/models/install.model.js | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/client/src/views/Schema.vue b/client/src/views/Schema.vue index 432e69e2..a04a4482 100644 --- a/client/src/views/Schema.vue +++ b/client/src/views/Schema.vue @@ -26,7 +26,7 @@
-
+
Something went wrong. Most likely the database is simply not reachable.
diff --git a/docs/installation.md b/docs/installation.md index 676fb014..4acee819 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -131,6 +131,7 @@ sudo docker-compose up -d * Add users and groups * Add AWX connection * Add credentials for custom external connections such as other mysql servers or credentials for rest api's or tho pass to ansible playbooks + * Connect to git repositories and choose whether you want you forms and/or playbooks to sync with a repository ## File structure diff --git a/docs/introduction.md b/docs/introduction.md index d9bdbca9..29291236 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -35,8 +35,10 @@ page_nav: * **Job History & Log** : See the history of your jobs, abort running and relaunch * **Environment variables** : Customizable with environment variables * **Credential manager** : Securily store, get and pass credentials to playbooks +* **Repository integration** : Sync your forms config files, ansible playbooks and other required files with a git repo * **Ansible and AWX** : Forms can target a local ansible instance or AWX/Tower * **Swagger API** : Has a rest-api and Swagger documentation +* **Job History & Log** : See the history of your jobs* * **Designer** : Although the forms are NOT built using a graphical designer, a YAML based editor/designer with validation is present # Form Capabilities @@ -51,8 +53,7 @@ page_nav: * **Visualization** : Many nice visualizations, such as icons, images, colors, responsive grid-system, help descriptions, ... * **Field validations** : Many types of field validations, such min,max,regex,in, ... * **Group fields** : Group fields together, vertically and horizontally -* **Advanced output modelling** : Model your form content is as json-output in the way you want it -* **Job History & Log** : See the history of your jobs +* **Advanced output modelling** : Model your form content into objects, the way you want it * **Approval points** : Stop the execution of a form for approval * **Multistep forms** : Execute multiple playbooks in steps from a single form * **Email notifications** : Send email notifications after form execution diff --git a/server/src/models/install.model.js b/server/src/models/install.model.js index 6bf8048f..ebd6fe53 100644 --- a/server/src/models/install.model.js +++ b/server/src/models/install.model.js @@ -71,7 +71,7 @@ const CheckModel = { async performChecks() { - const requiredTables = ['groups','users','tokens','credentials','ldap','awx','job_output','jobs','settings','azuread']; + const requiredTables = ['groups','users','tokens','credentials','ldap','awx','job_output','jobs','settings','azuread','repositories']; const requiredRecords = [ {label:'Group admins',tableName:'groups',query:{'name':'admins'}}, {label:'User admin',tableName:'users',query:{'username':'admin'}}, From 37af7b81e5e10ce12d02bc0fecd4cc729fdd838d Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Fri, 19 Jan 2024 10:24:19 +0100 Subject: [PATCH 38/50] ldap authentication forked --- server/src/lib/ldap-authentication.js | 465 ++++++++++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 server/src/lib/ldap-authentication.js diff --git a/server/src/lib/ldap-authentication.js b/server/src/lib/ldap-authentication.js new file mode 100644 index 00000000..d2c26dd9 --- /dev/null +++ b/server/src/lib/ldap-authentication.js @@ -0,0 +1,465 @@ +const assert = require('assert') +const ldap = require('ldapjs') + +// convert a SearchResultEntry object in ldapjs 3.0 +// to a user object to maintain backward compatibility + +function _searchResultToUser(pojo) { + assert(pojo.type == 'SearchResultEntry') + let user = { dn: pojo.objectName } + pojo.attributes.forEach((attribute) => { + user[attribute.type] = + attribute.values.length == 1 ? attribute.values[0] : attribute.values + }) + return user +} +// bind and return the ldap client +function _ldapBind(dn, password, starttls, ldapOpts) { + return new Promise(function (resolve, reject) { + ldapOpts.connectTimeout = ldapOpts.connectTimeout || 5000 + var client = ldap.createClient(ldapOpts) + + client.on('connect', function () { + if (starttls) { + client.starttls(ldapOpts.tlsOptions, null, function (error) { + if (error) { + reject(error) + return + } + client.bind(dn, password, function (err) { + if (err) { + reject(err) + client.unbind() + return + } + ldapOpts.log && ldapOpts.log.trace('bind success!') + resolve(client) + }) + }) + } else { + client.bind(dn, password, function (err) { + if (err) { + reject(err) + client.unbind() + return + } + ldapOpts.log && ldapOpts.log.trace('bind success!') + resolve(client) + }) + } + }) + + //Fix for issue https://github.com/shaozi/ldap-authentication/issues/13 + client.on('timeout', (err) => { + reject(err) + }) + client.on('connectTimeout', (err) => { + reject(err) + }) + client.on('error', (err) => { + reject(err) + }) + + client.on('connectError', function (error) { + if (error) { + reject(error) + return + } + }) + }) +} + +// search a user and return the object +async function _searchUser( + ldapClient, + searchBase, + usernameAttribute, + username, + attributes = null +) { + return new Promise(function (resolve, reject) { + var filter = new ldap.filters.EqualityFilter({ + attribute: usernameAttribute, + value: username, + }) + let searchOptions = { + filter: filter, + scope: 'sub', + attributes: attributes, + } + if (attributes) { + searchOptions.attributes = attributes + } + ldapClient.search(searchBase, searchOptions, function (err, res) { + var user = null + if (err) { + reject(err) + ldapClient.unbind() + return + } + res.on('searchEntry', function (entry) { + user = _searchResultToUser(entry.pojo) + }) + res.on('searchReference', function (referral) { + // TODO: we don't support reference yet + // If the server was able to locate the entry referred to by the baseObject + // but could not search one or more non-local entries, + // the server may return one or more SearchResultReference messages, + // each containing a reference to another set of servers for continuing the operation. + // referral.uris + }) + res.on('error', function (err) { + reject(err) + ldapClient.unbind() + }) + res.on('end', function (result) { + if (result.status != 0) { + reject(new Error('ldap search status is not 0, search failed')) + } else { + resolve(user) + } + ldapClient.unbind() + }) + }) + }) +} + +// search a groups which user is member +async function _searchUserGroups( + ldapClient, + searchBase, + user, + groupClass, + groupMemberAttribute = 'member', + groupMemberUserAttribute = 'dn' +) { + return new Promise(function (resolve, reject) { + ldapClient.search( + searchBase, + { + filter: `(&(objectclass=${groupClass})(${groupMemberAttribute}=${user[groupMemberUserAttribute]}))`, + scope: 'sub', + }, + function (err, res) { + var groups = [] + if (err) { + reject(err) + ldapClient.unbind() + return + } + res.on('searchEntry', function (entry) { + groups.push(entry.pojo) + }) + res.on('searchReference', function (referral) {}) + res.on('error', function (err) { + reject(err) + ldapClient.unbind() + }) + res.on('end', function (result) { + if (result.status != 0) { + reject(new Error('ldap search status is not 0, search failed')) + } else { + resolve(groups) + } + ldapClient.unbind() + }) + } + ) + }) +} + +async function authenticateWithAdmin( + adminDn, + adminPassword, + userSearchBase, + usernameAttribute, + username, + userPassword, + starttls, + ldapOpts, + groupsSearchBase, + groupClass, + groupMemberAttribute = 'member', + groupMemberUserAttribute = 'dn', + attributes = null +) { + var ldapAdminClient + try { + ldapAdminClient = await _ldapBind( + adminDn, + adminPassword, + starttls, + ldapOpts + ) + } catch (error) { + throw { admin: error } + } + var user = await _searchUser( + ldapAdminClient, + userSearchBase, + usernameAttribute, + username, + attributes + ) + ldapAdminClient.unbind() + if (!user || !user.dn) { + ldapOpts.log && + ldapOpts.log.trace( + `admin did not find user! (${usernameAttribute}=${username})` + ) + throw new LdapAuthenticationError( + 'user not found or usernameAttribute is wrong' + ) + } + var userDn = user.dn + let ldapUserClient + try { + ldapUserClient = await _ldapBind(userDn, userPassword, starttls, ldapOpts) + } catch (error) { + throw error + } + ldapUserClient.unbind() + if (groupsSearchBase && groupClass && groupMemberAttribute) { + try { + ldapAdminClient = await _ldapBind( + adminDn, + adminPassword, + starttls, + ldapOpts + ) + } catch (error) { + throw error + } + var groups = await _searchUserGroups( + ldapAdminClient, + groupsSearchBase, + user, + groupClass, + groupMemberAttribute, + groupMemberUserAttribute + ) + user.groups = groups + ldapAdminClient.unbind() + } + return user +} + +async function authenticateWithUser( + userDn, + userSearchBase, + usernameAttribute, + username, + userPassword, + starttls, + ldapOpts, + groupsSearchBase, + groupClass, + groupMemberAttribute = 'member', + groupMemberUserAttribute = 'dn', + attributes = null +) { + let ldapUserClient + try { + ldapUserClient = await _ldapBind(userDn, userPassword, starttls, ldapOpts) + } catch (error) { + throw error + } + if (!usernameAttribute || !userSearchBase) { + // if usernameAttribute is not provided, no user detail is needed. + ldapUserClient.unbind() + return true + } + var user = await _searchUser( + ldapUserClient, + userSearchBase, + usernameAttribute, + username, + attributes + ) + if (!user || !user.dn) { + ldapOpts.log && + ldapOpts.log.trace( + `user logged in, but user details could not be found. (${usernameAttribute}=${username}). Probabaly wrong attribute or searchBase?` + ) + throw new LdapAuthenticationError( + 'user logged in, but user details could not be found. Probabaly usernameAttribute or userSearchBase is wrong?' + ) + } + ldapUserClient.unbind() + if (groupsSearchBase && groupClass && groupMemberAttribute) { + try { + ldapUserClient = await _ldapBind(userDn, userPassword, starttls, ldapOpts) + } catch (error) { + throw error + } + var groups = await _searchUserGroups( + ldapUserClient, + groupsSearchBase, + user, + groupClass, + groupMemberAttribute, + groupMemberUserAttribute + ) + user.groups = groups + ldapUserClient.unbind() + } + return user +} + +async function verifyUserExists( + adminDn, + adminPassword, + userSearchBase, + usernameAttribute, + username, + starttls, + ldapOpts, + groupsSearchBase, + groupClass, + groupMemberAttribute = 'member', + groupMemberUserAttribute = 'dn', + attributes = null +) { + var ldapAdminClient + try { + ldapAdminClient = await _ldapBind( + adminDn, + adminPassword, + starttls, + ldapOpts + ) + } catch (error) { + throw { admin: error } + } + var user = await _searchUser( + ldapAdminClient, + userSearchBase, + usernameAttribute, + username, + attributes + ) + ldapAdminClient.unbind() + if (!user || !user.dn) { + ldapOpts.log && + ldapOpts.log.trace( + `admin did not find user! (${usernameAttribute}=${username})` + ) + throw new LdapAuthenticationError( + 'user not found or usernameAttribute is wrong' + ) + } + if (groupsSearchBase && groupClass && groupMemberAttribute) { + try { + ldapAdminClient = await _ldapBind( + adminDn, + adminPassword, + starttls, + ldapOpts + ) + } catch (error) { + throw error + } + var groups = await _searchUserGroups( + ldapAdminClient, + groupsSearchBase, + user, + groupClass, + groupMemberAttribute, + groupMemberUserAttribute + ) + user.groups = groups + ldapAdminClient.unbind() + } + return user +} + +async function authenticate(options) { + if (!options.userDn) { + assert(options.adminDn, 'Admin mode adminDn must be provided') + assert(options.adminPassword, 'Admin mode adminPassword must be provided') + assert(options.userSearchBase, 'Admin mode userSearchBase must be provided') + assert( + options.usernameAttribute, + 'Admin mode usernameAttribute must be provided' + ) + assert(options.username, 'Admin mode username must be provided') + } else { + assert(options.userDn, 'User mode userDn must be provided') + } + assert( + options.ldapOpts && options.ldapOpts.url, + 'ldapOpts.url must be provided' + ) + if (options.verifyUserExists) { + assert(options.adminDn, 'Admin mode adminDn must be provided') + assert( + options.adminPassword, + 'adminDn and adminPassword must be both provided.' + ) + return await verifyUserExists( + options.adminDn, + options.adminPassword, + options.userSearchBase, + options.usernameAttribute, + options.username, + options.starttls, + options.ldapOpts, + options.groupsSearchBase, + options.groupClass, + options.groupMemberAttribute, + options.groupMemberUserAttribute + ) + } + assert(options.userPassword, 'userPassword must be provided') + if (options.adminDn) { + assert( + options.adminPassword, + 'adminDn and adminPassword must be both provided.' + ) + return await authenticateWithAdmin( + options.adminDn, + options.adminPassword, + options.userSearchBase, + options.usernameAttribute, + options.username, + options.userPassword, + options.starttls, + options.ldapOpts, + options.groupsSearchBase, + options.groupClass, + options.groupMemberAttribute, + options.groupMemberUserAttribute, + options.attributes + ) + } + assert(options.userDn, 'adminDn/adminPassword OR userDn must be provided') + return await authenticateWithUser( + options.userDn, + options.userSearchBase, + options.usernameAttribute, + options.username, + options.userPassword, + options.starttls, + options.ldapOpts, + options.groupsSearchBase, + options.groupClass, + options.groupMemberAttribute, + options.groupMemberUserAttribute, + options.attributes + ) +} + +class LdapAuthenticationError extends Error { + constructor(message) { + super(message) + // Ensure the name of this error is the same as the class name + this.name = this.constructor.name + // This clips the constructor invocation from the stack trace. + // It's not absolutely essential, but it does make the stack trace a little nicer. + // @see Node.js reference (bottom) + Error.captureStackTrace(this, this.constructor) + } +} + +module.exports.authenticate = authenticate +module.exports.LdapAuthenticationError = LdapAuthenticationError \ No newline at end of file From dac199c8befb9e3d45298364ea99802670558e96 Mon Sep 17 00:00:00 2001 From: Mirko Van Colen Date: Fri, 19 Jan 2024 10:24:51 +0100 Subject: [PATCH 39/50] add verbose checkbox and add instancegroups --- client/src/views/Designer.vue | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/client/src/views/Designer.vue b/client/src/views/Designer.vue index 87a3e8af..9a2fbb82 100644 --- a/client/src/views/Designer.vue +++ b/client/src/views/Designer.vue @@ -131,7 +131,7 @@ lang="yaml" theme="monokai" width="100%" - height="60vh" + height="70vh" :lazymodel="true" @dirty="formDirty=true" :options="{ @@ -256,7 +256,7 @@ />
-